Gungho/Component/RobotRules.pm untested

filter: Statementstmt 0.0% 0 / 84 Branchbran 0.0% 0 / 16 Conditioncond 0.0% 0 / 30 MC/DCmcdc 0.0% 0 / 16 Subroutinesub 0.0% 0 / 14 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 169 CC 9 SCAR 20.4
CC
9
Cov
0%
send_request
20.0
allowed
12.0
handle_response
6.0
CRAP 7.7
1 # $Id: /mirror/gungho/lib/Gungho/Component/RobotRules.pm 31300 2007-11-29T11:51:03.339151Z lestrrat $
2 #
3 # Copyright (c) 2007 Daisuke Maki <daisuke@endeworks.jp>
4  
5 package Gungho::Component::RobotRules;
6 use strict;
7 use warnings;
8 use base qw(Gungho::Component);
9 use Gungho::Component::RobotRules::Rule;
10 use WWW::RobotRules::Parser;
11  
12 __PACKAGE__->mk_classdata($_) for qw(pending_robots_txt robot_rules_parser robot_rules_storage pending_count);
13  
14 sub setup
15 {
16 my $c = shift;
17 $c->pending_robots_txt({});
18 $c->pending_count(0);
19 $c->setup_robot_rules_storage();
20 $c->setup_robot_rules_parser();
21 $c->next::method(@_);
22 }
23  
24 sub send_request
25 {
26 my ($c, $request) = @_;
27  
28 my $allowed =
29 $request->uri->path eq '/robots.txt' ||
30 $c->allowed($request)
31 ;
32 if ($allowed == -2) {
33 $c->log->debug("[ROBOT RULES] Fetch for /robots.txt already scheduled for " . $request->original_uri->host_port);
34 } elsif ($allowed == -1) {
35 $c->log->debug("[ROBOT RULES] No robot rules found for " . $request->original_uri->host_port . ", going to fetch one");
36 } elsif ($allowed) {
37 return $c->next::method($request);
38 } else {
39 $c->log->debug("[ROBOT RULES] " . $request->uri . " is disallowed by robot rules");
40 }
41 return 0;
42 }
43  
44 sub allowed
45 {
46 my ($c, $request) = @_;
47  
48 my $rule = $c->robot_rules_storage->get_rule( $c, $request );
49 if (! $rule) {
50 if ($c->push_pending_robots_txt($request) == 0) {
51 return -2;
52 }
53  
54 my $uri = $request->original_uri;
55 $uri->path('/robots.txt');
56 $uri->query(undef);
57 $uri->fragment(undef);
58 my $req = Gungho::Request->new(GET => $uri);
59 $req->notes('auto_robot_rules' => 1);
60 $c->pushback_request( $req );
61 return -1;
62 } else {
63 return $rule->allowed( $c, $request->uri );
64 }
65 }
66  
67 sub handle_response
68 {
69 my $c = shift;
70 my ($request, $response) = @_;
71  
72 if ($request->uri->path eq '/robots.txt' && $request->notes('auto_robot_rules')) {
73 $c->log->debug("Handling robots.txt response for " . $request->uri);
74 $c->parse_robot_rules($request, $response);
75 $c->dispatch_pending_robots_txt($request);
76 return;
77 }
78  
79 $c->next::method(@_);
80 }
81  
82 sub push_pending_robots_txt
83 {
84 my ($c, $request) = @_;
85 $c->pending_count( $c->pending_count + 1 );
86 $c->log->debug(
87 "[ROBOT RULES]: Requests still pending: " . $c->pending_count
88 );
89 return $c->robot_rules_storage->push_pending_robots_txt( $c, $request );
90 }
91  
92 sub dispatch_pending_robots_txt
93 {
94 my ($c, $request) = @_;
95  
96 my $pending = $c->robot_rules_storage->get_pending_robots_txt($c, $request);
97 if ($pending && ref $pending eq 'HASH') {
98 foreach my $request (values %$pending) {
99 $c->pending_count( $c->pending_count - 1 );
100 $c->pushback_request( $request );
101 }
102 }
103 $c->log->debug(
104 "[ROBOT RULES]: Requests still pending: " . $c->pending_count
105 );
106 }
107  
108 sub setup_robot_rules_storage
109 {
110 my $c = shift;
111  
112 my $config = $c->config->{robotrules}{storage} || {};
113  
114 my $pkg = $config->{module} || 'DB_File';
115 my $pkg_config = $config->{config} || {};
116 $pkg = $c->load_gungho_module($pkg, 'Component::RobotRules::Storage');
117 my $storage = $pkg->new(%$config);
118 $storage->setup($c);
119 $c->robot_rules_storage( $storage );
120 }
121  
122 sub setup_robot_rules_parser
123 {
124 my $c = shift;
125  
126 my $config = $c->config->{robotrules}{parser} || {};
127  
128 my $pkg = $config->{module} || '+WWW::RobotRules::Parser';
129 my $pkg_config = $config->{config} || {};
130 $pkg = $c->load_gungho_module($pkg, 'Component');
131 my $parser = $pkg->new($config);
132 $c->robot_rules_parser( $parser );
133 }
134  
135 sub parse_robot_rules
136 {
137 my ($c, $request, $response) = @_;
138  
139 my $h = ($request && $response && $response->is_success && $response->content) ?
140 $c->robot_rules_parser->parse($request->original_uri, $response->content) :
141 {}
142 ;
143 $c->log->debug("Parse robot rules " . $request->uri . ": " . keys(%$h) . " rules");
144 my $rule = Gungho::Component::RobotRules::Rule->new($h);
145 $c->robot_rules_storage->put_rule($c, $request, $rule);
146 }
147  
148 1;
149  
150 =head1 NAME
151  
152 Gungho::Component::RobotRules - Respect robots.txt
153  
154 =head1 SYNOPSIS
155  
156 ---
157 components:
158 - RobotRules
159  
160 =head1 METHODS
161  
162 =head2 setup
163  
164 =head2 setup_robot_rules_parser
165  
166 =head2 setup_robot_rules_storage
167  
168 =head2 handle_response
169  
170 =head2 send_request
171  
172 =head2 allowed($request)
173  
174 Returns 1 if request is allowed to be fetched, 0 if not. -1 and -2 are returned
175 when there is a pending request to fetch /robots.txt
176  
177 =head2 dispatch_pending_robots_txt
178  
179 Dispatches requests that were pending because of a missing robots.txt entry
180  
181 =head2 push_pending_robots_txt
182  
183 Pushes a request in the wait queue for a robots.txt
184  
185 =head2 parse_robot_rules
186  
187 Parses the robot rule and stores it
188  
189 =cut