blib/lib/LWP/NoStopRobot.pm untested

filter: Statementstmt 0.0% 0 / 151 Branchbran 0.0% 0 / 38 Conditioncond 0.0% 0 / 27 MC/DCmcdc 0.0% 0 / 17 Subroutinesub 0.0% 0 / 22 Podpod 0.0% 0 / 14 totaltotal 0.0% 0 / 269 CC 20 SCAR 25.9
CC
20
Cov
0%
_robot_get
42.0
host_wait
20.0
new
20.0
CRAP 13.3
1 =head1 NAME
2  
3 NoStopRobot - a robot doesn't stop, but remembers where it has been
4  
5 =head1 SYNOPSIS
6  
7 use NoStopRobot
8 my $ua=NoStopRobot::new(....)
9 $ua->check_wait
10  
11 =head1 DESCRIPTION
12  
13 This module implements a user agent which remembers where it has been
14 and when so that the user can avoid too fast visits, but doesn't
15 actual implement that wait.
16  
17 =head1 ROBOT LOGIC
18  
19 The robot logic implemented here is somewhat more aggressive than that
20 implemented in WWW::RobotUA. We never actually sleep in any of the
21 functions here. This means that if a request is initiated it will
22 complete with robot checks and redirects all in one go.
23  
24 Instead the user should actually implement waits outside the module
25 using the `host_wait()' method. The key benefit of this is that it is
26 possible to check which request can be run first and reorder requests
27 to work as fast as wanted whilst maintaining good load spread between
28 different sites.
29  
30 Secondly (and as a direct consequence), if there are multiple requests
31 to different sites which end up as redirects to the same site, the
32 wait time logic will not warn against this. This is reasonable since
33 each request can be considered as a separate request to a separate
34 site.
35  
36 =head1 IMPLEMENTATION NOTES
37  
38 Becuase LWP::RobotUA collapses completely when called with URLs other
39 than HTTP this is implemented over the top of LWP::UserAgent (via
40 LWP::Auth_UA) rather than as a subclass of LWP::RobotUA.
41  
42 =cut
43  
44 package LWP::NoStopRobot;
45 $REVISION=q$Revision: 1.10 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ );
46  
47 use strict;
48 use warnings;
49 use Carp;
50 use LWP::Auth_UA;
51  
52 use vars qw(@ISA $VERSION);
53  
54 @ISA=qw(LWP::Auth_UA);
55 $VERSION = sprintf("%d.%02d", q$Revision: 1.10 $ =~ /(\d+)\.(\d+)/);
56  
57 require WWW::RobotRules;
58 require HTTP::Request;
59 require HTTP::Response;
60  
61 #=head2 _robot_get
62 #
63 #this is the function which actually downloads and checks robot rules.
64 #
65 #=cut
66  
67 sub _robot_get {
68 my $self=shift;
69 my $url=shift;
70 my $allowed;
71 die " usage $self->_robot_get(<url>)" unless ref $self and defined $url;
72 $url=new URI $url unless ref $url;
73 LWP::Debug::trace('()');
74  
75 if ( $url->scheme eq "http" ) {
76 #other schemes don't have defined robot rules!!! how do we handle??
77 # Do we try to access a new server?
78 $allowed = $self->{'rules'}->allowed($url);
79  
80 LWP::Debug::debug("HTTP URL: checking robots.txt. allowed is "
81 . $allowed);
82  
83 if ($allowed < 0) {
84 #FIXME https / ftp etc??
85 LWP::Debug::debug("Host not visited before, or robots.txt expired.");
86 # fetch "robots.txt"
87 my $robot_url = $url->clone;
88 $robot_url->path("robots.txt");
89 $robot_url->query(undef);
90 LWP::Debug::debug("Requesting $robot_url");
91  
92 # make access to robot.txt legal since this will be a recursive call
93 $self->{'rules'}->parse($robot_url, "");
94  
95 my $robot_req = new HTTP::Request 'GET', $robot_url;
96  
97 #FIXME: consider carefully the following:
98 #
99 # if we get a redirect then it means that no robots.txt is
100 # present. We only interpret the standard as meaning it must be
101 # in the exact location. The reason for this is that redirects
102 # normally mean that the page is not present and send us to the
103 # local index page.
104 #
105 # otherwise we have to accept possible multiple requests just to
106 # get the robots.txt. This either takes indefinite time if we
107 # are careful or causes load if we are not.
108 #
109 # my $robot_res = $self->request($robot_req);
110  
111 my $robot_res = $self->simple_request($robot_req);
112 my $fresh_until = $robot_res->fresh_until;
113 if ($robot_res->is_success) {
114 LWP::Debug::debug("Parsing robot rules");
115 $self->{'rules'}->parse($robot_url, $robot_res->content,
116 $fresh_until);
117 } else {
118 LWP::Debug::debug("No robots.txt file found");
119 $self->{'rules'}->parse($robot_url, "", $fresh_until);
120 }
121 # recalculate allowed...
122 $allowed = $self->{'rules'}->allowed($url);
123 LWP::Debug::debug("Got final allowed value: " . $allowed);
124  
125 }
126 } else {
127 LWP::Debug::debug("can't handle robot rules on " . $url->as_string);
128 $allowed=1;
129 }
130 return $allowed;
131 }
132  
133  
134 =item $self->robot_check($url)
135  
136 robot_check - given a URL carries out all actions needed to check
137 whether a request to that URL will be allowed by the robot rules but
138 doesn't actually send a request to the URL its self. This if the
139 function host_wait is called then it will accurately reflect the time
140 before a request can be made to that URL.
141  
142 =cut
143  
144 sub robot_check
145 {
146 my($self, $url) = @_;
147 my $allowed;
148  
149 LWP::Debug::trace('()');
150  
151 # Do we try to access a new server?
152 $allowed=$self->_robot_get($url);
153  
154 return $allowed;
155  
156 }
157  
158 =head1 simple_request
159  
160 simple_request carries out one HTTP request. It does robot checks to
161 ensure that the request is permitted, however, in contrast to RobotUA
162 it never sleeps. It merely records which sites it visits.
163  
164 N.B. there is one theoretical hole in this logic. If multiple sites
165 are redirected to the same site, it is possible for us to check
166  
167 =cut
168  
169  
170 sub simple_request
171 {
172 my($self, $request, $arg, $size) = @_;
173  
174 LWP::Debug::trace('()');
175 my $url=$request->url;
176  
177 my $allowed;
178  
179 $allowed=$self->_robot_get($url);
180 # Check rules
181 unless ($allowed) {
182 LWP::Debug::trace('simple_request() failed due to robot rules');
183 return new HTTP::Response
184 &HTTP::Status::RC_FORBIDDEN, 'Forbidden by robots.txt';
185 }
186  
187 my $netloc=_url_to_netloc($url);
188  
189 CASE: {
190 not defined $netloc and do {
191 LWP::Debug::debug("_url_to_netloc failed to return host_port() for: ". $url);
192 last;
193 };
194 #this is not strictly correct english, but iproute2 uses it
195 #and so I will :-)
196 #fix this till it really checks for a hostname + port
197 # $netloc =~ m/[a-zA-Z0-9].*[a-zA-Z0-9]:[a-zA-Z0-9]+/ or do {
198 # die "host_port $netloc is a garbage: " . $url;
199 # last;
200 # };
201 # we would wait here, but we don't
202 $self->{'rules'}->visit($netloc);
203 }
204  
205 # Perform the request
206 LWP::Debug::debug("Calling super->simple request");
207 my $res = $self->SUPER::simple_request($request, $arg, $size);
208  
209 CASE: {
210 $res->is_redirect && LWP::Debug::debug( "redirect to"
211 . $res->header('Location') );
212 $res->is_success && LWP::Debug::debug( "successful request"
213 . $res->status_line() );
214 $res->is_error && LWP::Debug::debug( "error request"
215 . $res->status_line() );
216 }
217  
218 $res;
219 }
220  
221  
222 sub _url_to_netloc
223 {
224 my $url=shift;
225  
226 $url=new URI $url unless ref $url;
227  
228 my $scheme=$url->scheme();
229  
230 #basically schemes which support host and port / that is server methods
231 #FIXME: hardwired lists are BAD.
232  
233 #wierd : mailto - could be supported in certain cases
234 #excluded : data / file
235  
236 $scheme =~
237 m/ftp|gopher|http|https|ldap|news|nntp|pop|rlogin|rsync|snews|telnet/
238 or do {
239 LWP::Debug::debug("_url_to_netloc can't deal with $scheme urls");
240 return
241 };
242  
243 my $netloc=$url->host_port();
244  
245 return $netloc;
246  
247 }
248  
249 =item $ua->host_wait($url)
250  
251 This funciton is like host_wait; but there are two differences.
252 Firstly, it should be called with a url (string or object). Secondly,
253 it should work for any url (actually URI), but will return undef for
254 urls which can't have a netloc derived from them.
255  
256 =cut
257  
258 sub host_wait_url
259 {
260 my($self, $url) = @_;
261 LWP::Debug::trace("($url)");
262 die " usage $self->_robot_get(<url>)" unless ref $self and defined $url;
263 my $netloc=_url_to_netloc($url);
264 return undef unless defined $netloc;
265 my $host_wait=$self->host_wait($netloc);
266 return $host_wait;
267 }
268  
269 =item $ua->host_wait($netloc)
270  
271 Returns the number of seconds (from now) you must wait before you can
272 make a new request to this host.
273  
274 =cut
275  
276 sub host_wait
277 {
278 my($self, $netloc) = @_;
279 LWP::Debug::trace("($netloc)");
280 return undef unless defined $netloc;
281 $self->{'no_wait'} and $netloc=~ m/$self->{'no_wait'}/ && do {
282 print STDERR "netloc matches no_wait regex.. zero wait\n";
283 return 0;
284 };
285 my $last = $self->{'rules'}->last_visit($netloc);
286 if ($last) {
287 my $wait = int($self->{'delay'} * 60 - (time - $last));
288 print STDERR "Last visit to netloc at $last: wait $wait secs.\n";
289 $wait = 0 if $wait < 0;
290 return $wait;
291 }
292 print STDERR "Never visited netloc don't wait any seconds.\n";
293 return 0;
294 }
295  
296 =item $ua->no_wait($regex)
297  
298 Sets a regular expression for links for which the robot agent should
299 not wait. Typically these would be local pages or servers in the same
300 organisation as the link checking is being carried out by.
301  
302 =cut
303  
304 sub no_wait {
305 my $self=shift;
306 $self->{'no_wait'}=shift;
307 croak "gimme a regexp" unless $self->{'no_wait'};
308 LWP::Debug::trace("No wait RE: " . $self->{'no_wait'} );
309 }
310  
311 # =item $ua->short_wait($regex)
312  
313 # Sets a regular expression for links for which the robot agent should
314 # not wait. Typically these would be local pages or servers in the same
315 # organisation as the link checking is being carried out by.
316  
317 # =cut
318  
319 # =item $ua->short_delay($regex)
320  
321 # Sets the amount of time to wait for
322  
323 # =cut
324  
325  
326  
327  
328 #--------------------------------------------------------
329 #-functions straight from robot ua
330  
331  
332  
333 # # $Id: NoStopRobot.pm,v 1.10 2002/02/09 16:30:40 mikedlr Exp $
334  
335 # package LWP::RobotUA;
336  
337 # require LWP::UserAgent;
338 # @ISA = qw(LWP::UserAgent);
339 # $VERSION = sprintf("%d.%02d", q$Revision: 1.10 $ =~ /(\d+)\.(\d+)/);
340  
341 # require WWW::RobotRules;
342 # require HTTP::Request;
343 # require HTTP::Response;
344  
345 # use Carp ();
346 # use LWP::Debug ();
347 # use HTTP::Status ();
348 # use HTTP::Date qw(time2str);
349 # use strict;
350  
351 # =head1 NAME
352  
353 # LWP::RobotUA - A class for Web Robots
354  
355 # =head1 SYNOPSIS
356  
357 # require LWP::RobotUA;
358 # $ua = new LWP::RobotUA 'my-robot/0.1', 'me@foo.com';
359 # $ua->delay(10); # be very nice, go slowly
360 # ...
361 # # just use it just like a normal LWP::UserAgent
362 # $res = $ua->request($req);
363  
364 # =head1 DESCRIPTION
365  
366 # This class implements a user agent that is suitable for robot
367 # applications. Robots should be nice to the servers they visit. They
368 # should consult the F</robots.txt> file to ensure that they are welcomed
369 # and they should not make requests too frequently.
370  
371 # But, before you consider writing a robot take a look at
372 # <URL:http://info.webcrawler.com/mak/projects/robots/robots.html>.
373  
374 # When you use a I<LWP::RobotUA> as your user agent, then you do not
375 # really have to think about these things yourself. Just send requests
376 # as you do when you are using a normal I<LWP::UserAgent> and this
377 # special agent will make sure you are nice.
378  
379 # =head1 METHODS
380  
381 # The LWP::RobotUA is a sub-class of LWP::UserAgent and implements the
382 # same methods. In addition the following methods are provided:
383  
384 # =over 4
385  
386 # =cut
387  
388  
389 # #
390 # # Additional attributes in addition to those found in LWP::UserAgent:
391 # #
392 # # $self->{'delay'} Required delay between request to the same
393 # # server in minutes.
394 # #
395 # # $self->{'rules'} A WWW::RobotRules object
396 # #
397  
398  
399 =item $ua = LWP::RobotUA->new($agent_name, $from, [$rules])
400  
401 Your robot's name and the mail address of the human responsible for
402 the robot (i.e. you) are required by the constructor.
403  
404 Optionally it allows you to specify the I<WWW::RobotRules> object to
405 use.
406  
407 =cut
408  
409 sub new
410 {
411 my($class,$name,$from,$rules) = @_;
412  
413 Carp::croak('LWP::RobotUA name required') unless $name;
414 Carp::croak('LWP::RobotUA from address required') unless $from;
415  
416 my $self = new LWP::Auth_UA;
417 $self = bless $self, $class;
418  
419 $self->{'delay'} = 1; # minutes
420 $self->{'agent'} = $name;
421 $self->{'from'} = $from;
422 $self->{'use_sleep'} = 1;
423  
424 if ($rules) {
425 $rules->agent($name);
426 $self->{'rules'} = $rules;
427 } else {
428 $self->{'rules'} = new WWW::RobotRules $name;
429 }
430  
431 $self;
432 }
433  
434  
435 =item $ua->delay([$minutes])
436  
437 Set the minimum delay between requests to the same server. The
438 default is 1 minute.
439  
440 =item $ua->use_sleep([$boolean])
441  
442 Get/set a value indicating whether the UA should sleep() if requests
443 arrive too fast (before $ua->delay minutes has passed). The default is
444 TRUE. If this value is FALSE then an internal SERVICE_UNAVAILABLE
445 response will be generated. It will have an Retry-After header that
446 indicates when it is OK to send another request to this server.
447  
448 =cut
449  
450 sub delay { shift->_elem('delay', @_); }
451 sub use_sleep { shift->_elem('use_sleep', @_); }
452  
453 sub agent
454 {
455 my $self = shift;
456 my $old = $self->SUPER::agent(@_);
457 if (@_) {
458 # Changing our name means to start fresh
459 $self->{'rules'}->agent($self->{'agent'});
460 }
461 $old;
462 }
463  
464  
465 =item $ua->rules([$rules])
466  
467 Set/get which I<WWW::RobotRules> object to use.
468  
469 =cut
470  
471 sub rules {
472 my $self = shift;
473 my $old = $self->_elem('rules', @_);
474 $self->{'rules'}->agent($self->{'agent'}) if @_;
475 $old;
476 }
477  
478  
479 =item $ua->no_visits($netloc)
480  
481 Returns the number of documents fetched from this server host. Yes I
482 know, this method should probably have been named num_visits() or
483 something like that. :-(
484  
485 =cut
486  
487 sub no_visits
488 {
489 my($self, $netloc) = @_;
490 $self->{'rules'}->no_visits($netloc);
491 }
492  
493 *host_count = \&no_visits; # backwards compatibility with LWP-5.02
494  
495  
496 # =item $ua->host_wait($netloc)
497  
498 # Returns the number of seconds (from now) you must wait before you can
499 # make a new request to this host.
500  
501 # =cut
502  
503 # sub host_wait
504 # {
505 # my($self, $netloc) = @_;
506 # return undef unless defined $netloc;
507 # my $last = $self->{'rules'}->last_visit($netloc);
508 # if ($last) {
509 # my $wait = int($self->{'delay'} * 60 - (time - $last));
510 # $wait = 0 if $wait < 0;
511 # return $wait;
512 # }
513 # return 0;
514 # }
515  
516  
517  
518 # sub simple_request
519 # {
520 # my($self, $request, $arg, $size) = @_;
521  
522 # LWP::Debug::trace('()');
523  
524 # # Do we try to access a new server?
525 # my $allowed = $self->{'rules'}->allowed($request->url);
526  
527 # if ($allowed < 0) {
528 # LWP::Debug::debug("Host is not visited before, or robots.txt expired.");
529 # # fetch "robots.txt"
530 # my $robot_url = $request->url->clone;
531 # $robot_url->path("robots.txt");
532 # $robot_url->query(undef);
533 # LWP::Debug::debug("Requesting $robot_url");
534  
535 # # make access to robot.txt legal since this will be a recursive call
536 # $self->{'rules'}->parse($robot_url, "");
537  
538 # my $robot_req = new HTTP::Request 'GET', $robot_url;
539 # my $robot_res = $self->request($robot_req);
540 # my $fresh_until = $robot_res->fresh_until;
541 # if ($robot_res->is_success) {
542 # my $c = $robot_res->content;
543 # if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) {
544 # LWP::Debug::debug("Parsing robot rules");
545 # $self->{'rules'}->parse($robot_url, $c, $fresh_until);
546 # }
547 # else {
548 # LWP::Debug::debug("Ignoring robots.txt");
549 # $self->{'rules'}->parse($robot_url, "", $fresh_until);
550 # }
551  
552 # } else {
553 # LWP::Debug::debug("No robots.txt file found");
554 # $self->{'rules'}->parse($robot_url, "", $fresh_until);
555 # }
556  
557 # # recalculate allowed...
558 # $allowed = $self->{'rules'}->allowed($request->url);
559 # }
560  
561 # # Check rules
562 # unless ($allowed) {
563 # return new HTTP::Response
564 # &HTTP::Status::RC_FORBIDDEN, 'Forbidden by robots.txt';
565 # }
566  
567 # my $netloc = $request->url->host_port;
568 # my $wait = $self->host_wait($netloc);
569  
570 # if ($wait) {
571 # LWP::Debug::debug("Must wait $wait seconds");
572 # if ($self->{'use_sleep'}) {
573 # sleep($wait)
574 # } else {
575 # my $res = new HTTP::Response
576 # &HTTP::Status::RC_SERVICE_UNAVAILABLE, 'Please, slow down';
577 # $res->header('Retry-After', time2str(time + $wait));
578 # return $res;
579 # }
580 # }
581  
582 # # Perform the request
583 # my $res = $self->SUPER::simple_request($request, $arg, $size);
584  
585 # $self->{'rules'}->visit($netloc);
586  
587 # $res;
588 # }
589  
590  
591 =item $ua->as_string
592  
593 Returns a string that describes the state of the UA.
594 Mainly useful for debugging.
595  
596 =cut
597  
598 sub as_string
599 {
600 my $self = shift;
601 my @s;
602 push(@s, "Robot: $self->{'agent'} operated by $self->{'from'} [$self]");
603 push(@s, " Minimum delay: " . int($self->{'delay'}*60) . "s");
604 push(@s, " Will sleep if too early") if $self->{'use_sleep'};
605 push(@s, " Rules = $self->{'rules'}");
606 join("\n", @s, '');
607 }
608  
609 1;
610  
611 # =back
612  
613 # =head1 SEE ALSO
614  
615 # L<LWP::UserAgent>, L<WWW::RobotRules>
616  
617 # =head1 COPYRIGHT
618  
619 # Copyright 1996-2000 Gisle Aas.
620  
621 # This library is free software; you can redistribute it and/or
622 # modify it under the same terms as Perl itself.
623  
624 =cut