| 1 | =head1 NAME | ||
| 2 | |||
| 3 | FastRobot - a robot which lets us check how long it will wait | ||
| 4 | |||
| 5 | =head1 SYNOPSIS | ||
| 6 | |||
| 7 | use FastRobot | ||
| 8 | my $ua=FastRobt::new(....) | ||
| 9 | $ua->check_wait | ||
| 10 | |||
| 11 | =head1 DESCRIPTION | ||
| 12 | |||
| 13 | Just like a RobotUA, but you can see how long it will wait for a given | ||
| 14 | link and then try another. | ||
| 15 | |||
| 16 | =cut | ||
| 17 | |||
| 18 | package LWP::FastRobot; | ||
| 19 | $REVISION=q$Revision: 1.5 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ ); | ||
| 20 | use Carp; | ||
| 21 | use LWP::RobotUA; | ||
| 22 | |||
| 23 | @ISA=qw(LWP::RobotUA); | ||
| 24 | $VERSION = sprintf("%d.%02d", q$Revision: 1.5 $ =~ /(\d+)\.(\d+)/); | ||
| 25 | |||
| 26 | require WWW::RobotRules; | ||
| 27 | require HTTP::Request; | ||
| 28 | require HTTP::Response; | ||
| 29 | |||
| 30 | =item $self->robot_check($url) | ||
| 31 | |||
| 32 | robot_check - checks whether a request will be allowed by the robot | ||
| 33 | rules but doesn't actually send a request. This has the advantage | ||
| 34 | that it means that the host_wait function is then accurate. | ||
| 35 | |||
| 36 | =cut | ||
| 37 | |||
| 38 | sub robot_check | ||
| 39 | { | ||
| 40 | my($self, $url) = @_; | ||
| 41 | |||
| 42 | LWP::Debug::trace('()'); | ||
| 43 | |||
| 44 | # Do we try to access a new server? | ||
| 45 | my $allowed = $self->{'rules'}->allowed($url); | ||
| 46 | |||
| 47 | if ($allowed < 0) { | ||
| 48 | LWP::Debug::debug("Host is not visited before, or robots.txt expired."); | ||
| 49 | # fetch "robots.txt" | ||
| 50 | my $robot_url = $url->clone; | ||
| 51 | $robot_url->path("robots.txt"); | ||
| 52 | $robot_url->query(undef); | ||
| 53 | LWP::Debug::debug("Requesting $robot_url"); | ||
| 54 | |||
| 55 | # make access to robot.txt legal since this will be a recursive call | ||
| 56 | $self->{'rules'}->parse($robot_url, ""); | ||
| 57 | |||
| 58 | my $robot_req = new HTTP::Request 'GET', $robot_url; | ||
| 59 | my $robot_res = $self->request($robot_req); | ||
| 60 | my $fresh_until = $robot_res->fresh_until; | ||
| 61 | if ($robot_res->is_success) { | ||
| 62 | LWP::Debug::debug("Parsing robot rules"); | ||
| 63 | $self->{'rules'}->parse($robot_url, $robot_res->content, | ||
| 64 | $fresh_until); | ||
| 65 | } else { | ||
| 66 | LWP::Debug::debug("No robots.txt file found"); | ||
| 67 | $self->{'rules'}->parse($robot_url, "", $fresh_until); | ||
| 68 | } | ||
| 69 | # recalculate allowed... | ||
| 70 | $allowed = $self->{'rules'}->allowed($url); | ||
| 71 | } | ||
| 72 | |||
| 73 | return $allowed; | ||
| 74 | |||
| 75 | } | ||
| 76 | |||
| 77 | |||
| 78 | =item $ua->host_wait($netloc) | ||
| 79 | |||
| 80 | Returns the number of seconds (from now) you must wait before you can | ||
| 81 | make a new request to this host. | ||
| 82 | |||
| 83 | =cut | ||
| 84 | |||
| 85 | sub host_wait | ||
| 86 | { | ||
| 87 | my($self, $netloc) = @_; | ||
| 88 | print STDERR "host wait called with netloc $netloc\n"; | ||
| 89 | return undef unless defined $netloc; | ||
| 90 | $self->{'no_wait'} and $netloc=~ m/$self->{'no_wait'}/ && do { | ||
| 91 | print STDERR "netloc matches no_wait regex.. zero wait\n"; | ||
| 92 | return 0; | ||
| 93 | }; | ||
| 94 | my $last = $self->{'rules'}->last_visit($netloc); | ||
| 95 | if ($last) { | ||
| 96 | my $wait = int($self->{'delay'} * 60 - (time - $last)); | ||
| 97 | print STDERR "Last visit to netloc at $last: wait $wait secs.\n"; | ||
| 98 | $wait = 0 if $wait < 0; | ||
| 99 | return $wait; | ||
| 100 | } | ||
| 101 | print STDERR "Never visited netloc don't wait any seconds.\n"; | ||
| 102 | return 0; | ||
| 103 | } | ||
| 104 | |||
| 105 | =item $ua->no_wait($regex) | ||
| 106 | |||
| 107 | Sets a regular expression for links for which the robot agent should | ||
| 108 | not wait. Typically these would be local pages or servers in the same | ||
| 109 | organisation as the link checking is being carried out by. | ||
| 110 | |||
| 111 | =cut | ||
| 112 | |||
| 113 | sub no_wait { | ||
| 114 | my $self=shift; | ||
| 115 | $self->{'no_wait'}=shift; | ||
| 116 | croak "gimme a regexp" unless $self->{'no_wait'}; | ||
| 117 | print STDERR "No wait RE: " . $self->{'no_wait'} . "\n"; | ||
| 118 | } | ||
| 119 | |||
| 120 | # =item $ua->short_wait($regex) | ||
| 121 | |||
| 122 | # Sets a regular expression for links for which the robot agent should | ||
| 123 | # not wait. Typically these would be local pages or servers in the same | ||
| 124 | # organisation as the link checking is being carried out by. | ||
| 125 | |||
| 126 | # =cut | ||
| 127 | |||
| 128 | # =item $ua->short_delay($regex) | ||
| 129 | |||
| 130 | # Sets the amount of time to wait for | ||
| 131 | |||
| 132 | # =cut | ||
| 133 |