blib/lib/LWP/FastRobot.pm untested

filter: Statementstmt 0.0% 0 / 57 Branchbran 0.0% 0 / 12 Conditioncond 0.0% 0 / 6 MC/DCmcdc 0.0% 0 / 3 Subroutinesub 0.0% 0 / 8 Podpod 0.0% 0 / 3 totaltotal 0.0% 0 / 89 CC 7 SCAR 22.9
CC
7
Cov
0%
host_wait
20.0
robot_check
12.0
no_wait
6.0
CRAP 9.9
1 =head1 NAME
2  
3 FastRobot - a robot which lets us check how long it will wait
4  
5 =head1 SYNOPSIS
6  
7 use FastRobot
8 my $ua=FastRobt::new(....)
9 $ua->check_wait
10  
11 =head1 DESCRIPTION
12  
13 Just like a RobotUA, but you can see how long it will wait for a given
14 link and then try another.
15  
16 =cut
17  
18 package LWP::FastRobot;
19 $REVISION=q$Revision: 1.5 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ );
20 use Carp;
21 use LWP::RobotUA;
22  
23 @ISA=qw(LWP::RobotUA);
24 $VERSION = sprintf("%d.%02d", q$Revision: 1.5 $ =~ /(\d+)\.(\d+)/);
25  
26 require WWW::RobotRules;
27 require HTTP::Request;
28 require HTTP::Response;
29  
30 =item $self->robot_check($url)
31  
32 robot_check - checks whether a request will be allowed by the robot
33 rules but doesn't actually send a request. This has the advantage
34 that it means that the host_wait function is then accurate.
35  
36 =cut
37  
38 sub robot_check
39 {
40 my($self, $url) = @_;
41  
42 LWP::Debug::trace('()');
43  
44 # Do we try to access a new server?
45 my $allowed = $self->{'rules'}->allowed($url);
46  
47 if ($allowed < 0) {
48 LWP::Debug::debug("Host is not visited before, or robots.txt expired.");
49 # fetch "robots.txt"
50 my $robot_url = $url->clone;
51 $robot_url->path("robots.txt");
52 $robot_url->query(undef);
53 LWP::Debug::debug("Requesting $robot_url");
54  
55 # make access to robot.txt legal since this will be a recursive call
56 $self->{'rules'}->parse($robot_url, "");
57  
58 my $robot_req = new HTTP::Request 'GET', $robot_url;
59 my $robot_res = $self->request($robot_req);
60 my $fresh_until = $robot_res->fresh_until;
61 if ($robot_res->is_success) {
62 LWP::Debug::debug("Parsing robot rules");
63 $self->{'rules'}->parse($robot_url, $robot_res->content,
64 $fresh_until);
65 } else {
66 LWP::Debug::debug("No robots.txt file found");
67 $self->{'rules'}->parse($robot_url, "", $fresh_until);
68 }
69 # recalculate allowed...
70 $allowed = $self->{'rules'}->allowed($url);
71 }
72  
73 return $allowed;
74  
75 }
76  
77  
78 =item $ua->host_wait($netloc)
79  
80 Returns the number of seconds (from now) you must wait before you can
81 make a new request to this host.
82  
83 =cut
84  
85 sub host_wait
86 {
87 my($self, $netloc) = @_;
88 print STDERR "host wait called with netloc $netloc\n";
89 return undef unless defined $netloc;
90 $self->{'no_wait'} and $netloc=~ m/$self->{'no_wait'}/ && do {
91 print STDERR "netloc matches no_wait regex.. zero wait\n";
92 return 0;
93 };
94 my $last = $self->{'rules'}->last_visit($netloc);
95 if ($last) {
96 my $wait = int($self->{'delay'} * 60 - (time - $last));
97 print STDERR "Last visit to netloc at $last: wait $wait secs.\n";
98 $wait = 0 if $wait < 0;
99 return $wait;
100 }
101 print STDERR "Never visited netloc don't wait any seconds.\n";
102 return 0;
103 }
104  
105 =item $ua->no_wait($regex)
106  
107 Sets a regular expression for links for which the robot agent should
108 not wait. Typically these would be local pages or servers in the same
109 organisation as the link checking is being carried out by.
110  
111 =cut
112  
113 sub no_wait {
114 my $self=shift;
115 $self->{'no_wait'}=shift;
116 croak "gimme a regexp" unless $self->{'no_wait'};
117 print STDERR "No wait RE: " . $self->{'no_wait'} . "\n";
118 }
119  
120 # =item $ua->short_wait($regex)
121  
122 # Sets a regular expression for links for which the robot agent should
123 # not wait. Typically these would be local pages or servers in the same
124 # organisation as the link checking is being carried out by.
125  
126 # =cut
127  
128 # =item $ua->short_delay($regex)
129  
130 # Sets the amount of time to wait for
131  
132 # =cut
133