« blib/lib/WWW/Link_Controller/InfoStruc.pm
↑ Summary
blib/lib/WWW/Link_Controller/ReadConf/utils.pm »
| 1 | =head1 NAME | ||
| 2 | |||
| 3 | NoStopRobot - a robot doesn't stop, but remembers where it has been | ||
| 4 | |||
| 5 | =head1 SYNOPSIS | ||
| 6 | |||
| 7 | use NoStopRobot | ||
| 8 | my $ua=NoStopRobot::new(....) | ||
| 9 | $ua->check_wait | ||
| 10 | |||
| 11 | =head1 DESCRIPTION | ||
| 12 | |||
| 13 | This module implements a user agent which remembers where it has been | ||
| 14 | and when so that the user can avoid too fast visits, but doesn't | ||
| 15 | actual implement that wait. | ||
| 16 | |||
| 17 | =head1 ROBOT LOGIC | ||
| 18 | |||
| 19 | The robot logic implemented here is somewhat more aggressive than that | ||
| 20 | implemented in WWW::RobotUA. We never actually sleep in any of the | ||
| 21 | functions here. This means that if a request is initiated it will | ||
| 22 | complete with robot checks and redirects all in one go. | ||
| 23 | |||
| 24 | Instead the user should actually implement waits outside the module | ||
| 25 | using the `host_wait()' method. The key benefit of this is that it is | ||
| 26 | possible to check which request can be run first and reorder requests | ||
| 27 | to work as fast as wanted whilst maintaining good load spread between | ||
| 28 | different sites. | ||
| 29 | |||
| 30 | Secondly (and as a direct consequence), if there are multiple requests | ||
| 31 | to different sites which end up as redirects to the same site, the | ||
| 32 | wait time logic will not warn against this. This is reasonable since | ||
| 33 | each request can be considered as a separate request to a separate | ||
| 34 | site. | ||
| 35 | |||
| 36 | =head1 IMPLEMENTATION NOTES | ||
| 37 | |||
| 38 | Becuase LWP::RobotUA collapses completely when called with URLs other | ||
| 39 | than HTTP this is implemented over the top of LWP::UserAgent (via | ||
| 40 | LWP::Auth_UA) rather than as a subclass of LWP::RobotUA. | ||
| 41 | |||
| 42 | =cut | ||
| 43 | |||
| 44 | package LWP::NoStopRobot; | ||
| 45 | $REVISION=q$Revision: 1.10 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ ); | ||
| 46 | |||
| 47 | use strict; | ||
| 48 | use warnings; | ||
| 49 | use Carp; | ||
| 50 | use LWP::Auth_UA; | ||
| 51 | |||
| 52 | use vars qw(@ISA $VERSION); | ||
| 53 | |||
| 54 | @ISA=qw(LWP::Auth_UA); | ||
| 55 | $VERSION = sprintf("%d.%02d", q$Revision: 1.10 $ =~ /(\d+)\.(\d+)/); | ||
| 56 | |||
| 57 | require WWW::RobotRules; | ||
| 58 | require HTTP::Request; | ||
| 59 | require HTTP::Response; | ||
| 60 | |||
| 61 | #=head2 _robot_get | ||
| 62 | # | ||
| 63 | #this is the function which actually downloads and checks robot rules. | ||
| 64 | # | ||
| 65 | #=cut | ||
| 66 | |||
| 67 | sub _robot_get { | ||
| 68 | my $self=shift; | ||
| 69 | my $url=shift; | ||
| 70 | my $allowed; | ||
| 71 | die " usage $self->_robot_get(<url>)" unless ref $self and defined $url; | ||
| 72 | $url=new URI $url unless ref $url; | ||
| 73 | LWP::Debug::trace('()'); | ||
| 74 | |||
| 75 | if ( $url->scheme eq "http" ) { | ||
| 76 | #other schemes don't have defined robot rules!!! how do we handle?? | ||
| 77 | # Do we try to access a new server? | ||
| 78 | $allowed = $self->{'rules'}->allowed($url); | ||
| 79 | |||
| 80 | LWP::Debug::debug("HTTP URL: checking robots.txt. allowed is " | ||
| 81 | . $allowed); | ||
| 82 | |||
| 83 | if ($allowed < 0) { | ||
| 84 | #FIXME https / ftp etc?? | ||
| 85 | LWP::Debug::debug("Host not visited before, or robots.txt expired."); | ||
| 86 | # fetch "robots.txt" | ||
| 87 | my $robot_url = $url->clone; | ||
| 88 | $robot_url->path("robots.txt"); | ||
| 89 | $robot_url->query(undef); | ||
| 90 | LWP::Debug::debug("Requesting $robot_url"); | ||
| 91 | |||
| 92 | # make access to robot.txt legal since this will be a recursive call | ||
| 93 | $self->{'rules'}->parse($robot_url, ""); | ||
| 94 | |||
| 95 | my $robot_req = new HTTP::Request 'GET', $robot_url; | ||
| 96 | |||
| 97 | #FIXME: consider carefully the following: | ||
| 98 | # | ||
| 99 | # if we get a redirect then it means that no robots.txt is | ||
| 100 | # present. We only interpret the standard as meaning it must be | ||
| 101 | # in the exact location. The reason for this is that redirects | ||
| 102 | # normally mean that the page is not present and send us to the | ||
| 103 | # local index page. | ||
| 104 | # | ||
| 105 | # otherwise we have to accept possible multiple requests just to | ||
| 106 | # get the robots.txt. This either takes indefinite time if we | ||
| 107 | # are careful or causes load if we are not. | ||
| 108 | # | ||
| 109 | # my $robot_res = $self->request($robot_req); | ||
| 110 | |||
| 111 | my $robot_res = $self->simple_request($robot_req); | ||
| 112 | my $fresh_until = $robot_res->fresh_until; | ||
| 113 | if ($robot_res->is_success) { | ||
| 114 | LWP::Debug::debug("Parsing robot rules"); | ||
| 115 | $self->{'rules'}->parse($robot_url, $robot_res->content, | ||
| 116 | $fresh_until); | ||
| 117 | } else { | ||
| 118 | LWP::Debug::debug("No robots.txt file found"); | ||
| 119 | $self->{'rules'}->parse($robot_url, "", $fresh_until); | ||
| 120 | } | ||
| 121 | # recalculate allowed... | ||
| 122 | $allowed = $self->{'rules'}->allowed($url); | ||
| 123 | LWP::Debug::debug("Got final allowed value: " . $allowed); | ||
| 124 | |||
| 125 | } | ||
| 126 | } else { | ||
| 127 | LWP::Debug::debug("can't handle robot rules on " . $url->as_string); | ||
| 128 | $allowed=1; | ||
| 129 | } | ||
| 130 | return $allowed; | ||
| 131 | } | ||
| 132 | |||
| 133 | |||
| 134 | =item $self->robot_check($url) | ||
| 135 | |||
| 136 | robot_check - given a URL carries out all actions needed to check | ||
| 137 | whether a request to that URL will be allowed by the robot rules but | ||
| 138 | doesn't actually send a request to the URL its self. This if the | ||
| 139 | function host_wait is called then it will accurately reflect the time | ||
| 140 | before a request can be made to that URL. | ||
| 141 | |||
| 142 | =cut | ||
| 143 | |||
| 144 | sub robot_check | ||
| 145 | { | ||
| 146 | my($self, $url) = @_; | ||
| 147 | my $allowed; | ||
| 148 | |||
| 149 | LWP::Debug::trace('()'); | ||
| 150 | |||
| 151 | # Do we try to access a new server? | ||
| 152 | $allowed=$self->_robot_get($url); | ||
| 153 | |||
| 154 | return $allowed; | ||
| 155 | |||
| 156 | } | ||
| 157 | |||
| 158 | =head1 simple_request | ||
| 159 | |||
| 160 | simple_request carries out one HTTP request. It does robot checks to | ||
| 161 | ensure that the request is permitted, however, in contrast to RobotUA | ||
| 162 | it never sleeps. It merely records which sites it visits. | ||
| 163 | |||
| 164 | N.B. there is one theoretical hole in this logic. If multiple sites | ||
| 165 | are redirected to the same site, it is possible for us to check | ||
| 166 | |||
| 167 | =cut | ||
| 168 | |||
| 169 | |||
| 170 | sub simple_request | ||
| 171 | { | ||
| 172 | my($self, $request, $arg, $size) = @_; | ||
| 173 | |||
| 174 | LWP::Debug::trace('()'); | ||
| 175 | my $url=$request->url; | ||
| 176 | |||
| 177 | my $allowed; | ||
| 178 | |||
| 179 | $allowed=$self->_robot_get($url); | ||
| 180 | # Check rules | ||
| 181 | unless ($allowed) { | ||
| 182 | LWP::Debug::trace('simple_request() failed due to robot rules'); | ||
| 183 | return new HTTP::Response | ||
| 184 | &HTTP::Status::RC_FORBIDDEN, 'Forbidden by robots.txt'; | ||
| 185 | } | ||
| 186 | |||
| 187 | my $netloc=_url_to_netloc($url); | ||
| 188 | |||
| 189 | CASE: { | ||
| 190 | not defined $netloc and do { | ||
| 191 | LWP::Debug::debug("_url_to_netloc failed to return host_port() for: ". $url); | ||
| 192 | last; | ||
| 193 | }; | ||
| 194 | #this is not strictly correct english, but iproute2 uses it | ||
| 195 | #and so I will :-) | ||
| 196 | #fix this till it really checks for a hostname + port | ||
| 197 | # $netloc =~ m/[a-zA-Z0-9].*[a-zA-Z0-9]:[a-zA-Z0-9]+/ or do { | ||
| 198 | # die "host_port $netloc is a garbage: " . $url; | ||
| 199 | # last; | ||
| 200 | # }; | ||
| 201 | # we would wait here, but we don't | ||
| 202 | $self->{'rules'}->visit($netloc); | ||
| 203 | } | ||
| 204 | |||
| 205 | # Perform the request | ||
| 206 | LWP::Debug::debug("Calling super->simple request"); | ||
| 207 | my $res = $self->SUPER::simple_request($request, $arg, $size); | ||
| 208 | |||
| 209 | CASE: { | ||
| 210 | $res->is_redirect && LWP::Debug::debug( "redirect to" | ||
| 211 | . $res->header('Location') ); | ||
| 212 | $res->is_success && LWP::Debug::debug( "successful request" | ||
| 213 | . $res->status_line() ); | ||
| 214 | $res->is_error && LWP::Debug::debug( "error request" | ||
| 215 | . $res->status_line() ); | ||
| 216 | } | ||
| 217 | |||
| 218 | $res; | ||
| 219 | } | ||
| 220 | |||
| 221 | |||
| 222 | sub _url_to_netloc | ||
| 223 | { | ||
| 224 | my $url=shift; | ||
| 225 | |||
| 226 | $url=new URI $url unless ref $url; | ||
| 227 | |||
| 228 | my $scheme=$url->scheme(); | ||
| 229 | |||
| 230 | #basically schemes which support host and port / that is server methods | ||
| 231 | #FIXME: hardwired lists are BAD. | ||
| 232 | |||
| 233 | #wierd : mailto - could be supported in certain cases | ||
| 234 | #excluded : data / file | ||
| 235 | |||
| 236 | $scheme =~ | ||
| 237 | m/ftp|gopher|http|https|ldap|news|nntp|pop|rlogin|rsync|snews|telnet/ | ||
| 238 | or do { | ||
| 239 | LWP::Debug::debug("_url_to_netloc can't deal with $scheme urls"); | ||
| 240 | return | ||
| 241 | }; | ||
| 242 | |||
| 243 | my $netloc=$url->host_port(); | ||
| 244 | |||
| 245 | return $netloc; | ||
| 246 | |||
| 247 | } | ||
| 248 | |||
| 249 | =item $ua->host_wait($url) | ||
| 250 | |||
| 251 | This funciton is like host_wait; but there are two differences. | ||
| 252 | Firstly, it should be called with a url (string or object). Secondly, | ||
| 253 | it should work for any url (actually URI), but will return undef for | ||
| 254 | urls which can't have a netloc derived from them. | ||
| 255 | |||
| 256 | =cut | ||
| 257 | |||
| 258 | sub host_wait_url | ||
| 259 | { | ||
| 260 | my($self, $url) = @_; | ||
| 261 | LWP::Debug::trace("($url)"); | ||
| 262 | die " usage $self->_robot_get(<url>)" unless ref $self and defined $url; | ||
| 263 | my $netloc=_url_to_netloc($url); | ||
| 264 | return undef unless defined $netloc; | ||
| 265 | my $host_wait=$self->host_wait($netloc); | ||
| 266 | return $host_wait; | ||
| 267 | } | ||
| 268 | |||
| 269 | =item $ua->host_wait($netloc) | ||
| 270 | |||
| 271 | Returns the number of seconds (from now) you must wait before you can | ||
| 272 | make a new request to this host. | ||
| 273 | |||
| 274 | =cut | ||
| 275 | |||
| 276 | sub host_wait | ||
| 277 | { | ||
| 278 | my($self, $netloc) = @_; | ||
| 279 | LWP::Debug::trace("($netloc)"); | ||
| 280 | return undef unless defined $netloc; | ||
| 281 | $self->{'no_wait'} and $netloc=~ m/$self->{'no_wait'}/ && do { | ||
| 282 | print STDERR "netloc matches no_wait regex.. zero wait\n"; | ||
| 283 | return 0; | ||
| 284 | }; | ||
| 285 | my $last = $self->{'rules'}->last_visit($netloc); | ||
| 286 | if ($last) { | ||
| 287 | my $wait = int($self->{'delay'} * 60 - (time - $last)); | ||
| 288 | print STDERR "Last visit to netloc at $last: wait $wait secs.\n"; | ||
| 289 | $wait = 0 if $wait < 0; | ||
| 290 | return $wait; | ||
| 291 | } | ||
| 292 | print STDERR "Never visited netloc don't wait any seconds.\n"; | ||
| 293 | return 0; | ||
| 294 | } | ||
| 295 | |||
| 296 | =item $ua->no_wait($regex) | ||
| 297 | |||
| 298 | Sets a regular expression for links for which the robot agent should | ||
| 299 | not wait. Typically these would be local pages or servers in the same | ||
| 300 | organisation as the link checking is being carried out by. | ||
| 301 | |||
| 302 | =cut | ||
| 303 | |||
| 304 | sub no_wait { | ||
| 305 | my $self=shift; | ||
| 306 | $self->{'no_wait'}=shift; | ||
| 307 | croak "gimme a regexp" unless $self->{'no_wait'}; | ||
| 308 | LWP::Debug::trace("No wait RE: " . $self->{'no_wait'} ); | ||
| 309 | } | ||
| 310 | |||
| 311 | # =item $ua->short_wait($regex) | ||
| 312 | |||
| 313 | # Sets a regular expression for links for which the robot agent should | ||
| 314 | # not wait. Typically these would be local pages or servers in the same | ||
| 315 | # organisation as the link checking is being carried out by. | ||
| 316 | |||
| 317 | # =cut | ||
| 318 | |||
| 319 | # =item $ua->short_delay($regex) | ||
| 320 | |||
| 321 | # Sets the amount of time to wait for | ||
| 322 | |||
| 323 | # =cut | ||
| 324 | |||
| 325 | |||
| 326 | |||
| 327 | |||
| 328 | #-------------------------------------------------------- | ||
| 329 | #-functions straight from robot ua | ||
| 330 | |||
| 331 | |||
| 332 | |||
| 333 | # # $Id: NoStopRobot.pm,v 1.10 2002/02/09 16:30:40 mikedlr Exp $ | ||
| 334 | |||
| 335 | # package LWP::RobotUA; | ||
| 336 | |||
| 337 | # require LWP::UserAgent; | ||
| 338 | # @ISA = qw(LWP::UserAgent); | ||
| 339 | # $VERSION = sprintf("%d.%02d", q$Revision: 1.10 $ =~ /(\d+)\.(\d+)/); | ||
| 340 | |||
| 341 | # require WWW::RobotRules; | ||
| 342 | # require HTTP::Request; | ||
| 343 | # require HTTP::Response; | ||
| 344 | |||
| 345 | # use Carp (); | ||
| 346 | # use LWP::Debug (); | ||
| 347 | # use HTTP::Status (); | ||
| 348 | # use HTTP::Date qw(time2str); | ||
| 349 | # use strict; | ||
| 350 | |||
| 351 | # =head1 NAME | ||
| 352 | |||
| 353 | # LWP::RobotUA - A class for Web Robots | ||
| 354 | |||
| 355 | # =head1 SYNOPSIS | ||
| 356 | |||
| 357 | # require LWP::RobotUA; | ||
| 358 | # $ua = new LWP::RobotUA 'my-robot/0.1', 'me@foo.com'; | ||
| 359 | # $ua->delay(10); # be very nice, go slowly | ||
| 360 | # ... | ||
| 361 | # # just use it just like a normal LWP::UserAgent | ||
| 362 | # $res = $ua->request($req); | ||
| 363 | |||
| 364 | # =head1 DESCRIPTION | ||
| 365 | |||
| 366 | # This class implements a user agent that is suitable for robot | ||
| 367 | # applications. Robots should be nice to the servers they visit. They | ||
| 368 | # should consult the F</robots.txt> file to ensure that they are welcomed | ||
| 369 | # and they should not make requests too frequently. | ||
| 370 | |||
| 371 | # But, before you consider writing a robot take a look at | ||
| 372 | # <URL:http://info.webcrawler.com/mak/projects/robots/robots.html>. | ||
| 373 | |||
| 374 | # When you use a I<LWP::RobotUA> as your user agent, then you do not | ||
| 375 | # really have to think about these things yourself. Just send requests | ||
| 376 | # as you do when you are using a normal I<LWP::UserAgent> and this | ||
| 377 | # special agent will make sure you are nice. | ||
| 378 | |||
| 379 | # =head1 METHODS | ||
| 380 | |||
| 381 | # The LWP::RobotUA is a sub-class of LWP::UserAgent and implements the | ||
| 382 | # same methods. In addition the following methods are provided: | ||
| 383 | |||
| 384 | # =over 4 | ||
| 385 | |||
| 386 | # =cut | ||
| 387 | |||
| 388 | |||
| 389 | # # | ||
| 390 | # # Additional attributes in addition to those found in LWP::UserAgent: | ||
| 391 | # # | ||
| 392 | # # $self->{'delay'} Required delay between request to the same | ||
| 393 | # # server in minutes. | ||
| 394 | # # | ||
| 395 | # # $self->{'rules'} A WWW::RobotRules object | ||
| 396 | # # | ||
| 397 | |||
| 398 | |||
| 399 | =item $ua = LWP::RobotUA->new($agent_name, $from, [$rules]) | ||
| 400 | |||
| 401 | Your robot's name and the mail address of the human responsible for | ||
| 402 | the robot (i.e. you) are required by the constructor. | ||
| 403 | |||
| 404 | Optionally it allows you to specify the I<WWW::RobotRules> object to | ||
| 405 | use. | ||
| 406 | |||
| 407 | =cut | ||
| 408 | |||
| 409 | sub new | ||
| 410 | { | ||
| 411 | my($class,$name,$from,$rules) = @_; | ||
| 412 | |||
| 413 | Carp::croak('LWP::RobotUA name required') unless $name; | ||
| 414 | Carp::croak('LWP::RobotUA from address required') unless $from; | ||
| 415 | |||
| 416 | my $self = new LWP::Auth_UA; | ||
| 417 | $self = bless $self, $class; | ||
| 418 | |||
| 419 | $self->{'delay'} = 1; # minutes | ||
| 420 | $self->{'agent'} = $name; | ||
| 421 | $self->{'from'} = $from; | ||
| 422 | $self->{'use_sleep'} = 1; | ||
| 423 | |||
| 424 | if ($rules) { | ||
| 425 | $rules->agent($name); | ||
| 426 | $self->{'rules'} = $rules; | ||
| 427 | } else { | ||
| 428 | $self->{'rules'} = new WWW::RobotRules $name; | ||
| 429 | } | ||
| 430 | |||
| 431 | $self; | ||
| 432 | } | ||
| 433 | |||
| 434 | |||
| 435 | =item $ua->delay([$minutes]) | ||
| 436 | |||
| 437 | Set the minimum delay between requests to the same server. The | ||
| 438 | default is 1 minute. | ||
| 439 | |||
| 440 | =item $ua->use_sleep([$boolean]) | ||
| 441 | |||
| 442 | Get/set a value indicating whether the UA should sleep() if requests | ||
| 443 | arrive too fast (before $ua->delay minutes has passed). The default is | ||
| 444 | TRUE. If this value is FALSE then an internal SERVICE_UNAVAILABLE | ||
| 445 | response will be generated. It will have an Retry-After header that | ||
| 446 | indicates when it is OK to send another request to this server. | ||
| 447 | |||
| 448 | =cut | ||
| 449 | |||
| 450 | sub delay { shift->_elem('delay', @_); } | ||
| 451 | sub use_sleep { shift->_elem('use_sleep', @_); } | ||
| 452 | |||
| 453 | sub agent | ||
| 454 | { | ||
| 455 | my $self = shift; | ||
| 456 | my $old = $self->SUPER::agent(@_); | ||
| 457 | if (@_) { | ||
| 458 | # Changing our name means to start fresh | ||
| 459 | $self->{'rules'}->agent($self->{'agent'}); | ||
| 460 | } | ||
| 461 | $old; | ||
| 462 | } | ||
| 463 | |||
| 464 | |||
| 465 | =item $ua->rules([$rules]) | ||
| 466 | |||
| 467 | Set/get which I<WWW::RobotRules> object to use. | ||
| 468 | |||
| 469 | =cut | ||
| 470 | |||
| 471 | sub rules { | ||
| 472 | my $self = shift; | ||
| 473 | my $old = $self->_elem('rules', @_); | ||
| 474 | $self->{'rules'}->agent($self->{'agent'}) if @_; | ||
| 475 | $old; | ||
| 476 | } | ||
| 477 | |||
| 478 | |||
| 479 | =item $ua->no_visits($netloc) | ||
| 480 | |||
| 481 | Returns the number of documents fetched from this server host. Yes I | ||
| 482 | know, this method should probably have been named num_visits() or | ||
| 483 | something like that. :-( | ||
| 484 | |||
| 485 | =cut | ||
| 486 | |||
| 487 | sub no_visits | ||
| 488 | { | ||
| 489 | my($self, $netloc) = @_; | ||
| 490 | $self->{'rules'}->no_visits($netloc); | ||
| 491 | } | ||
| 492 | |||
| 493 | *host_count = \&no_visits; # backwards compatibility with LWP-5.02 | ||
| 494 | |||
| 495 | |||
| 496 | # =item $ua->host_wait($netloc) | ||
| 497 | |||
| 498 | # Returns the number of seconds (from now) you must wait before you can | ||
| 499 | # make a new request to this host. | ||
| 500 | |||
| 501 | # =cut | ||
| 502 | |||
| 503 | # sub host_wait | ||
| 504 | # { | ||
| 505 | # my($self, $netloc) = @_; | ||
| 506 | # return undef unless defined $netloc; | ||
| 507 | # my $last = $self->{'rules'}->last_visit($netloc); | ||
| 508 | # if ($last) { | ||
| 509 | # my $wait = int($self->{'delay'} * 60 - (time - $last)); | ||
| 510 | # $wait = 0 if $wait < 0; | ||
| 511 | # return $wait; | ||
| 512 | # } | ||
| 513 | # return 0; | ||
| 514 | # } | ||
| 515 | |||
| 516 | |||
| 517 | |||
| 518 | # sub simple_request | ||
| 519 | # { | ||
| 520 | # my($self, $request, $arg, $size) = @_; | ||
| 521 | |||
| 522 | # LWP::Debug::trace('()'); | ||
| 523 | |||
| 524 | # # Do we try to access a new server? | ||
| 525 | # my $allowed = $self->{'rules'}->allowed($request->url); | ||
| 526 | |||
| 527 | # if ($allowed < 0) { | ||
| 528 | # LWP::Debug::debug("Host is not visited before, or robots.txt expired."); | ||
| 529 | # # fetch "robots.txt" | ||
| 530 | # my $robot_url = $request->url->clone; | ||
| 531 | # $robot_url->path("robots.txt"); | ||
| 532 | # $robot_url->query(undef); | ||
| 533 | # LWP::Debug::debug("Requesting $robot_url"); | ||
| 534 | |||
| 535 | # # make access to robot.txt legal since this will be a recursive call | ||
| 536 | # $self->{'rules'}->parse($robot_url, ""); | ||
| 537 | |||
| 538 | # my $robot_req = new HTTP::Request 'GET', $robot_url; | ||
| 539 | # my $robot_res = $self->request($robot_req); | ||
| 540 | # my $fresh_until = $robot_res->fresh_until; | ||
| 541 | # if ($robot_res->is_success) { | ||
| 542 | # my $c = $robot_res->content; | ||
| 543 | # if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) { | ||
| 544 | # LWP::Debug::debug("Parsing robot rules"); | ||
| 545 | # $self->{'rules'}->parse($robot_url, $c, $fresh_until); | ||
| 546 | # } | ||
| 547 | # else { | ||
| 548 | # LWP::Debug::debug("Ignoring robots.txt"); | ||
| 549 | # $self->{'rules'}->parse($robot_url, "", $fresh_until); | ||
| 550 | # } | ||
| 551 | |||
| 552 | # } else { | ||
| 553 | # LWP::Debug::debug("No robots.txt file found"); | ||
| 554 | # $self->{'rules'}->parse($robot_url, "", $fresh_until); | ||
| 555 | # } | ||
| 556 | |||
| 557 | # # recalculate allowed... | ||
| 558 | # $allowed = $self->{'rules'}->allowed($request->url); | ||
| 559 | # } | ||
| 560 | |||
| 561 | # # Check rules | ||
| 562 | # unless ($allowed) { | ||
| 563 | # return new HTTP::Response | ||
| 564 | # &HTTP::Status::RC_FORBIDDEN, 'Forbidden by robots.txt'; | ||
| 565 | # } | ||
| 566 | |||
| 567 | # my $netloc = $request->url->host_port; | ||
| 568 | # my $wait = $self->host_wait($netloc); | ||
| 569 | |||
| 570 | # if ($wait) { | ||
| 571 | # LWP::Debug::debug("Must wait $wait seconds"); | ||
| 572 | # if ($self->{'use_sleep'}) { | ||
| 573 | # sleep($wait) | ||
| 574 | # } else { | ||
| 575 | # my $res = new HTTP::Response | ||
| 576 | # &HTTP::Status::RC_SERVICE_UNAVAILABLE, 'Please, slow down'; | ||
| 577 | # $res->header('Retry-After', time2str(time + $wait)); | ||
| 578 | # return $res; | ||
| 579 | # } | ||
| 580 | # } | ||
| 581 | |||
| 582 | # # Perform the request | ||
| 583 | # my $res = $self->SUPER::simple_request($request, $arg, $size); | ||
| 584 | |||
| 585 | # $self->{'rules'}->visit($netloc); | ||
| 586 | |||
| 587 | # $res; | ||
| 588 | # } | ||
| 589 | |||
| 590 | |||
| 591 | =item $ua->as_string | ||
| 592 | |||
| 593 | Returns a string that describes the state of the UA. | ||
| 594 | Mainly useful for debugging. | ||
| 595 | |||
| 596 | =cut | ||
| 597 | |||
| 598 | sub as_string | ||
| 599 | { | ||
| 600 | my $self = shift; | ||
| 601 | my @s; | ||
| 602 | push(@s, "Robot: $self->{'agent'} operated by $self->{'from'} [$self]"); | ||
| 603 | push(@s, " Minimum delay: " . int($self->{'delay'}*60) . "s"); | ||
| 604 | push(@s, " Will sleep if too early") if $self->{'use_sleep'}; | ||
| 605 | push(@s, " Rules = $self->{'rules'}"); | ||
| 606 | join("\n", @s, ''); | ||
| 607 | } | ||
| 608 | |||
| 609 | 1; | ||
| 610 | |||
| 611 | # =back | ||
| 612 | |||
| 613 | # =head1 SEE ALSO | ||
| 614 | |||
| 615 | # L<LWP::UserAgent>, L<WWW::RobotRules> | ||
| 616 | |||
| 617 | # =head1 COPYRIGHT | ||
| 618 | |||
| 619 | # Copyright 1996-2000 Gisle Aas. | ||
| 620 | |||
| 621 | # This library is free software; you can redistribute it and/or | ||
| 622 | # modify it under the same terms as Perl itself. | ||
| 623 | |||
| 624 | =cut |