| 1 | #!/usr/local/bin/perl -w | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | test-link - test links and update the link database | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | test-link [arguments] | ||
| 10 | |||
| 11 | -V --version Give version information for this program | ||
| 12 | -h --help --usage Describe usage of this program. | ||
| 13 | --help-opt=OPTION Give help information for a given option | ||
| 14 | -v --verbose[=VERBOSITY] Give information about what the program is doing. | ||
| 15 | Set value to control what information is given. | ||
| 16 | --quite -q --silent Program should generate no output except in case of | ||
| 17 | error. | ||
| 18 | |||
| 19 | --config-file=FILENAME Load in an additional configuration file | ||
| 20 | -u --user-address=STRING Email address for user running link testing. | ||
| 21 | -H --halt-time=MINUTES stop after given number of minutes | ||
| 22 | |||
| 23 | --never-stop keep running without stopping | ||
| 24 | --no-robot Don't follow robot rules. Dangerous!!! | ||
| 25 | -w --no-waitre=NETLOC-REGEX Home HOST regex: no robot rules.. (danger?)!!! | ||
| 26 | --test-now Test links now not when scheduled (testing only) | ||
| 27 | --untested Test all links which have not been tested. | ||
| 28 | --sequential Put links into schedule in order tested (for testing) | ||
| 29 | -H --halt-time=MINUTES stop after given number of minutes | ||
| 30 | -L --latest-time=MINUTES latest time from schedule to stop | ||
| 31 | -m --max-links=INTEGER Maximum number of links to test (-1=no limit) | ||
| 32 | |||
| 33 | =head1 DESCRIPTION | ||
| 34 | |||
| 35 | This program tests links and stores the information about what it | ||
| 36 | found into the Link database. | ||
| 37 | |||
| 38 | Needs:- | ||
| 39 | |||
| 40 | * link database | ||
| 41 | * schedule database | ||
| 42 | |||
| 43 | =head1 CONFIGURATION | ||
| 44 | |||
| 45 | Configuration is done using the WWW::Link_Controller::ReadConf (3) module. | ||
| 46 | |||
| 47 | You may want to explicitly set the user name. | ||
| 48 | |||
| 49 | =head1 ROBOT BEHAVIOR | ||
| 50 | |||
| 51 | This program is designed to be a well behaved netizen.. That means | ||
| 52 | that it will try not to put alot of load on a single site. However, | ||
| 53 | the program also attempts to work efficeiently through all of the | ||
| 54 | links it has to check. | ||
| 55 | |||
| 56 | In order to achieve these goals the test-link will wait for a delay | ||
| 57 | period between checks to the same site, but it will try to re-order | ||
| 58 | it's work so that it always has some link to check. It looks ahead | ||
| 59 | up to 100 links. | ||
| 60 | |||
| 61 | Making this queue longer will probably not help with efficiency since | ||
| 62 | an overload is probably a sign that you have many links from the same | ||
| 63 | site. If that site is your own to check or you can get an arrangement | ||
| 64 | with them then you could use a regular expression to allow faster | ||
| 65 | checking. | ||
| 66 | |||
| 67 | =head1 SCHEDULING | ||
| 68 | |||
| 69 | Most of the scheduling is handled by Schedule::Softtime which provides | ||
| 70 | an `I'll get round to you when I can be bothered' scheduler. We | ||
| 71 | guarantee that we will never schedule a link earlier than I<min-time> | ||
| 72 | (defaults to a day) from now. | ||
| 73 | |||
| 74 | The suggested time is created by the link (see WWW::Link) for details. | ||
| 75 | We then check that it's at least a certain amount (hard wired to be | ||
| 76 | one day at present) into the future. | ||
| 77 | |||
| 78 | =cut | ||
| 79 | |||
| 80 | 3 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 81 | 3 | ▶ | use English; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 82 | 3 | ▶ | use vars qw($now $no_links_so_far); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 83 | |||
| 84 | 3 | ▶ | use Fcntl; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 85 | 0 | ▶ | use DB_File; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 86 | use Data::Dumper; | ||
| 87 | use MLDBM qw(DB_File); | ||
| 88 | |||
| 89 | #use LinkIndex; | ||
| 90 | use WWW::Link; | ||
| 91 | use Schedule::SoftTime; | ||
| 92 | use WWW::Link::Tester::Adaptive; | ||
| 93 | use WWW::RobotRules::AnyDBM_File; #we must cache the robots file | ||
| 94 | use LWP::NoStopRobot; | ||
| 95 | |||
| 96 | use LWP::Debug; | ||
| 97 | |||
| 98 | #Configuration - we go through %ENV, so you'd better not be running SUID | ||
| 99 | #eval to ignore if a file doesn't exist.. e.g. the system config | ||
| 100 | |||
| 101 | use WWW::Link_Controller; | ||
| 102 | use WWW::Link_Controller::Lock; | ||
| 103 | use WWW::Link_Controller::ReadConf; | ||
| 104 | |||
| 105 | use Getopt::Function qw(maketrue makevalue); | ||
| 106 | |||
| 107 | use WWW::Link_Controller::Version; | ||
| 108 | |||
| 109 | sub status_change ($); | ||
| 110 | |||
| 111 | #signal handling | ||
| 112 | |||
| 113 | #basically, we sync the database and then do what was asked for.. | ||
| 114 | #this is wrong cos we should be setting these with sigaction so that | ||
| 115 | #we don't get screwed by old SYSV signals. | ||
| 116 | |||
| 117 | $::suspend=0; | ||
| 118 | $::abort=0; | ||
| 119 | $::all_links_read=0; | ||
| 120 | $::latest_time_read=0; | ||
| 121 | @::link_queue=(); | ||
| 122 | |||
| 123 | |||
| 124 | $::suspfunc=sub {my $signal=shift; $::suspend=1; $SIG{$signal}=$::abortfunc; }; | ||
| 125 | $SIG{TSTP}=$::suspfunc; | ||
| 126 | |||
| 127 | #external termination signals | ||
| 128 | $::abortfunc=sub {my $signal=shift; $::abort=1; $SIG{$signal}=$::abortfunc; }; | ||
| 129 | my $signame; | ||
| 130 | foreach $signame (("INT", "QUIT", "HUP", "USR1", "USR2", "TERM", "PWR")) { | ||
| 131 | $SIG{$signame}=$::abortfunc; | ||
| 132 | } | ||
| 133 | |||
| 134 | #internal error condition signals... | ||
| 135 | #BUS SEGV ILL TRAP IOT FPE | ||
| 136 | #used for other things | ||
| 137 | #ALRM PIPE CHLD IO | ||
| 138 | #irrelevant | ||
| 139 | #KILL STOP CONT PROF WINCH | ||
| 140 | #dunno | ||
| 141 | # TTIN TTOU URG XCPU XFSZ VTALRM | ||
| 142 | |||
| 143 | sub check_sigs (); | ||
| 144 | sub get_first_link(); | ||
| 145 | sub get_next_link(;$); | ||
| 146 | |||
| 147 | $no_links_so_far = 0; | ||
| 148 | $::last_queue_time = 0; | ||
| 149 | $::lookforward=100; #how many links we check forward if the first in the | ||
| 150 | #queue can't run yet because we are protecting | ||
| 151 | #that host | ||
| 152 | |||
| 153 | $::min_delay=60 * 60 * 1; #minimum scheduling delay is 1 hour | ||
| 154 | #FIXME: configurable parameter?? | ||
| 155 | |||
| 156 | $::no_waitre=""; | ||
| 157 | $::verbose=0; | ||
| 158 | $::never_stop=0; | ||
| 159 | $::start_time = time(); | ||
| 160 | $::next_sched=$::start_time + 1000; | ||
| 161 | $::halt_time=undef; | ||
| 162 | $::test_now=0; | ||
| 163 | $::max_links = 1000 unless defined $::max_links; | ||
| 164 | $::max_links =~ m/^\d+$/ or die "$::max_links must be a natural number"; | ||
| 165 | $::robot=1; | ||
| 166 | $::sequential=undef; | ||
| 167 | |||
| 168 | $::opthandler = new Getopt::Function | ||
| 169 | [ "version V>version", | ||
| 170 | "usage h>usage help>usage", | ||
| 171 | "help-opt=s", | ||
| 172 | "verbose:i v>verbose", | ||
| 173 | "silent quite>silent q>silent", | ||
| 174 | "no-warn", | ||
| 175 | #the following info comes from the config file, but overriding it on the | ||
| 176 | #command line would be nice | ||
| 177 | # "link-file=s l>link-file", #do we need this??? | ||
| 178 | # "schedule-file=s s>schedule-file", #do we need this??? | ||
| 179 | "", | ||
| 180 | "config-file=s c>config-file", | ||
| 181 | "user-address=s u>user-address", | ||
| 182 | "halt-time=i H>halt-time", | ||
| 183 | "", | ||
| 184 | "never-stop", | ||
| 185 | "no-robot", #no short form.. don't want to encourage this much | ||
| 186 | "no-waitre=s w>no-waitre", | ||
| 187 | "test-now", #also don't encourage | ||
| 188 | "untested", #also don't encourage | ||
| 189 | "sequential", | ||
| 190 | "halt-time=i H>halt-time", | ||
| 191 | "max-links=i m>max-links", | ||
| 192 | ], { | ||
| 193 | "no-warn" => [ sub { $::no_warn = 1; }, | ||
| 194 | "Avoid issuing warnings about non-fatal problems." ], | ||
| 195 | "user-address" => [ \&makevalue, | ||
| 196 | "Email address for user running link testing.", | ||
| 197 | "STRING" ], | ||
| 198 | "never-stop" => [ \&maketrue, | ||
| 199 | "keep running without stopping" ], | ||
| 200 | "halt-time" => [ sub { $::halt_time=$::start_time + $::value * 60;}, | ||
| 201 | "stop after given number of minutes", | ||
| 202 | "MINUTES" ], | ||
| 203 | "no-robot" => [ \&maketrue, | ||
| 204 | "Don't follow robot rules. Dangerous!!!" ], | ||
| 205 | #FIXME no-wait | ||
| 206 | "no-waitre" => [ \&makevalue, | ||
| 207 | "Home HOST regex: no robot rules.. (danger?)!!!", | ||
| 208 | "NETLOC-REGEX" ], | ||
| 209 | "test-now" => [ \&maketrue, | ||
| 210 | "Test links now not when scheduled (testing only)" ], | ||
| 211 | "untested" => [ \&maketrue, | ||
| 212 | "Test all links which have not been tested." ], | ||
| 213 | "sequential" => [ \&maketrue, | ||
| 214 | "Put links into schedule in order tested " | ||
| 215 | . "(for testing)" ], | ||
| 216 | "max-links" => [ \&makevalue, | ||
| 217 | "Maximum number of links to test (-1=no limit)", | ||
| 218 | "INTEGER" ], | ||
| 219 | "config-file" => [ sub { | ||
| 220 | eval {require $::value}; | ||
| 221 | die "Additional config failed: $@" | ||
| 222 | if $@; | ||
| 223 | #if it's not there die anyway. | ||
| 224 | # compare ReadConf.pm | ||
| 225 | }, | ||
| 226 | "Load in an additional configuration file", | ||
| 227 | "FILENAME" ] | ||
| 228 | }; | ||
| 229 | |||
| 230 | $::opthandler->std_opts; | ||
| 231 | |||
| 232 | $::opthandler->check_opts; | ||
| 233 | |||
| 234 | $::halt_time=$::start_time unless defined $::halt_time; | ||
| 235 | |||
| 236 | sub usage() { | ||
| 237 | print <<EOF; | ||
| 238 | test-link [arguments] | ||
| 239 | |||
| 240 | EOF | ||
| 241 | $::opthandler->list_opts; | ||
| 242 | print <<EOF; | ||
| 243 | |||
| 244 | Read the link database and test those links which are due to have been | ||
| 245 | tested, exiting when the next link to be tested is due after the program | ||
| 246 | start time. | ||
| 247 | |||
| 248 | Don't use --no-robot, except for when you are doing local | ||
| 249 | testing (that is, you aren't connected to the internet proper). | ||
| 250 | |||
| 251 | Don't use --never-stop or --test-now except when you are watching what | ||
| 252 | is happening. | ||
| 253 | EOF | ||
| 254 | } | ||
| 255 | |||
| 256 | sub version() { | ||
| 257 | print <<'EOF'; | ||
| 258 | test-link version | ||
| 259 | $Id: test-link.pl,v 1.19 2002/01/18 20:32:48 mikedlr Exp $ | ||
| 260 | EOF | ||
| 261 | } | ||
| 262 | |||
| 263 | $Schedule::SoftTime::no_warn=1 if $::no_warn; | ||
| 264 | $WWW::Link_Controller::Lock::silent=1 if $::silent; | ||
| 265 | |||
| 266 | LWP::Debug::level("+trace") if $::verbose; | ||
| 267 | LWP::Debug::level("+debug") if $::verbose & 1024; | ||
| 268 | |||
| 269 | |||
| 270 | #command-line-end | ||
| 271 | |||
| 272 | die "don't accept arguments, giving up\n" if @ARGV; | ||
| 273 | |||
| 274 | die 'you must define the $::links configuration variable' | ||
| 275 | unless $::links; | ||
| 276 | |||
| 277 | print STDERR "locking link database file $::links\n" unless $::silent; | ||
| 278 | |||
| 279 | WWW::Link_Controller::Lock::lock($::links); | ||
| 280 | $::linkdbm = tie %::links, "MLDBM", $::links, O_RDWR, 0666, $DB_HASH | ||
| 281 | or die $!; | ||
| 282 | |||
| 283 | $::refresh_time=$::linkdbm{$WWW::Link_Controller::refresh_key}; | ||
| 284 | defined $::refresh_time | ||
| 285 | and ( not $::refresh_time =~ m/^(?:\d+(?:\.\d*)?|\.\d+)$/) and do { | ||
| 286 | warn "invalid update time on database $::refresh_time"; | ||
| 287 | $::refresh_time=undef; | ||
| 288 | }; | ||
| 289 | |||
| 290 | die 'you must define the $::schedule configuration variable' | ||
| 291 | unless $::schedule; | ||
| 292 | die "there is no schedule file $::schedule" unless -e $::schedule; | ||
| 293 | #FIXME: this call should not create a schedule database. Only use an | ||
| 294 | #existing one. At time of writing Schedule.pm doesn't support that. | ||
| 295 | $::sched=new Schedule::SoftTime $::schedule; | ||
| 296 | |||
| 297 | print STDERR "test-link begins @ time:$::start_time testing with " unless $::silent; | ||
| 298 | |||
| 299 | my $ua; | ||
| 300 | if ($::robot) { | ||
| 301 | print STDERR "a robot ua\n" unless $::silent; | ||
| 302 | my $rules = new WWW::RobotRules::AnyDBM_File 'my-robot/1.0', '.robot.cache'; | ||
| 303 | $ua = new LWP::NoStopRobot 'LinkControllerBot/' | ||
| 304 | . $WWW::Link_Controller::Version::VERSION, $::user_address; | ||
| 305 | $ua->delay(1); | ||
| 306 | unless ($::no_waitre eq "") { | ||
| 307 | print STDERR "NoWaitre is $::no_waitre\n" unless $::silent; | ||
| 308 | $ua->no_wait($::no_waitre); | ||
| 309 | } | ||
| 310 | } else { | ||
| 311 | print STDERR "a normal ua\n" unless $::silent; | ||
| 312 | $ua = new LWP::Auth_UA 'No_Robot_Link_Controller/' | ||
| 313 | . $WWW::Link_Controller::Version::VERSION, $::user_address; | ||
| 314 | } | ||
| 315 | |||
| 316 | |||
| 317 | my $tester = new WWW::Link::Tester::Adaptive $ua; | ||
| 318 | |||
| 319 | my ($link, $check_time); | ||
| 320 | |||
| 321 | END { | ||
| 322 | if ($no_links_so_far) { | ||
| 323 | print "test link exited at ", $now, | ||
| 324 | " having checked $no_links_so_far links\n" unless $::silent; | ||
| 325 | } | ||
| 326 | } | ||
| 327 | |||
| 328 | #main loop; reading the next link which is ready to check then | ||
| 329 | #checking it. | ||
| 330 | |||
| 331 | for ( ($link, $check_time)=get_first_link(); | ||
| 332 | defined $link ; | ||
| 333 | ($link, $check_time)=get_next_link() ) { | ||
| 334 | my $now=time(); | ||
| 335 | check_sigs(); | ||
| 336 | # unless ($::never_stop) { | ||
| 337 | # if ($check_time > $::halt_time) { | ||
| 338 | # print STDERR "exiting because finished links up to halt time\n"; | ||
| 339 | # exit 0; | ||
| 340 | # } | ||
| 341 | # } | ||
| 342 | # #FIXME database locking... | ||
| 343 | |||
| 344 | my $url=$link->url(); | ||
| 345 | my $host_wait = $ua->host_wait_url($url); | ||
| 346 | $host_wait=0 unless defined $host_wait; | ||
| 347 | my $sched_wait=$check_time - $now + 1; | ||
| 348 | my $wait = $sched_wait; | ||
| 349 | ($host_wait > $sched_wait) and ( $wait = $host_wait ); | ||
| 350 | |||
| 351 | if ($::test_now) { | ||
| 352 | print STDERR "going ahead with link.. " . | ||
| 353 | "should have waited $wait secs.\n" unless $::silent; | ||
| 354 | } else { | ||
| 355 | while ($now < $check_time) { | ||
| 356 | my $wait=$check_time - $now + 1; | ||
| 357 | print STDERR "waiting for $wait seconds to test link\n" if $::verbose; | ||
| 358 | sleep ($wait); | ||
| 359 | check_sigs(); | ||
| 360 | my $now=time(); | ||
| 361 | } | ||
| 362 | } | ||
| 363 | |||
| 364 | #for one link only testing. should do maxlinks. | ||
| 365 | $::verbose and do { | ||
| 366 | print STDERR "testing link $link "; | ||
| 367 | ( $now < $check_time ) | ||
| 368 | and print STDERR ( $check_time - $now ) . " seconds early\n"; | ||
| 369 | ( $now == $check_time ) | ||
| 370 | and print STDERR " on time\n"; | ||
| 371 | ( $now > $check_time ) | ||
| 372 | and print STDERR ( $now - $check_time ) . " seconds late\n"; | ||
| 373 | }; | ||
| 374 | |||
| 375 | print STDERR "url is ",$url,"\n" if $::verbose; | ||
| 376 | |||
| 377 | my $was_broken=$link->is_broken(); | ||
| 378 | |||
| 379 | $tester->test_link($link); | ||
| 380 | |||
| 381 | status_change($link) if ($link->is_broken() and not $was_broken); | ||
| 382 | |||
| 383 | print STDERR "updating link\n" if $::verbose;; | ||
| 384 | update_link($link); | ||
| 385 | print STDERR "rescheduling $link\n" if $::verbose; | ||
| 386 | $::sched->unschedule($check_time); | ||
| 387 | auto_schedule_link($link); | ||
| 388 | $no_links_so_far++; | ||
| 389 | |||
| 390 | if ($::max_links and $no_links_so_far == $::max_links) { | ||
| 391 | print STDERR "Checked maximum number of links. Exiting\n" | ||
| 392 | unless $::silent; | ||
| 393 | exit 0; | ||
| 394 | } | ||
| 395 | } | ||
| 396 | |||
| 397 | print STDERR "No more links to check. Exiting\n" unless $::silent; | ||
| 398 | exit 0; | ||
| 399 | |||
| 400 | exit; | ||
| 401 | |||
| 402 | sub check_sigs () { | ||
| 403 | |||
| 404 | #FIXME we aren't syncing the schedule.. this isn't critical as it | ||
| 405 | #can always be rebuilt | ||
| 406 | |||
| 407 | if ($::abort==1) { | ||
| 408 | #perhaps we should destroy it? | ||
| 409 | $::linkdbm->sync(); | ||
| 410 | die "Aborted due to signal; databases updated\n"; | ||
| 411 | } | ||
| 412 | if ($::suspend==1) { | ||
| 413 | $::linkdbm->sync(); | ||
| 414 | $::suspend=0; | ||
| 415 | print STDERR "Stopped due to user generated signal; databases updated\n"; | ||
| 416 | kill STOP => $$; | ||
| 417 | #at this stage we need to reinstall the signal handler.. what | ||
| 418 | #happens if we miss the opportunity? Fortunately nothing, we just | ||
| 419 | #get stopped, but we've already synced the database.. | ||
| 420 | } | ||
| 421 | } | ||
| 422 | |||
| 423 | |||
| 424 | #=head1 get_first_link () | ||
| 425 | |||
| 426 | sub get_first_link() { | ||
| 427 | get_next_link(1); | ||
| 428 | } | ||
| 429 | |||
| 430 | #=head2 get_next_link | ||
| 431 | # | ||
| 432 | #This function looks forward in the queue of links and decides which | ||
| 433 | #one should be checked next. It needs to know about the UserAgent so | ||
| 434 | #that it can work out how long it will take before the link can | ||
| 435 | #actually be scheduled. | ||
| 436 | # | ||
| 437 | #note that if there is a large chunk of links in a row from one host | ||
| 438 | #then we can't share round hits amoung various hosts and slow down | ||
| 439 | #alot. That's one reason why the schedule building process should | ||
| 440 | #randomly spread links into the future. | ||
| 441 | # | ||
| 442 | #=cut | ||
| 443 | |||
| 444 | sub get_next_link(;$) { | ||
| 445 | my $first=shift; #should we load the first link from the queue | ||
| 446 | my ($link, $time); | ||
| 447 | #loop checks queue is empty and then reads from database to queue if needed | ||
| 448 | CASE: { | ||
| 449 | $::verbose && ($#::link_queue > -1) and do { | ||
| 450 | print STDERR "link queue looks as follows\n"; | ||
| 451 | for (my $index=0; $index <= $#::link_queue; $index++) { | ||
| 452 | print STDERR $index . ") Link: " . $::link_queue[$index]->[0] | ||
| 453 | . " time " . $::link_queue[$index]->[1] . "\n" | ||
| 454 | . " URL: " . $::link_queue[$index]->[0]->url() . "\n"; | ||
| 455 | } | ||
| 456 | last CASE; | ||
| 457 | }; | ||
| 458 | $::verbose && print STDERR "link queue is empty\n"; | ||
| 459 | } | ||
| 460 | |||
| 461 | my $index=-1; #a counter for looking through the queue | ||
| 462 | DBREAD: while () { | ||
| 463 | #loop reads through the remainder of the queue. | ||
| 464 | QUEUER: while ($#::link_queue > $index) { | ||
| 465 | $index++; | ||
| 466 | print STDERR "index in queue is $index\n" | ||
| 467 | if $::verbose; | ||
| 468 | ($link, $time)=@{$::link_queue[$index]}; | ||
| 469 | print STDERR "considering link $link from queue\n" if $::verbose;; | ||
| 470 | if (skip_for_time($link)) { | ||
| 471 | print STDERR "link $link on queue is still waiting around\n" | ||
| 472 | if $::verbose; | ||
| 473 | next QUEUER; | ||
| 474 | } | ||
| 475 | #link from queue is ready to be checked | ||
| 476 | print STDERR "link $link on queue now ready to go\n" | ||
| 477 | if $::verbose; | ||
| 478 | splice @::link_queue, $index, 1; | ||
| 479 | return $link, $time; | ||
| 480 | } | ||
| 481 | |||
| 482 | print STDERR "none of links in queue are ready\n" if $::verbose; | ||
| 483 | |||
| 484 | $now=time(); | ||
| 485 | last DBREAD if ( $::all_links_read | ||
| 486 | or ($#::link_queue > $::lookforward) | ||
| 487 | or ($::latest_time_read > $now and (not $::never_stop)) | ||
| 488 | ); | ||
| 489 | |||
| 490 | if ($first) { | ||
| 491 | print STDERR "find first link from the schedule\n" if $::verbose; | ||
| 492 | ($link, $time)=first_link(); | ||
| 493 | $first=0; | ||
| 494 | } else { | ||
| 495 | print STDERR "find another link from the schedule\n" if $::verbose; | ||
| 496 | ($link, $time)=next_link(); | ||
| 497 | } | ||
| 498 | |||
| 499 | defined $time and do { | ||
| 500 | if ( $time < $::last_queue_time) { | ||
| 501 | warn "Queue gave out of order link"; | ||
| 502 | } else { | ||
| 503 | $::last_queue_time = $time; | ||
| 504 | } | ||
| 505 | }; | ||
| 506 | |||
| 507 | ( defined ( $link ) ) or do { | ||
| 508 | print STDERR "no more links in the schedule\n" if $::verbose; | ||
| 509 | $::all_links_read=1; | ||
| 510 | last DBREAD; | ||
| 511 | }; | ||
| 512 | $::latest_time_read=$time; | ||
| 513 | ($::latest_time_read > $now and (not $::never_stop)) and do { | ||
| 514 | print STDERR "no ready links on the queue\n" if $::verbose; | ||
| 515 | $::all_links_read=1; | ||
| 516 | last DBREAD; | ||
| 517 | }; | ||
| 518 | (($time < $::halt_time) || $::never_stop ) or do { | ||
| 519 | print STDERR "link is after halt time; not reading\n" if $::verbose; | ||
| 520 | $::all_links_read=1; | ||
| 521 | last DBREAD; | ||
| 522 | }; | ||
| 523 | #a link for the queue | ||
| 524 | |||
| 525 | #FIXME: load robot rules for this link now so we can then see schedule | ||
| 526 | |||
| 527 | push @::link_queue, [$link, $time]; | ||
| 528 | print STDERR "added\n" . $#::link_queue . ") Link: " . $link | ||
| 529 | . " time " . $time . "\n" . " URL: " . $link->url() . "\n" | ||
| 530 | if $::verbose; | ||
| 531 | } | ||
| 532 | |||
| 533 | my $ref=shift @::link_queue; | ||
| 534 | defined $ref or do { | ||
| 535 | print STDERR "no links in queue; finished working\n" | ||
| 536 | if $::verbose; | ||
| 537 | return undef; | ||
| 538 | }; | ||
| 539 | print STDERR "we will just wait for the first link from the queue..\n" | ||
| 540 | if $::verbose; | ||
| 541 | ($link, $time)=@$ref; | ||
| 542 | return $link, $time; | ||
| 543 | } | ||
| 544 | |||
| 545 | sub skip_for_time () { | ||
| 546 | $::robot or return 0; | ||
| 547 | my $link=shift; | ||
| 548 | my $url; | ||
| 549 | $url = URI->new($link->url); | ||
| 550 | #sometimes this accesses the robots.txt so slow and difficult | ||
| 551 | check_sigs(); | ||
| 552 | print STDERR "about to check for robots access\n" if $::verbose; | ||
| 553 | my $scheme = $url->scheme(); | ||
| 554 | ($scheme eq "http" or $scheme eq "ftp") or return 0; | ||
| 555 | $ua->robot_check($url); | ||
| 556 | check_sigs(); | ||
| 557 | my $host_wait = $ua->host_wait_url($url); | ||
| 558 | $host_wait=0 unless defined $host_wait; | ||
| 559 | print STDERR "host_wait gives ", $host_wait, " seconds.\n"; | ||
| 560 | CASE: { | ||
| 561 | ( $host_wait > 5) && do { | ||
| 562 | #the link is going to have to wait a while anyway | ||
| 563 | print STDERR "Link ", $link->url(), " is waiting for net access.\n"; | ||
| 564 | return 1; | ||
| 565 | }; | ||
| 566 | ( $host_wait > 0) && do { | ||
| 567 | print STDERR "Link ", $link->url(), | ||
| 568 | " is ready to go in $host_wait secs.\n"; | ||
| 569 | return 0; | ||
| 570 | }; | ||
| 571 | print STDERR "Link ", $link->url(), " is ready to go.\n"; | ||
| 572 | return 0; | ||
| 573 | } | ||
| 574 | } | ||
| 575 | |||
| 576 | # =head2 first_link | ||
| 577 | |||
| 578 | # Returns the first link that needs to be checked. | ||
| 579 | |||
| 580 | # =cut | ||
| 581 | |||
| 582 | sub first_link { | ||
| 583 | my $time; | ||
| 584 | my $name; | ||
| 585 | my $link; | ||
| 586 | ($time, $name)=$::sched->first_item(); | ||
| 587 | return undef unless defined $time; # out of items | ||
| 588 | $link=$::links{$name}; | ||
| 589 | CASE: { | ||
| 590 | ! (defined $link) && do { | ||
| 591 | #the link has probably been deleted from the database | ||
| 592 | warn "non existant link scheduled for checking"; | ||
| 593 | #what happens if you remove the cursor item?? oh well | ||
| 594 | ($link, $time)=next_link(); | ||
| 595 | last CASE; | ||
| 596 | }; | ||
| 597 | $::untested && $link->is_not_checked() and do { | ||
| 598 | ($link, $time)=next_link(); | ||
| 599 | last CASE; | ||
| 600 | } | ||
| 601 | } | ||
| 602 | return $link, $time; #can be undefined.. | ||
| 603 | } | ||
| 604 | |||
| 605 | # =head2 next_link | ||
| 606 | |||
| 607 | # Returns the next link that needs to be checked. | ||
| 608 | |||
| 609 | # =cut | ||
| 610 | |||
| 611 | sub DAY () {60*60*24;} | ||
| 612 | |||
| 613 | sub next_link { | ||
| 614 | my $self=shift; | ||
| 615 | my $time; | ||
| 616 | my $name; | ||
| 617 | my $link; | ||
| 618 | while () { | ||
| 619 | ($time, $name)=$::sched->next_item(); | ||
| 620 | return undef unless defined $time; # out of items | ||
| 621 | $link=$::links{$name}; | ||
| 622 | $::untested && (! $link->is_not_checked() ) and do { | ||
| 623 | print STDERR "link " . $link->url() . " already tested.. skipping\n" | ||
| 624 | if $::verbose; | ||
| 625 | next; | ||
| 626 | }; | ||
| 627 | defined $::refresh_time and | ||
| 628 | $link->last_refresh() < ( $::refresh_time - $::max_link_age * DAY) | ||
| 629 | and do { | ||
| 630 | print STDERR "deleting old link " . $link->url() . " from database"; | ||
| 631 | delete $::links{$name}; | ||
| 632 | next; | ||
| 633 | }; | ||
| 634 | |||
| 635 | last if defined $link; | ||
| 636 | #the link has probably been deleted from the database | ||
| 637 | warn "non existant link " . $link->url() . " scheduled for checking"; | ||
| 638 | #what happens if you remove the cursor item?? oh well | ||
| 639 | } | ||
| 640 | print STDERR "Next link: " . $link . " URL: " . $link->url() . " for time " . | ||
| 641 | $time . "\n" if $::verbose; | ||
| 642 | return $link, $time; | ||
| 643 | } | ||
| 644 | |||
| 645 | # =head2 update_link | ||
| 646 | |||
| 647 | # Updates the link in the database. | ||
| 648 | |||
| 649 | # =cut | ||
| 650 | |||
| 651 | sub update_link { | ||
| 652 | my $link=shift; | ||
| 653 | # here be danger.. this could cause multiplication of links in | ||
| 654 | # the database if the url function returns variable values.. | ||
| 655 | # it `shouldn´t´ | ||
| 656 | print STDERR "New link is " , $link, Dumper ( $link ) | ||
| 657 | if $::verbose & 16; | ||
| 658 | print STDERR "Before " , Dumper( $::links{$link->url()} ), "\n" | ||
| 659 | if $::verbose & 256; | ||
| 660 | WWW::Link_Controller::Lock::checklock(); | ||
| 661 | $::links{$link->url()} = $link; | ||
| 662 | print STDERR "After " , Dumper( $::links{$link->url()} ), "\n" | ||
| 663 | if $::verbose & 128; | ||
| 664 | #FIXME Fsync??? | ||
| 665 | } | ||
| 666 | |||
| 667 | # =head2 auto_schedule_link | ||
| 668 | |||
| 669 | # Schedules a link according to when it wants to be scheduled. We actually | ||
| 670 | # schedule the URI which should be a unique identifier. | ||
| 671 | |||
| 672 | # N.B. this is for use after a link has been tested and checks that | ||
| 673 | # the link wants to be tested in the future. | ||
| 674 | |||
| 675 | # =cut | ||
| 676 | |||
| 677 | sub auto_schedule_link { | ||
| 678 | my $link=shift; | ||
| 679 | die "usage auto_schedule_link(<link>)" unless ref $link; | ||
| 680 | my ($sched_time,$vary)=$link->time_want_test(); | ||
| 681 | die "link failed to return sched time" unless $sched_time; | ||
| 682 | my $time=time; | ||
| 683 | if ( $::sequential ) { | ||
| 684 | $sched_time=$::next_sched; | ||
| 685 | } else { | ||
| 686 | warn "time logic wrong; just tested ($time) but wants tested at $sched_time" | ||
| 687 | if $sched_time < $time; | ||
| 688 | print STDERR "Link wants test between $sched_time and " | ||
| 689 | . ( $sched_time + $vary ) . "\n" if $::verbose; | ||
| 690 | |||
| 691 | my $earliest=time() + $::min_delay; | ||
| 692 | if ( $sched_time < $earliest ) { | ||
| 693 | $sched_time = $earliest ; | ||
| 694 | print STDERR "forcing link to test at $earliest for minimum delay\n"; | ||
| 695 | } | ||
| 696 | |||
| 697 | $sched_time += rand($vary); | ||
| 698 | if ( $sched_time < $::last_queue_time ) { | ||
| 699 | $sched_time=$::last_queue_time+1 ; | ||
| 700 | print STDERR "forcing link after end of queue at $::last_queue_time\n"; | ||
| 701 | } | ||
| 702 | print STDERR " will schedule at $sched_time\n" if $::verbose; | ||
| 703 | } | ||
| 704 | $::sched->schedule(int($sched_time) , $link->url() ); | ||
| 705 | } | ||
| 706 | |||
| 707 | =head2 status log handling | ||
| 708 | |||
| 709 | During it's operation, test-link can write a log file (to a file given | ||
| 710 | in the $::link_stat_log configuration variable). This can be used to | ||
| 711 | alerts to the webmaster about newly broken links. | ||
| 712 | |||
| 713 | =cut | ||
| 714 | |||
| 715 | sub status_change ($) { | ||
| 716 | my $link=shift; | ||
| 717 | defined $::link_stat_log or return undef; | ||
| 718 | open STAT, ">$::link_stat_log" or die "couldn't open status log"; | ||
| 719 | print STAT $link->url(), "\n"; | ||
| 720 | close STAT or die "couldn't open status log"; | ||
| 721 | } | ||
| 722 | |||
| 723 | =head1 LOCKING | ||
| 724 | |||
| 725 | test-link uses a very simple application level lock to protect the | ||
| 726 | links database. If you bypass this locking it could corrupt the | ||
| 727 | database. Only other runs of test-link will follow this locking. | ||
| 728 | |||
| 729 | During a run you can run link-report, but there is in principle no | ||
| 730 | guarantee that it works properly at all. However it shouldn't | ||
| 731 | normally do any damage since it has read only access to the database. | ||
| 732 | |||
| 733 | Note that the lock is done on the links database filename. | ||
| 734 | |||
| 735 | Other programs such as build-schedule and link creation programs should not | ||
| 736 | |||
| 737 | =head1 BUGS | ||
| 738 | |||
| 739 | The locking used in the current design could be considered a bug.. | ||
| 740 | |||
| 741 | There should be a mechanism for detecting that the computer is not | ||
| 742 | connected to the network at all and aborting the run completely. This | ||
| 743 | would avoid false positive broken links. | ||
| 744 | |||
| 745 | There is a problem with redirects. The second request has to wait for | ||
| 746 | the robot rules to permit it after the first. We should allow a | ||
| 747 | number of levels of redirects without waiting... Maybe this is fixed | ||
| 748 | best with a parallel agent. | ||
| 749 | |||
| 750 | =head1 SEE ALSO | ||
| 751 | |||
| 752 | L<verify-link-control(1)>; L<extract-links(1)>; L<build-schedule> | ||
| 753 | L<link-report(1)>; L<fix-link(1)>; L<link-report.cgi(1)>; L<fix-link.cgi> | ||
| 754 | L<suggest(1)>; L<link-report.cgi(1)>; L<configure-link-control> | ||
| 755 | |||
| 756 | The LinkController manual in the distribution in HTML, info, or | ||
| 757 | postscript formats, included in the distribution. | ||
| 758 | |||
| 759 | http://scotclimb.org.uk/software/linkcont/ - the | ||
| 760 | LinkController homepage. | ||
| 761 | |||
| 762 | =cut | ||