| 1 | #!/usr/local/bin/perl -w | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | build-schedule - build a schedule for link checking. | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | build-schedule [options] | ||
| 10 | |||
| 11 | -V --version Give version information for this program | ||
| 12 | -h --help --usage Describe usage of this program. | ||
| 13 | --help-opt=OPTION Give help information for a given option | ||
| 14 | -v --verbose[=VERBOSITY] Give information about what the program is doing. | ||
| 15 | Set value to control what information is given. | ||
| 16 | -l --url-list=FILENAME File with complete list of URLs to schedule | ||
| 17 | -s --schedule=FILENAME Override location of the schedule | ||
| 18 | -t --spread-time=SECONDS Time over which to spread checking; default 10 days | ||
| 19 | |||
| 20 | --config-file=FILENAME Load in an additional configuration file | ||
| 21 | |||
| 22 | =head1 DESCRIPTION | ||
| 23 | |||
| 24 | This program takes a links database and builds an entire schedule from | ||
| 25 | scratch for them. This can then be later used by daily-check-links to | ||
| 26 | check all of the links. | ||
| 27 | |||
| 28 | By default C<build-schedule> builds a schedule for all of the links in | ||
| 29 | your links database but you can give it a list of URLs in a file in | ||
| 30 | which case the schedule built will only contain those links. | ||
| 31 | |||
| 32 | The schedule tries to spread link checking into the future with more | ||
| 33 | links being checked soon and fewer later. As links are scheduled for | ||
| 34 | repeat checking this should work out to a reasonably even rate of | ||
| 35 | checking. | ||
| 36 | |||
| 37 | If there is an already a schedule file in existance, it will be read | ||
| 38 | and the times in the old database will be reused. The old database | ||
| 39 | will be renamed with a .bak extension. | ||
| 40 | |||
| 41 | =head1 EXAMPLES | ||
| 42 | |||
| 43 | build-schedule | ||
| 44 | |||
| 45 | If everything is set up correctly and you don't have any special needs | ||
| 46 | then you don't have to give any arguments. | ||
| 47 | |||
| 48 | build-schedule --ignore-links --spread-time=7200 | ||
| 49 | |||
| 50 | Causes an immediate pulse of activity which tests all of the links in | ||
| 51 | the database within the next two hours. Needless to say, this will | ||
| 52 | cause considerable amounts of network traffic and is a bad idea in | ||
| 53 | most normal situations. | ||
| 54 | |||
| 55 | =head1 BUGS | ||
| 56 | |||
| 57 | The program doesn't check the validity of its output database, so if | ||
| 58 | the output filesystem fills up things can go horrible. Check afterwards. | ||
| 59 | |||
| 60 | =head1 SEE ALSO | ||
| 61 | |||
| 62 | L<verify-link-control(1)>; L<extract-links(1)>; L<build-schedule> | ||
| 63 | L<link-report(1)>; L<fix-link(1)>; L<link-report.cgi(1)>; L<fix-link.cgi> | ||
| 64 | L<suggest(1)>; L<link-report.cgi(1)>; L<configure-link-control> | ||
| 65 | |||
| 66 | The LinkController manual in the distribution in HTML, info, or | ||
| 67 | postscript formats, included in the distribution. | ||
| 68 | |||
| 69 | http://scotclimb.org.uk/software/linkcont/ - the | ||
| 70 | LinkController homepage. | ||
| 71 | |||
| 72 | =cut | ||
| 73 | |||
| 74 | 3 | ▶ | use vars qw($sched_margin); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 75 | sub check_time ($$;\%) ; | ||
| 76 | |||
| 77 | $sched_margin=10; #earliest time we schedule a link to run | ||
| 78 | |||
| 79 | 3 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 80 | |||
| 81 | 3 | ▶ | use Fcntl; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 82 | 0 | ▶ | use DB_File; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 83 | use Data::Dumper; | ||
| 84 | use MLDBM qw(DB_File); | ||
| 85 | |||
| 86 | use WWW::Link; | ||
| 87 | use Schedule::SoftTime; | ||
| 88 | |||
| 89 | #Configuration - we go through %ENV, so you'd better not be running SUID | ||
| 90 | #eval to ignore if a file doesn't exist.. e.g. the system config | ||
| 91 | |||
| 92 | use WWW::Link_Controller; | ||
| 93 | use WWW::Link_Controller::ReadConf; | ||
| 94 | |||
| 95 | use Getopt::Function qw(maketrue makevalue); | ||
| 96 | |||
| 97 | $::verbose=0; | ||
| 98 | $::spread_time=60.0 * 60.0 * 24.0 * 10.0; | ||
| 99 | $::start_offset=0; | ||
| 100 | $::ignore_link=0; | ||
| 101 | $::ignore_db=0; | ||
| 102 | |||
| 103 | $::opthandler = new Getopt::Function | ||
| 104 | [ "version V>version", | ||
| 105 | "usage h>usage help>usage", | ||
| 106 | "help-opt=s", | ||
| 107 | "verbose:i v>verbose", | ||
| 108 | "silent quite>silent q>silent", | ||
| 109 | "no-warn", | ||
| 110 | "", | ||
| 111 | "url-list=s l>url-list", | ||
| 112 | "schedule=s s>schedule", | ||
| 113 | "spread-time=i t>spread-time", | ||
| 114 | "start-offset=i S>start-offset", | ||
| 115 | "ignore-db d>ignore-db", | ||
| 116 | "ignore-link i>ignore-link", | ||
| 117 | "no-warn", | ||
| 118 | "config-file=s", | ||
| 119 | # "old-schedule=s o>old-schedlue", | ||
| 120 | ], { | ||
| 121 | "no-warn" => [ sub { $::no_warn = 1; }, | ||
| 122 | "Avoid issuing warnings about non-fatal problems." ], | ||
| 123 | "url-list" => [ \&makevalue, | ||
| 124 | "File with complete list of URLs to schedule", | ||
| 125 | "FILENAME" ], | ||
| 126 | # "old-schedule" => [ \&makevalue, "Location of the old schedule", | ||
| 127 | # "FILENAME" ], | ||
| 128 | "schedule" => [ \&makevalue, "Override location of the schedule", | ||
| 129 | "FILENAME" ], | ||
| 130 | "config-file" => [ sub { | ||
| 131 | eval {require $::value}; | ||
| 132 | die "Additional config failed: $@" | ||
| 133 | if $@; | ||
| 134 | #if it's not there die anyway. | ||
| 135 | # compare ReadConf.pm | ||
| 136 | }, | ||
| 137 | "Load in an additional configuration file", | ||
| 138 | "FILENAME" ], | ||
| 139 | "spread-time" => [ \&makevalue, | ||
| 140 | "Time over which to spread checking; default 10 " | ||
| 141 | . "days", | ||
| 142 | "SECONDS" ], | ||
| 143 | "ignore-link" => [ \&maketrue, | ||
| 144 | "Set the time with no regard to link status" ], | ||
| 145 | "ignore-db" => [ \&maketrue, | ||
| 146 | "Set the time with no regard to curent setting" ], | ||
| 147 | "start-offset" => [ \&makevalue, | ||
| 148 | "Time offset from now for starting work (can be " | ||
| 149 | . "negative)", | ||
| 150 | "SECONDS" ], | ||
| 151 | # "clear" => [ \&maketrue, "Clear out old entries."] | ||
| 152 | }; | ||
| 153 | |||
| 154 | |||
| 155 | $::opthandler->std_opts; | ||
| 156 | |||
| 157 | $::opthandler->check_opts; | ||
| 158 | |||
| 159 | sub usage() { | ||
| 160 | print <<EOF; | ||
| 161 | build-schedule [options] | ||
| 162 | |||
| 163 | EOF | ||
| 164 | $::opthandler->list_opts; | ||
| 165 | print <<EOF; | ||
| 166 | |||
| 167 | Examine a database of link objects and build a schedule to check those links | ||
| 168 | which can be used by test-link. | ||
| 169 | EOF | ||
| 170 | } | ||
| 171 | sub version() { | ||
| 172 | print <<'EOF'; | ||
| 173 | build-schedule version | ||
| 174 | $Id: build-schedule.pl,v 1.14 2002/01/18 20:32:48 mikedlr Exp $ | ||
| 175 | EOF | ||
| 176 | } | ||
| 177 | |||
| 178 | $Schedule::SoftTime::silent = 1 if $::silent; | ||
| 179 | #command-line-end | ||
| 180 | |||
| 181 | die 'you must define the $::links configuration variable' | ||
| 182 | unless $::links; | ||
| 183 | tie %::links, MLDBM, $::links, O_RDONLY, 0666, $DB_HASH | ||
| 184 | or die $!; | ||
| 185 | |||
| 186 | die 'you must define the $::schedule configuration variable' | ||
| 187 | unless $::schedule; | ||
| 188 | |||
| 189 | -e $::schedule && do { | ||
| 190 | rename($::schedule, $::schedule . ".bak") | ||
| 191 | or die "could not rename old schedule"; | ||
| 192 | $::old_file=$::schedule . ".bak" unless $::old_file; | ||
| 193 | }; | ||
| 194 | |||
| 195 | defined $::url_list and not -e $::url_list and | ||
| 196 | die "url list file doesn't exist"; | ||
| 197 | |||
| 198 | #FIXME. This is bad; we must read the entire old schedule into memory in | ||
| 199 | #order to invert it. We could use a temp file? Still seems ugly.. | ||
| 200 | |||
| 201 | my $oldsched=new Schedule::SoftTime $::old_file; | ||
| 202 | my %old_inverted=(); | ||
| 203 | my ($key,$value); | ||
| 204 | print STDERR "Loading old database\n" if $::verbose; | ||
| 205 | ($key,$value)=$oldsched->first_item; | ||
| 206 | $count=0; | ||
| 207 | while ($key){ | ||
| 208 | $count++; | ||
| 209 | die "link name not defined for scheduled time??" unless defined $value; | ||
| 210 | print STDERR "$count " if ($::verbose & 32 ) && ($count % 100)==0; | ||
| 211 | warn "Link $value scheduled twice. At $key and at " | ||
| 212 | . $old_inverted{$value} . " that's wrong." | ||
| 213 | if defined $old_inverted{$value}; | ||
| 214 | $old_inverted{$value}=$key; | ||
| 215 | } continue { | ||
| 216 | ($key,$value)=$oldsched->next_item; | ||
| 217 | } | ||
| 218 | |||
| 219 | print STDERR "Old database loaded\n" if $::verbose; | ||
| 220 | |||
| 221 | my $sched=new Schedule::SoftTime $::schedule; | ||
| 222 | my $now = time + $::start_offset; | ||
| 223 | my $daystart; | ||
| 224 | my $name; | ||
| 225 | my $link; | ||
| 226 | |||
| 227 | #FIXME: full check for valid URLs at this stage. Has to select all | ||
| 228 | #possible valid absoloute urls. | ||
| 229 | |||
| 230 | |||
| 231 | if ($::url_list) { | ||
| 232 | #shedule links from a file | ||
| 233 | open URLS, "sort $::url_list | uniq |"; | ||
| 234 | URL: while (my $url=<URLS>) { | ||
| 235 | chomp $url; | ||
| 236 | $url =~ m/\w{3,}:.*\w{3,}/ or do { | ||
| 237 | warn "link $url not sensible??"; | ||
| 238 | }; | ||
| 239 | my $link=$::links{$url}; | ||
| 240 | unless ($link) { | ||
| 241 | print STDERR "URL $url doesn't have a link in the database\n" | ||
| 242 | if $::verbose; | ||
| 243 | next URL; | ||
| 244 | } | ||
| 245 | my $check_time=check_time($link, $url, %old_inverted); | ||
| 246 | print STDERR "Scheduling $url at $check_time\n" if $::verbose; | ||
| 247 | $sched->schedule($check_time, $link->url); | ||
| 248 | } | ||
| 249 | } else { | ||
| 250 | #shedule all links which are available. | ||
| 251 | |||
| 252 | while (my ($url, $link)=each %::links) { | ||
| 253 | |||
| 254 | $url =~ m/${WWW::Link_Controller::special_regex}/ && do { | ||
| 255 | print STDERR "ignoring special key $url\n" if $::verbose; | ||
| 256 | next; | ||
| 257 | }; | ||
| 258 | |||
| 259 | $url =~ m,[a-z][a-z0-9]*:, or do { | ||
| 260 | warn "ignoring database key $url - not absolute url"; | ||
| 261 | next; | ||
| 262 | }; | ||
| 263 | |||
| 264 | defined $link or do { | ||
| 265 | warn "key $url had undefined link" unless $::no_warn; | ||
| 266 | next; | ||
| 267 | }; | ||
| 268 | |||
| 269 | $url =~ m/\w{3,}:.*\w{3,}/ or do { | ||
| 270 | warn "link $url not sensible??" unless $::no_warn; | ||
| 271 | }; | ||
| 272 | print STDERR "Scheduling URL $url\n" if $::verbose; | ||
| 273 | my $check_time=check_time($link, $url, %old_inverted); | ||
| 274 | print STDERR "check $url at $check_time\n" unless $::silent; | ||
| 275 | $sched->schedule($check_time, $link->url); | ||
| 276 | } | ||
| 277 | } | ||
| 278 | |||
| 279 | exit; | ||
| 280 | |||
| 281 | #what should we really do if a link is in the old database but has been | ||
| 282 | #scheduled to run prior to now? I think leave it and just let it queue? | ||
| 283 | |||
| 284 | sub check_time ($$;\%) { | ||
| 285 | my $link=shift; | ||
| 286 | my $url=shift; | ||
| 287 | my $inverted=shift; | ||
| 288 | my $old_time=$old_inverted{$url}; | ||
| 289 | |||
| 290 | confess "usage check_time(<link>,<url>,<invert>) not ($link,$url,$inverted)" | ||
| 291 | unless ref $link and defined $url and ((ref $inverted) =~ m/HASH/); | ||
| 292 | |||
| 293 | defined $old_time and not $::ignore_db and do { | ||
| 294 | print STDERR "Time found for $url in old database: $old_time\n" | ||
| 295 | if $::verbose; | ||
| 296 | return $old_time; | ||
| 297 | }; | ||
| 298 | my $check_time=$link->time_want_test(); | ||
| 299 | print STDERR "New link in schedule. Time suggested: $check_time\n" | ||
| 300 | if $::verbose; | ||
| 301 | if ( ( $check_time < ($now + $sched_margin)) or ($::ignore_link) ) { | ||
| 302 | $check_time = $now + randsecs (); | ||
| 303 | print STDERR "Overriding too old time and using instead $check_time\n" | ||
| 304 | if $::verbose; | ||
| 305 | } | ||
| 306 | return $check_time; | ||
| 307 | } | ||
| 308 | |||
| 309 | # =head3 randsecs | ||
| 310 | |||
| 311 | # This function gives a number of seconds between zero and ten days | ||
| 312 | # spread into the future. | ||
| 313 | |||
| 314 | # =cut | ||
| 315 | |||
| 316 | sub randsecs { | ||
| 317 | my $number = int ( logrand() * $spread_time ) ; | ||
| 318 | print STDERR "randsecs generated $number\n" if $::verbose; | ||
| 319 | die "should generate positive number" unless $number >= 0; | ||
| 320 | return $number; | ||
| 321 | } | ||
| 322 | |||
| 323 | # =head3 logrand | ||
| 324 | |||
| 325 | # logrand gives a number which is random, but distributed sort of | ||
| 326 | # logaritmically between 0 and 1 | ||
| 327 | |||
| 328 | # =cut | ||
| 329 | |||
| 330 | sub logrand { | ||
| 331 | my $return=log ( rand(1) +1); | ||
| 332 | print STDERR "logrand generated $return\n" if $::verbose; | ||
| 333 | return $return; | ||
| 334 | } | ||
| 335 | |||
| 336 | |||
| 337 | # =head3 add_days | ||
| 338 | |||
| 339 | # given a time, this function adds a number of days to that time. I | ||
| 340 | # can't see any clean way of doing this. You cannot guarantee that | ||
| 341 | # adding twenty four hours worth of seconds will get the same day can | ||
| 342 | # you? Leap seconds? For now we just do it. | ||
| 343 | |||
| 344 | # A (neat?) hack would be to convert the time forward, run daystart and | ||
| 345 | # see that the difference of the two times was indeed approximately X | ||
| 346 | # times 24 hours. | ||
| 347 | |||
| 348 | # =cut | ||
| 349 | |||
| 350 | sub add_days { | ||
| 351 | my $time=shift; | ||
| 352 | my $days=shift; | ||
| 353 | $days=$days * 24 * 60 * 60; | ||
| 354 | return $time + $days; | ||
| 355 | } | ||