blib/script/build-schedule

filter: Statementstmt 83.3% 10 / 12 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 4 / 4 Podpod n/a totaltotal 87.5% 14 / 16 CC 1 SCAR 0.7
CC
1
Cov
83%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« blib/script/link-report ↑ Summary blib/script/verify-link-control »
1 #!/usr/local/bin/perl -w
2  
3 =head1 NAME
4  
5 build-schedule - build a schedule for link checking.
6  
7 =head1 SYNOPSIS
8  
9 build-schedule [options]
10  
11 -V --version Give version information for this program
12 -h --help --usage Describe usage of this program.
13 --help-opt=OPTION Give help information for a given option
14 -v --verbose[=VERBOSITY] Give information about what the program is doing.
15 Set value to control what information is given.
16 -l --url-list=FILENAME File with complete list of URLs to schedule
17 -s --schedule=FILENAME Override location of the schedule
18 -t --spread-time=SECONDS Time over which to spread checking; default 10 days
19  
20 --config-file=FILENAME Load in an additional configuration file
21  
22 =head1 DESCRIPTION
23  
24 This program takes a links database and builds an entire schedule from
25 scratch for them. This can then be later used by daily-check-links to
26 check all of the links.
27  
28 By default C<build-schedule> builds a schedule for all of the links in
29 your links database but you can give it a list of URLs in a file in
30 which case the schedule built will only contain those links.
31  
32 The schedule tries to spread link checking into the future with more
33 links being checked soon and fewer later. As links are scheduled for
34 repeat checking this should work out to a reasonably even rate of
35 checking.
36  
37 If there is an already a schedule file in existance, it will be read
38 and the times in the old database will be reused. The old database
39 will be renamed with a .bak extension.
40  
41 =head1 EXAMPLES
42  
43 build-schedule
44  
45 If everything is set up correctly and you don't have any special needs
46 then you don't have to give any arguments.
47  
48 build-schedule --ignore-links --spread-time=7200
49  
50 Causes an immediate pulse of activity which tests all of the links in
51 the database within the next two hours. Needless to say, this will
52 cause considerable amounts of network traffic and is a bad idea in
53 most normal situations.
54  
55 =head1 BUGS
56  
57 The program doesn't check the validity of its output database, so if
58 the output filesystem fills up things can go horrible. Check afterwards.
59  
60 =head1 SEE ALSO
61  
62 L<verify-link-control(1)>; L<extract-links(1)>; L<build-schedule>
63 L<link-report(1)>; L<fix-link(1)>; L<link-report.cgi(1)>; L<fix-link.cgi>
64 L<suggest(1)>; L<link-report.cgi(1)>; L<configure-link-control>
65  
66 The LinkController manual in the distribution in HTML, info, or
67 postscript formats, included in the distribution.
68  
69 http://scotclimb.org.uk/software/linkcont/ - the
70 LinkController homepage.
71  
72 =cut
73  
74 3 ▶use vars qw($sched_margin);
Subroutine BEGIN: called CC=1 SCAR=0.0
75 sub check_time ($$;\%) ;
76  
77 $sched_margin=10; #earliest time we schedule a link to run
78  
79 3 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
80  
81 3 ▶use Fcntl;
Subroutine BEGIN: called CC=1 SCAR=0.0
82 0 ▶use DB_File;
Subroutine BEGIN: called CC=1 SCAR=2.6
83 use Data::Dumper;
84 use MLDBM qw(DB_File);
85  
86 use WWW::Link;
87 use Schedule::SoftTime;
88  
89 #Configuration - we go through %ENV, so you'd better not be running SUID
90 #eval to ignore if a file doesn't exist.. e.g. the system config
91  
92 use WWW::Link_Controller;
93 use WWW::Link_Controller::ReadConf;
94  
95 use Getopt::Function qw(maketrue makevalue);
96  
97 $::verbose=0;
98 $::spread_time=60.0 * 60.0 * 24.0 * 10.0;
99 $::start_offset=0;
100 $::ignore_link=0;
101 $::ignore_db=0;
102  
103 $::opthandler = new Getopt::Function
104 [ "version V>version",
105 "usage h>usage help>usage",
106 "help-opt=s",
107 "verbose:i v>verbose",
108 "silent quite>silent q>silent",
109 "no-warn",
110 "",
111 "url-list=s l>url-list",
112 "schedule=s s>schedule",
113 "spread-time=i t>spread-time",
114 "start-offset=i S>start-offset",
115 "ignore-db d>ignore-db",
116 "ignore-link i>ignore-link",
117 "no-warn",
118 "config-file=s",
119 # "old-schedule=s o>old-schedlue",
120 ], {
121 "no-warn" => [ sub { $::no_warn = 1; },
122 "Avoid issuing warnings about non-fatal problems." ],
123 "url-list" => [ \&makevalue,
124 "File with complete list of URLs to schedule",
125 "FILENAME" ],
126 # "old-schedule" => [ \&makevalue, "Location of the old schedule",
127 # "FILENAME" ],
128 "schedule" => [ \&makevalue, "Override location of the schedule",
129 "FILENAME" ],
130 "config-file" => [ sub {
131 eval {require $::value};
132 die "Additional config failed: $@"
133 if $@;
134 #if it's not there die anyway.
135 # compare ReadConf.pm
136 },
137 "Load in an additional configuration file",
138 "FILENAME" ],
139 "spread-time" => [ \&makevalue,
140 "Time over which to spread checking; default 10 "
141 . "days",
142 "SECONDS" ],
143 "ignore-link" => [ \&maketrue,
144 "Set the time with no regard to link status" ],
145 "ignore-db" => [ \&maketrue,
146 "Set the time with no regard to curent setting" ],
147 "start-offset" => [ \&makevalue,
148 "Time offset from now for starting work (can be "
149 . "negative)",
150 "SECONDS" ],
151 # "clear" => [ \&maketrue, "Clear out old entries."]
152 };
153  
154  
155 $::opthandler->std_opts;
156  
157 $::opthandler->check_opts;
158  
159 sub usage() {
160 print <<EOF;
161 build-schedule [options]
162  
163 EOF
164 $::opthandler->list_opts;
165 print <<EOF;
166  
167 Examine a database of link objects and build a schedule to check those links
168 which can be used by test-link.
169 EOF
170 }
171 sub version() {
172 print <<'EOF';
173 build-schedule version
174 $Id: build-schedule.pl,v 1.14 2002/01/18 20:32:48 mikedlr Exp $
175 EOF
176 }
177  
178 $Schedule::SoftTime::silent = 1 if $::silent;
179 #command-line-end
180  
181 die 'you must define the $::links configuration variable'
182 unless $::links;
183 tie %::links, MLDBM, $::links, O_RDONLY, 0666, $DB_HASH
184 or die $!;
185  
186 die 'you must define the $::schedule configuration variable'
187 unless $::schedule;
188  
189 -e $::schedule && do {
190 rename($::schedule, $::schedule . ".bak")
191 or die "could not rename old schedule";
192 $::old_file=$::schedule . ".bak" unless $::old_file;
193 };
194  
195 defined $::url_list and not -e $::url_list and
196 die "url list file doesn't exist";
197  
198 #FIXME. This is bad; we must read the entire old schedule into memory in
199 #order to invert it. We could use a temp file? Still seems ugly..
200  
201 my $oldsched=new Schedule::SoftTime $::old_file;
202 my %old_inverted=();
203 my ($key,$value);
204 print STDERR "Loading old database\n" if $::verbose;
205 ($key,$value)=$oldsched->first_item;
206 $count=0;
207 while ($key){
208 $count++;
209 die "link name not defined for scheduled time??" unless defined $value;
210 print STDERR "$count " if ($::verbose & 32 ) && ($count % 100)==0;
211 warn "Link $value scheduled twice. At $key and at "
212 . $old_inverted{$value} . " that's wrong."
213 if defined $old_inverted{$value};
214 $old_inverted{$value}=$key;
215 } continue {
216 ($key,$value)=$oldsched->next_item;
217 }
218  
219 print STDERR "Old database loaded\n" if $::verbose;
220  
221 my $sched=new Schedule::SoftTime $::schedule;
222 my $now = time + $::start_offset;
223 my $daystart;
224 my $name;
225 my $link;
226  
227 #FIXME: full check for valid URLs at this stage. Has to select all
228 #possible valid absoloute urls.
229  
230  
231 if ($::url_list) {
232 #shedule links from a file
233 open URLS, "sort $::url_list | uniq |";
234 URL: while (my $url=<URLS>) {
235 chomp $url;
236 $url =~ m/\w{3,}:.*\w{3,}/ or do {
237 warn "link $url not sensible??";
238 };
239 my $link=$::links{$url};
240 unless ($link) {
241 print STDERR "URL $url doesn't have a link in the database\n"
242 if $::verbose;
243 next URL;
244 }
245 my $check_time=check_time($link, $url, %old_inverted);
246 print STDERR "Scheduling $url at $check_time\n" if $::verbose;
247 $sched->schedule($check_time, $link->url);
248 }
249 } else {
250 #shedule all links which are available.
251  
252 while (my ($url, $link)=each %::links) {
253  
254 $url =~ m/${WWW::Link_Controller::special_regex}/ && do {
255 print STDERR "ignoring special key $url\n" if $::verbose;
256 next;
257 };
258  
259 $url =~ m,[a-z][a-z0-9]*:, or do {
260 warn "ignoring database key $url - not absolute url";
261 next;
262 };
263  
264 defined $link or do {
265 warn "key $url had undefined link" unless $::no_warn;
266 next;
267 };
268  
269 $url =~ m/\w{3,}:.*\w{3,}/ or do {
270 warn "link $url not sensible??" unless $::no_warn;
271 };
272 print STDERR "Scheduling URL $url\n" if $::verbose;
273 my $check_time=check_time($link, $url, %old_inverted);
274 print STDERR "check $url at $check_time\n" unless $::silent;
275 $sched->schedule($check_time, $link->url);
276 }
277 }
278  
279 exit;
280  
281 #what should we really do if a link is in the old database but has been
282 #scheduled to run prior to now? I think leave it and just let it queue?
283  
284 sub check_time ($$;\%) {
285 my $link=shift;
286 my $url=shift;
287 my $inverted=shift;
288 my $old_time=$old_inverted{$url};
289  
290 confess "usage check_time(<link>,<url>,<invert>) not ($link,$url,$inverted)"
291 unless ref $link and defined $url and ((ref $inverted) =~ m/HASH/);
292  
293 defined $old_time and not $::ignore_db and do {
294 print STDERR "Time found for $url in old database: $old_time\n"
295 if $::verbose;
296 return $old_time;
297 };
298 my $check_time=$link->time_want_test();
299 print STDERR "New link in schedule. Time suggested: $check_time\n"
300 if $::verbose;
301 if ( ( $check_time < ($now + $sched_margin)) or ($::ignore_link) ) {
302 $check_time = $now + randsecs ();
303 print STDERR "Overriding too old time and using instead $check_time\n"
304 if $::verbose;
305 }
306 return $check_time;
307 }
308  
309 # =head3 randsecs
310  
311 # This function gives a number of seconds between zero and ten days
312 # spread into the future.
313  
314 # =cut
315  
316 sub randsecs {
317 my $number = int ( logrand() * $spread_time ) ;
318 print STDERR "randsecs generated $number\n" if $::verbose;
319 die "should generate positive number" unless $number >= 0;
320 return $number;
321 }
322  
323 # =head3 logrand
324  
325 # logrand gives a number which is random, but distributed sort of
326 # logaritmically between 0 and 1
327  
328 # =cut
329  
330 sub logrand {
331 my $return=log ( rand(1) +1);
332 print STDERR "logrand generated $return\n" if $::verbose;
333 return $return;
334 }
335  
336  
337 # =head3 add_days
338  
339 # given a time, this function adds a number of days to that time. I
340 # can't see any clean way of doing this. You cannot guarantee that
341 # adding twenty four hours worth of seconds will get the same day can
342 # you? Leap seconds? For now we just do it.
343  
344 # A (neat?) hack would be to convert the time forward, run daystart and
345 # see that the difference of the two times was indeed approximately X
346 # times 24 hours.
347  
348 # =cut
349  
350 sub add_days {
351 my $time=shift;
352 my $days=shift;
353 $days=$days * 24 * 60 * 60;
354 return $time + $days;
355 }
« blib/script/link-report ↑ Summary blib/script/verify-link-control »