blib/script/extract-links

filter: Statementstmt 92.5% 25 / 27 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 9 / 9 Podpod n/a totaltotal 94.4% 34 / 36 CC 1 SCAR 0.3
CC
1
Cov
93%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« blib/script/test-link ↑ Summary blib/lib/LWP/MultiUA.pm »
1 #!/usr/local/bin/perl -w
2  
3 =head1 NAME
4  
5 extract-links - build links database
6  
7 =head1 SYNOPSIS
8  
9 extract-links [options]
10 extract-links [options] URI directory
11 extract-links [options] URL
12  
13 =head1 DESCRIPTION
14  
15 This program extracts links and builds the databases used by
16 link-controllers various other programs.
17  
18 The first database built is the links database containing information
19 about the status of all of the links that are being checked. The
20 second two are in cdb format and can be used as indexes for
21 identifying which files contain which URIs and vica versa.
22  
23 =head1 FILE MODE
24  
25 In file mode this goes through a directory tree. The program requires
26 a base URI which is the URI which would be used to refernce the
27 directory in which all of the files are contained on the world wide
28 web. This is used to convert internal references into full URIs which
29 can be used to check that the files are visible from the outside.
30  
31 =head1 WWW MODE
32  
33 In WWW mode the program goes through a set of World Wide Web pages
34 generating the databases.
35  
36 The program requires a base-URL which where it starts from. It's
37 default mode is to only work down from that URL, that is it will only
38 get URIs from WWW pages who's URL starts with the base-url
39  
40 =head1 FILTERING
41  
42 There are two regular expressions which can be given for filtering.
43 If the regular expression given with <--exclude-regex> matches the
44 file name then it will not be read in. If the regular expression
45 given with C<--prune-regex> matches a directory name then that entire
46 directory and all subdirectories are excluded.
47  
48 =head1 CONFIGURATION
49  
50 By default, extract-links extracts and refreshes all of the
51 infostructures listed in the file $::infostrucs. The file looks like
52  
53 #mode url directory
54 www http://myserver.example.com /var/www/html223
55  
56 This is covered in detail in the LinkController reference manual.
57  
58 =head1 FILES
59  
60 There are several configuration
61  
62 $HOME/.link-control.pl - base configuration file
63  
64 This contains configuration variables which point to further files.
65  
66 $::links - link database
67  
68 $::infostrucs - infostructure configuration file
69  
70 Full details of the format of these configuration files can be found
71 in the LinkController reference manual.
72  
73 =head1 NOTES
74  
75 Unlike other programs which tend to resort to closing and re-opening
76 files with lists of links, or holding them all in memory, this program
77 uses a file containing list of links to follow the recursion in the
78 WWW without actually using recursive functions. This relies on output
79 to an unbuffered file being available for input immediately
80 afterwards.
81  
82 We also use a temporary database to record which links have been seen
83 before. This could get LARGE.
84  
85 =head1 BUGS
86  
87 The HTML parsing is done by the perl html parser. This provides
88 excelent and controllable results, but a custom parser carefully
89 written in C would be alot faster. This program takes a long time to
90 run. Since this program is run under human control this matters. If
91 anyone knows of an efficient I<but good> C based parser, suggestion
92 would be greatfully accepted. Direct interface compatibility with the
93 current Perl parser would be even better.
94  
95 I think the program can get trapped in directories it can change into,
96 but can't read (mode o+x-w). This should be fixed.
97  
98 This program could put a large load on a given server if accidentally
99 let go where it shouldn't be. This is your responsibility since it
100 isn't reasonable to slow it down for when it's being used on a local
101 machine or LAN. Some warning should be provided.. e.g. out of local
102 domain check.
103  
104 I don't really know if the tied database is really needed. I want to
105 allow massive link collections though.
106  
107 =head1 SEE ALSO
108  
109 L<verify-link-control(1)>; L<extract-links(1)>; L<build-schedule>
110 L<link-report(1)>; L<fix-link(1)>; L<link-report.cgi(1)>; L<fix-link.cgi>
111 L<suggest(1)>; L<link-report.cgi(1)>; L<configure-link-control>
112  
113 L<cdbmake>, L<cdbget>, L<cdbmultiget>
114  
115 The LinkController manual in the distribution in HTML, info, or
116 postscript formats, included in the distribution.
117  
118 http://scotclimb.org.uk/software/linkcont - the
119 LinkController homepage.
120  
121 =cut
122  
123 require 5.001;
124  
125 7 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
126 7 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
127 7 ▶use Fcntl;
Subroutine BEGIN: called CC=1 SCAR=0.0
128  
129 7 ▶use Cwd;
Subroutine BEGIN: called CC=1 SCAR=0.0
130 7 ▶use File::Find;
Subroutine BEGIN: called CC=1 SCAR=0.0
131  
132 7 ▶use LWP::MediaTypes;
Subroutine BEGIN: called CC=1 SCAR=0.0
133 require HTML::LinkExtor;
134 #use URI;
135 7 ▶use LWP::Auth_UA;
Subroutine BEGIN: called CC=1 SCAR=0.0
136 7 ▶use CDB_File::BiIndex::Generator;
Subroutine BEGIN: called CC=1 SCAR=0.0
137  
138 0 ▶use DB_File; #for temporary database
Subroutine BEGIN: called CC=1 SCAR=2.6
139 use Data::Dumper;
140 use MLDBM qw(DB_File); #chosen over GDBM for network byte order.
141  
142 use WWW::Link;
143  
144 use WWW::Link_Controller::ReadConf;
145 use WWW::Link_Controller::Lock;
146  
147 use CDB_File::BiIndex::Generator;
148  
149 use WWW::Link_Controller;
150 use WWW::Link_Controller::URL;
151  
152 use WWW::Link_Controller::InfoStruc;
153  
154 sub link_handler ($%) ;
155 sub findweb ($%) ;
156 sub make_act_on_file ($$%);
157 sub make_attrib_func ($$%);
158 sub chunk_accept_func ($) ;
159 sub read_infostruc_file ($$) ;
160  
161 ##############################################################################
162 #start command line processing
163 use Getopt::Function qw(maketrue makevalue);
164  
165  
166 $::write_links = 1; # not currently configurable, but should be
167  
168 $::link_database=undef;
169 @::infostrucs=();
170 $::default_infostrucs=undef;
171 $::start_time=time;
172 $::in_uri_list=undef;
173 $::out_uri_list=undef;
174 $::verbose=0;
175 $::opthandler = new Getopt::Function
176 [ "version V>version",
177 "usage h>usage help>usage",
178 "help-opt=s",
179 "verbose:i v>verbose",
180 "silent quiet>silent q>silent",
181 "",
182 "exclude-regex=s e>exclude-regex",
183 "prune-regex=s p>prune-regex",
184 "default-infostrucs d>default-infostrucs",
185 "",
186 "link-database=s l>link-database",
187 "config-file=s c>config-file",
188 "",
189 "out-uri-list=s o>out-uri-list",
190 "in-uri-list=s i>in-uri-list",
191 ],{
192 "default-infostrucs" => [ \&maketrue,
193 "handle all default infostrucs (as well as ones "
194 . "listed on command line)" ],
195 "exclude-regex" => [ \&makevalue,
196 "Exclude expression for excluding files.",
197 "REGEX" ],
198 "prune-regex" => [ \&makevalue,
199 "Regular expression for excluding entire "
200 . "directories.",
201 "REGEX" ],
202 "link-database" => [ \&makevalue,
203 "Database to create link records into.",
204 "FILENAME" ],
205 "out-uri-list" => [ \&makevalue,
206 "File to output the URI of each link found to",
207 "FILENAME" ],
208 "in-uri-list" => [ \&makevalue,
209 "File to input URIs from to create links",
210 "FILENAME" ],
211 "config-file" => [ sub {
212 print STDERR "loading extra config\n" if $::verbose;
213 eval {require $::value};
214 die "Additional config failed: $@"
215 if $@;
216 #if it's not there die anyway.
217 # compare ReadConf.pm
218 },
219 "Load in an additional configuration file",
220 "FILENAME" ]
221 };
222  
223 $::opthandler->std_opts;
224  
225 $::opthandler->check_opts;
226  
227 sub usage() {
228 print <<EOF;
229 extract-links [arguments] [url-base [file-base]]
230  
231 EOF
232 $::opthandler->list_opts;
233 print <<EOF;
234  
235 Extract the link and index information from a directory containing
236 HTML files or from a set of WWW pages with URLs which begin with the
237 given URL and which can be found by starting from that URL and
238 searching other such pages.
239 EOF
240 }
241  
242 sub version() {
243 print <<'EOF';
244 extract-links version
245 $Id: extract-links.pl,v 1.11 2002/02/03 21:17:36 mikedlr Exp $
246 EOF
247 }
248  
249 $::links=$::link_database if defined $::link_database;
250 $WWW::Link_Controller::Lock::silent=1 if $::silent;
251  
252 if ( @ARGV ) {
253 $::default_infostrucs=0 unless defined $::default_infostrucs;
254 my $url_base=shift;
255 my $file_base=shift;
256 my $mode = defined $file_base ? "directory" : "www";
257 push @::infostrucs, $url_base;
258 $::infostrucs{$url_base} =
259 {
260 mode => $mode,
261 url_base => $url_base,
262 file_base => $file_base
263 };
264 }
265  
266 $::default_infostrucs = 0
267 if ( not defined $::default_infostrucs ) and $::in_uri_list;
268 $::default_infostrucs=1 unless defined $::default_infostrucs;
269  
270 %::ignored=();
271 $::ignored=0;
272  
273 ##############################################################################
274 #end command line processing
275  
276 #
277 LWP::MediaTypes::add_type("text/html" => qw(shtml));
278  
279 die 'you must define the $::links configuration variable'
280 unless $::links;
281  
282 #if we have a communal links database then we store our links in a
283 #file for the link testing system to pick up.
284  
285 -f $::links and not -w _ and do {
286 defined $::out_uri_list or print STDERR <<EOF;
287 Can't write to links database. Creating uri list in
288 ~/.link-control-links instead.
289 EOF
290 $::write_links = 0;
291 };
292  
293 open (URILIST, ">$::out_uri_list") if defined $::out_uri_list;
294  
295 $::checklist="/tmp/extract-links.seen.tmp.$$";
296  
297 #checklist protects the system overall from recording the same
298 #link more than once e.g. in the urilists output and optimises
299 #against accessing the link database more than once for each
300 #link
301  
302 tie %::checklist, "DB_File", $::checklist, O_RDWR|O_CREAT, 0666
303 or die $!;
304  
305 eval "END { unlink \$::checklist if defined \$::checklist }";
306  
307  
308 $::write_links && do {
309  
310 WWW::Link_Controller::Lock::lock($::links);
311 my $linkdbm;
312 print STDERR "opening link file $::links\n" if $::verbose;
313 $linkdbm = tie %::link_db, "MLDBM", $::links, O_CREAT|O_RDWR, 0666, $DB_HASH
314 or die "failed to open database `$::links': $!";
315  
316 $::refresh_time=$::link_db{$WWW::Link_Controller::refresh_key};
317  
318 if ( defined $::refresh_time ) {
319 $::refresh_time =~ m/^(?:\d+(?:\.\d*)?|\.\d+)$/ or do {
320 warn "invalid update time on database $::refresh_time";
321 $::refresh_time=undef;
322 };
323 $::refresh_time > $::start_time and do {
324 warn "update time in database in future $::refresh_time. Updating";
325 $::refresh_time=undef;
326 };
327 }
328 $::link_db{$WWW::Link_Controller::refresh_key}=$::start_time;
329 };
330  
331 die "you must define the \$::page_index configuration variable"
332 unless defined $::page_index;
333 die "you must define the \$::link_index configuration variable"
334 unless defined $::link_index;
335  
336  
337 $::index=new CDB_File::BiIndex::Generator
338 $::page_index , $::link_index;
339  
340 die "you must define the \$::infostrucs configuration variable"
341 if $::default_infostrucs and not defined $::infostrucs ;
342  
343  
344 if ( $::default_infostrucs ) {
345 print STDERR "loading default infostrucs\n" unless $::silent;
346 WWW::Link_Controller::InfoStruc::default_infostrucs();
347 } else {
348 print STDERR "ignoring default infostrucs\n" unless $::silent;
349 }
350  
351 die "no infostructures to extract links from" unless @::infostrucs;
352 INFOS: foreach my $base_url ( @::infostrucs ) {
353 my $defs = $::infostrucs{$base_url};
354  
355 my $file_base=$defs->{file_base};
356 defined $file_base and not -e $file_base and die
357 "file base $file_base doesn't exist for $base_url";
358 # defined $directory and not -d $directory and die
359 # "base directory $directory isn't a directory for $url";
360  
361  
362 CASE: {
363 my $mode=$defs->{mode};
364 $mode eq "directory" && do {
365 die "file_base not defined for infostructure $base_url"
366 unless defined $file_base;
367 -e $file_base || do {
368 warn "$file_base does not exist, ignoring infostruc $base_url";
369 next INFOS;
370 };
371  
372 -d $file_base && do {
373 $file_base = $file_base . "/" unless $file_base =~ m,/$,;
374 $base_url = $base_url . "/" unless $base_url =~ m,/$,;
375 };
376  
377 print STDERR <<EOF if $::verbose & 2;
378 About to extract links from infostructure $base_url
379 using $file_base as the base directory.
380 EOF
381 my $act_on_file=make_act_on_file ($base_url,$file_base,%$defs);
382 find ( $act_on_file, $file_base);
383 last;
384 };
385  
386 $mode eq "www" && do {
387 print STDERR <<EOF if $::verbose & 2;
388 About to extract links from infostructure $base_url
389 using recursive web download.
390 EOF
391 findweb ( $base_url, %$defs);
392 last;
393 };
394  
395 die "invalid mode $mode";
396  
397 }
398  
399 }
400 print STDERR "--------------------------\n"
401 . "finished extracting links; building databases\n"
402 if $::verbose & 4;
403  
404 $::index->finish();
405  
406 if ( $::ignored and not $::silent) {
407 print "Ignored $::ignored files of the following types\n";
408 while (my ($type, $count)=each %::ignored) {
409 my $repeat=30 - length($type) - length($count);
410 $repeat=0 if $repeat < 0;
411 $repeat+=5;
412 print $type, "." x $repeat, $count, "\n";
413 }
414 print "If $::ignored seems alot then maybe you want to send in a patch?\n";
415 }
416 exit 0;
417  
418  
419 #act on file, designed to be called by File::Find for each
420 #file in the infostructure.
421  
422 sub make_act_on_file ($$%){
423 my $url_base=shift;
424 my $file_base=shift;
425 my %settings=@_;
426  
427 my $prune_regex = $settings{prune_re};
428 my $include_regex = $settings{resource_include_re};
429 my $exclude_regex = $settings{resource_exclude_re};
430  
431 return sub {
432  
433 -d && do {
434 $File::Find::prune=1 if defined $prune_regex
435 and $File::Find::name =~ m/$prune_regex/o;
436 return 1;
437 };
438  
439 #file name for this page
440 my $page_name=$File::Find::name;
441 #base url for this page
442 my $base = $File::Find::dir . '/';
443 $base =~ s/^$file_base/$url_base/;
444 #url for this page
445 my $page_url="$base$_";
446  
447 return if -l;
448 ( defined $include_regex and not $page_url =~ m/$include_regex/ ) or
449 ( defined $exclude_regex and $page_url =~ m/$exclude_regex/ ) and do {
450 print STDERR "excluding $page_name\n"
451 if $::verbose & 4;
452 return;
453 };
454  
455 # FIXME: dangerous, there could be a link to an external file which
456 # means the external file should be included in the infostructure
457  
458 my $media=guess_media_type($_);
459 ($media =~ /html/) or do {
460 print STDERR "ignoring $page_name\n"
461 if $::verbose & 4;
462 $::ignored{$media}=0 unless defined $::ignored{$media};
463 $::ignored{$media}++;
464 $::ignored++;
465 return;
466 };
467  
468 my @linklist=();
469  
470 print STDERR "acting on $page_name, url $page_url\n"
471 if $::verbose & 4;
472  
473 my $act_url=make_attrib_func($page_url,\@linklist, %settings);
474  
475 my $p = HTML::LinkExtor->new($act_url);
476  
477 unless ( eval { $p->parse_file($_) } ) {
478 warn $@ ;
479 return;
480 }
481  
482 die "no url??" unless $page_url;
483 page_handler($page_url, \@linklist);
484 }
485  
486 }
487  
488  
489 #Findweb - download pages from the internet
490 #
491 #This function starts at a url and works from there through all linked
492 #pages which are below that page in the URL hirearchy.
493 #
494  
495 sub findweb ($%) {
496 my $url=shift;
497 my %settings=@_;
498  
499 my $basere='^' . quotemeta ($url);
500  
501 my $work_url;
502  
503 #in future this should be parameterisable for each infostructure e.g.
504 #do we use robot exclusion rules etc.
505  
506 my $tmpfile="/tmp/extract-links.visited.tmp.$$";
507 unlink $tmpfile;
508 -e $tmpfile and die "failed to remove $tmpfile";
509  
510 #the visited hash protects us against reading in the same page more
511 #than once during this infostructure run. thereby preventing loops.
512  
513 tie my %visited, "DB_File", $tmpfile, O_RDWR|O_CREAT, 0666
514 or die $!;
515  
516 my $ua = new LWP::Auth_UA;
517  
518 my $worktmp="/tmp/extract-links.work.tmp.$$";
519  
520 open(my $workout, ">$worktmp")
521 or die "couldn't open working file $worktmp for writing";
522 open(my $workin, "<$worktmp")
523 or die "couldn't open working $worktmp file for reading";
524 autoflush $workout 1;
525 print $workout "$url\n" ;
526  
527 PAGE: while (defined($work_url = <$workin>)) {
528 chomp $work_url;
529  
530 defined $::exclude_regex && $work_url =~ m/$::exclude_regex/ and do {
531 print STDERR "skipping $work_url\n" if $::verbose & 4;
532 next PAGE;
533 };
534 print STDERR "looking at $work_url\n" if $::verbose & 4;
535  
536 #the following is just an optimisation which means that we don't
537 #download pages that we won't process and can be got rid of if you
538 #find that pages are being missed.
539  
540 unless ( ($work_url =~ m,/$, ) #a directory..
541 or ( guess_media_type($work_url) =~
542 m<html|^application/octet-stream$>
543 #something that's could come out as html
544 )
545 )
546 {
547 print STDERR "non-readable looking url $work_url";
548 next PAGE;
549 }
550  
551 #1 what's the base for this file
552  
553 my $rebound=1;
554  
555 my %redirect=();
556  
557 while ($rebound) {
558  
559 if ( $visited{$work_url} ) {
560 print STDERR "skipping already visited url: $work_url \n"
561 if $::verbose & 4;
562 next PAGE;
563 }
564  
565 $visited{$work_url}=1;
566  
567 unless ($work_url =~ m/$basere/ ) { # we are trapped in one area.
568 warn "out of bounds url $work_url";
569 next PAGE;
570 }
571  
572 print STDERR "going ahead with first visit to url: $work_url \n"
573 unless $::silent;
574  
575 my $oldurl=$work_url;
576 my $request = new HTTP::Request('GET', $work_url);
577  
578 my @linklist=();
579  
580 my $link_func = make_attrib_func($work_url, \@linklist,
581 work_url_re => $basere,
582 workfile => $workout,
583 %settings,
584 );
585  
586 my $extractor = HTML::LinkExtor->new($link_func);
587  
588 my $accept_func=chunk_accept_func($extractor);
589  
590  
591 print STDERR "making request on $work_url\n" unless $::silent;
592  
593 #download and parse .. big chunks becuase should be on local network
594 #so can expect fast response
595  
596 my $response = $ua->simple_request($request, $accept_func, 8192) ;
597  
598 if ($response->is_redirect()) {
599 print STDERR "got a redirect ";
600 my $where=$response->content_location();
601 if ($rebound > 100) { #who in their right mind?
602 $rebound=0; #FIXME .. tell server person.
603 print STDERR "too many.. give up\n";
604 next PAGE;
605 }
606 if ($redirect{$where}) {
607 $rebound=0; #FIXME .. tell server person.
608 print STDERR "back to where we were before.. give up\n";
609 die "redirect loop found"
610 }
611 print STDERR "try again\n";
612 $redirect{$where}=1;
613 $rebound++;
614 $work_url = $where;
615 die "try again";
616 } else {
617 $rebound=0;
618 }
619  
620 # FIXME redirects?
621  
622 page_handler($work_url, \@linklist);
623  
624 # next PAGE unless $response->is_success() or $oldurl ne $work_url;
625 }
626 }
627 unlink $tmpfile, $worktmp;
628 }
629  
630  
631 sub make_attrib_func ($$%) {
632 my $base_url=shift;
633 my $link_array=shift;
634 my %settings=@_;
635  
636 my $fh=$settings{workfile};
637 my $urlre=$settings{work_url_re};
638  
639 my $include_regex = $settings{link_include_re};
640 my $exclude_regex = $settings{link_exclude_re};
641  
642 #optional file handle we will print to
643 # a closeure with a reference to the linklist which is
644 # called repeatedly by the link extractor with the link
645 # containing attributes from the page.
646  
647 my $act_attrib= sub {
648 my($tag, %attr) = @_;
649  
650 #this is a closure which uses
651 #$page_name
652 #$page_record
653  
654 my $linkname;
655 my $attr;
656  
657 #my $linkname=$attr{'href'};
658 LINK: while (($attr, $linkname)=each %attr) {
659 unless ( $linkname ) {
660 warn "Undefined link name generated";
661 next LINK;
662 }
663 print STDERR "extracted link $linkname\n" if
664 $::verbose & 8;
665  
666 ( defined $include_regex and not $linkname =~ m/$include_regex/ ) or
667 ( defined $exclude_regex and $linkname =~ m/$exclude_regex/ ) and do {
668 print STDERR "excluding link $linkname\n"
669 if $::verbose & 4;
670 next LINK;
671 };
672  
673 my ( $url, $fragment )
674 = WWW::Link_Controller::URL::fixup_link_url($linkname,$base_url);
675  
676 next unless defined $url;
677  
678 push @$link_array, $url;
679  
680 link_handler ( $url, fragment => $fragment );
681  
682 #this prints the link out for use in or working URL list
683 #for or web recursion process.
684 print $fh $url . "\n" if defined $fh and $url =~ m/$urlre/;
685 }
686 }; # END definition of $act_link function
687 return $act_attrib
688 }
689  
690  
691 sub chunk_accept_func ($){
692 my $extractor=shift;
693 #my %settings=@_; - no use for now
694  
695 #acceptfunc is a closeure with the link extractor which which is
696 #called by the user agent with the parts of the page as they
697 #are downloaded
698  
699 #it then calls the link extractor which in turn calls the $act_url
700 #function
701  
702 #Redirects etc. are handled by dying immediately. This is caught by
703 #the user agent which then gives it back in the response and never
704 #calls us again.
705  
706 my $func = sub {
707 my ($chunk, $response, $protocol) = @_;
708 printf STDERR "Checking response\n" if $::verbose & 8;
709 die "only successes implemented now" unless $response->is_success();
710 unless ( $response->content_type =~ m,html$,) {
711 print STDERR "Wrong content type. give up.\n";
712 die "can only handle html so far" ;
713 }
714 $extractor->parse($chunk);
715 };
716 return $func;
717 }
718  
719  
720 sub page_handler ($$) {
721 my $page_url=shift;
722 my $link_list_ref=shift;
723 if (@$link_list_ref) {
724 print STDERR "adding links in $page_url to index\n"
725 if $::verbose & 8;
726 $::index->add_list_first($page_url,$link_list_ref);
727 }
728 }
729  
730 #link_handler should be called for each valid URL found.
731 #it will output it to the right places etc.
732  
733 sub link_handler ($%) {
734 my $url=shift;
735  
736  
737 my %link_info=@_;
738  
739 my $fragment=$link_info{fragment};
740 print STDERR "ignoring fragment $fragment\n"
741 if $::verbose & 16 and defined $fragment;
742  
743  
744 $::checklist{$url} && do {
745 print STDERR "URL: $url already seen\n"
746 if $::verbose & 32;
747 return;
748 };
749 $::checklist{$url} = 1;
750  
751 $::write_links && do {
752 my $link = $::link_db{$url};
753  
754 unless ( defined $link) {
755 print STDERR "URL: $url being added to link database\n"
756 if $::verbose & 16;
757 $link=WWW::Link->new($url);
758 } else {
759 print STDERR "URL: $url already in link database\n"
760 if $::verbose & 16;
761 $link->last_refresh($::start_time);
762 }
763  
764 $::link_db{$url}=$link;
765 };
766  
767 print URILIST $url . "\n" if defined $::out_uri_list;
768 }
769  
770  
771  
772 #call tree
773 #
774 # LWP::UserAgent
775 # calls the chunk accept function
776 # which calls the HTML::LinkExtor
777 # which calls the attrib_func
778 # which calls link_handler()
779  
780 # File::Find
781 # calls the act_on file func
782 # which calls the HTML::LinkExtor
783 # which calls the attrib_func
784 # which calls link_handler()
« blib/script/test-link ↑ Summary blib/lib/LWP/MultiUA.pm »