| 1 | #!/usr/local/bin/perl -w | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | extract-links - build links database | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | extract-links [options] | ||
| 10 | extract-links [options] URI directory | ||
| 11 | extract-links [options] URL | ||
| 12 | |||
| 13 | =head1 DESCRIPTION | ||
| 14 | |||
| 15 | This program extracts links and builds the databases used by | ||
| 16 | link-controllers various other programs. | ||
| 17 | |||
| 18 | The first database built is the links database containing information | ||
| 19 | about the status of all of the links that are being checked. The | ||
| 20 | second two are in cdb format and can be used as indexes for | ||
| 21 | identifying which files contain which URIs and vica versa. | ||
| 22 | |||
| 23 | =head1 FILE MODE | ||
| 24 | |||
| 25 | In file mode this goes through a directory tree. The program requires | ||
| 26 | a base URI which is the URI which would be used to refernce the | ||
| 27 | directory in which all of the files are contained on the world wide | ||
| 28 | web. This is used to convert internal references into full URIs which | ||
| 29 | can be used to check that the files are visible from the outside. | ||
| 30 | |||
| 31 | =head1 WWW MODE | ||
| 32 | |||
| 33 | In WWW mode the program goes through a set of World Wide Web pages | ||
| 34 | generating the databases. | ||
| 35 | |||
| 36 | The program requires a base-URL which where it starts from. It's | ||
| 37 | default mode is to only work down from that URL, that is it will only | ||
| 38 | get URIs from WWW pages who's URL starts with the base-url | ||
| 39 | |||
| 40 | =head1 FILTERING | ||
| 41 | |||
| 42 | There are two regular expressions which can be given for filtering. | ||
| 43 | If the regular expression given with <--exclude-regex> matches the | ||
| 44 | file name then it will not be read in. If the regular expression | ||
| 45 | given with C<--prune-regex> matches a directory name then that entire | ||
| 46 | directory and all subdirectories are excluded. | ||
| 47 | |||
| 48 | =head1 CONFIGURATION | ||
| 49 | |||
| 50 | By default, extract-links extracts and refreshes all of the | ||
| 51 | infostructures listed in the file $::infostrucs. The file looks like | ||
| 52 | |||
| 53 | #mode url directory | ||
| 54 | www http://myserver.example.com /var/www/html223 | ||
| 55 | |||
| 56 | This is covered in detail in the LinkController reference manual. | ||
| 57 | |||
| 58 | =head1 FILES | ||
| 59 | |||
| 60 | There are several configuration | ||
| 61 | |||
| 62 | $HOME/.link-control.pl - base configuration file | ||
| 63 | |||
| 64 | This contains configuration variables which point to further files. | ||
| 65 | |||
| 66 | $::links - link database | ||
| 67 | |||
| 68 | $::infostrucs - infostructure configuration file | ||
| 69 | |||
| 70 | Full details of the format of these configuration files can be found | ||
| 71 | in the LinkController reference manual. | ||
| 72 | |||
| 73 | =head1 NOTES | ||
| 74 | |||
| 75 | Unlike other programs which tend to resort to closing and re-opening | ||
| 76 | files with lists of links, or holding them all in memory, this program | ||
| 77 | uses a file containing list of links to follow the recursion in the | ||
| 78 | WWW without actually using recursive functions. This relies on output | ||
| 79 | to an unbuffered file being available for input immediately | ||
| 80 | afterwards. | ||
| 81 | |||
| 82 | We also use a temporary database to record which links have been seen | ||
| 83 | before. This could get LARGE. | ||
| 84 | |||
| 85 | =head1 BUGS | ||
| 86 | |||
| 87 | The HTML parsing is done by the perl html parser. This provides | ||
| 88 | excelent and controllable results, but a custom parser carefully | ||
| 89 | written in C would be alot faster. This program takes a long time to | ||
| 90 | run. Since this program is run under human control this matters. If | ||
| 91 | anyone knows of an efficient I<but good> C based parser, suggestion | ||
| 92 | would be greatfully accepted. Direct interface compatibility with the | ||
| 93 | current Perl parser would be even better. | ||
| 94 | |||
| 95 | I think the program can get trapped in directories it can change into, | ||
| 96 | but can't read (mode o+x-w). This should be fixed. | ||
| 97 | |||
| 98 | This program could put a large load on a given server if accidentally | ||
| 99 | let go where it shouldn't be. This is your responsibility since it | ||
| 100 | isn't reasonable to slow it down for when it's being used on a local | ||
| 101 | machine or LAN. Some warning should be provided.. e.g. out of local | ||
| 102 | domain check. | ||
| 103 | |||
| 104 | I don't really know if the tied database is really needed. I want to | ||
| 105 | allow massive link collections though. | ||
| 106 | |||
| 107 | =head1 SEE ALSO | ||
| 108 | |||
| 109 | L<verify-link-control(1)>; L<extract-links(1)>; L<build-schedule> | ||
| 110 | L<link-report(1)>; L<fix-link(1)>; L<link-report.cgi(1)>; L<fix-link.cgi> | ||
| 111 | L<suggest(1)>; L<link-report.cgi(1)>; L<configure-link-control> | ||
| 112 | |||
| 113 | L<cdbmake>, L<cdbget>, L<cdbmultiget> | ||
| 114 | |||
| 115 | The LinkController manual in the distribution in HTML, info, or | ||
| 116 | postscript formats, included in the distribution. | ||
| 117 | |||
| 118 | http://scotclimb.org.uk/software/linkcont - the | ||
| 119 | LinkController homepage. | ||
| 120 | |||
| 121 | =cut | ||
| 122 | |||
| 123 | require 5.001; | ||
| 124 | |||
| 125 | 7 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 126 | 7 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 127 | 7 | ▶ | use Fcntl; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 128 | |||
| 129 | 7 | ▶ | use Cwd; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 130 | 7 | ▶ | use File::Find; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 131 | |||
| 132 | 7 | ▶ | use LWP::MediaTypes; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 133 | require HTML::LinkExtor; | ||
| 134 | #use URI; | ||
| 135 | 7 | ▶ | use LWP::Auth_UA; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 136 | 7 | ▶ | use CDB_File::BiIndex::Generator; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 137 | |||
| 138 | 0 | ▶ | use DB_File; #for temporary database |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 139 | use Data::Dumper; | ||
| 140 | use MLDBM qw(DB_File); #chosen over GDBM for network byte order. | ||
| 141 | |||
| 142 | use WWW::Link; | ||
| 143 | |||
| 144 | use WWW::Link_Controller::ReadConf; | ||
| 145 | use WWW::Link_Controller::Lock; | ||
| 146 | |||
| 147 | use CDB_File::BiIndex::Generator; | ||
| 148 | |||
| 149 | use WWW::Link_Controller; | ||
| 150 | use WWW::Link_Controller::URL; | ||
| 151 | |||
| 152 | use WWW::Link_Controller::InfoStruc; | ||
| 153 | |||
| 154 | sub link_handler ($%) ; | ||
| 155 | sub findweb ($%) ; | ||
| 156 | sub make_act_on_file ($$%); | ||
| 157 | sub make_attrib_func ($$%); | ||
| 158 | sub chunk_accept_func ($) ; | ||
| 159 | sub read_infostruc_file ($$) ; | ||
| 160 | |||
| 161 | ############################################################################## | ||
| 162 | #start command line processing | ||
| 163 | use Getopt::Function qw(maketrue makevalue); | ||
| 164 | |||
| 165 | |||
| 166 | $::write_links = 1; # not currently configurable, but should be | ||
| 167 | |||
| 168 | $::link_database=undef; | ||
| 169 | @::infostrucs=(); | ||
| 170 | $::default_infostrucs=undef; | ||
| 171 | $::start_time=time; | ||
| 172 | $::in_uri_list=undef; | ||
| 173 | $::out_uri_list=undef; | ||
| 174 | $::verbose=0; | ||
| 175 | $::opthandler = new Getopt::Function | ||
| 176 | [ "version V>version", | ||
| 177 | "usage h>usage help>usage", | ||
| 178 | "help-opt=s", | ||
| 179 | "verbose:i v>verbose", | ||
| 180 | "silent quiet>silent q>silent", | ||
| 181 | "", | ||
| 182 | "exclude-regex=s e>exclude-regex", | ||
| 183 | "prune-regex=s p>prune-regex", | ||
| 184 | "default-infostrucs d>default-infostrucs", | ||
| 185 | "", | ||
| 186 | "link-database=s l>link-database", | ||
| 187 | "config-file=s c>config-file", | ||
| 188 | "", | ||
| 189 | "out-uri-list=s o>out-uri-list", | ||
| 190 | "in-uri-list=s i>in-uri-list", | ||
| 191 | ],{ | ||
| 192 | "default-infostrucs" => [ \&maketrue, | ||
| 193 | "handle all default infostrucs (as well as ones " | ||
| 194 | . "listed on command line)" ], | ||
| 195 | "exclude-regex" => [ \&makevalue, | ||
| 196 | "Exclude expression for excluding files.", | ||
| 197 | "REGEX" ], | ||
| 198 | "prune-regex" => [ \&makevalue, | ||
| 199 | "Regular expression for excluding entire " | ||
| 200 | . "directories.", | ||
| 201 | "REGEX" ], | ||
| 202 | "link-database" => [ \&makevalue, | ||
| 203 | "Database to create link records into.", | ||
| 204 | "FILENAME" ], | ||
| 205 | "out-uri-list" => [ \&makevalue, | ||
| 206 | "File to output the URI of each link found to", | ||
| 207 | "FILENAME" ], | ||
| 208 | "in-uri-list" => [ \&makevalue, | ||
| 209 | "File to input URIs from to create links", | ||
| 210 | "FILENAME" ], | ||
| 211 | "config-file" => [ sub { | ||
| 212 | print STDERR "loading extra config\n" if $::verbose; | ||
| 213 | eval {require $::value}; | ||
| 214 | die "Additional config failed: $@" | ||
| 215 | if $@; | ||
| 216 | #if it's not there die anyway. | ||
| 217 | # compare ReadConf.pm | ||
| 218 | }, | ||
| 219 | "Load in an additional configuration file", | ||
| 220 | "FILENAME" ] | ||
| 221 | }; | ||
| 222 | |||
| 223 | $::opthandler->std_opts; | ||
| 224 | |||
| 225 | $::opthandler->check_opts; | ||
| 226 | |||
| 227 | sub usage() { | ||
| 228 | print <<EOF; | ||
| 229 | extract-links [arguments] [url-base [file-base]] | ||
| 230 | |||
| 231 | EOF | ||
| 232 | $::opthandler->list_opts; | ||
| 233 | print <<EOF; | ||
| 234 | |||
| 235 | Extract the link and index information from a directory containing | ||
| 236 | HTML files or from a set of WWW pages with URLs which begin with the | ||
| 237 | given URL and which can be found by starting from that URL and | ||
| 238 | searching other such pages. | ||
| 239 | EOF | ||
| 240 | } | ||
| 241 | |||
| 242 | sub version() { | ||
| 243 | print <<'EOF'; | ||
| 244 | extract-links version | ||
| 245 | $Id: extract-links.pl,v 1.11 2002/02/03 21:17:36 mikedlr Exp $ | ||
| 246 | EOF | ||
| 247 | } | ||
| 248 | |||
| 249 | $::links=$::link_database if defined $::link_database; | ||
| 250 | $WWW::Link_Controller::Lock::silent=1 if $::silent; | ||
| 251 | |||
| 252 | if ( @ARGV ) { | ||
| 253 | $::default_infostrucs=0 unless defined $::default_infostrucs; | ||
| 254 | my $url_base=shift; | ||
| 255 | my $file_base=shift; | ||
| 256 | my $mode = defined $file_base ? "directory" : "www"; | ||
| 257 | push @::infostrucs, $url_base; | ||
| 258 | $::infostrucs{$url_base} = | ||
| 259 | { | ||
| 260 | mode => $mode, | ||
| 261 | url_base => $url_base, | ||
| 262 | file_base => $file_base | ||
| 263 | }; | ||
| 264 | } | ||
| 265 | |||
| 266 | $::default_infostrucs = 0 | ||
| 267 | if ( not defined $::default_infostrucs ) and $::in_uri_list; | ||
| 268 | $::default_infostrucs=1 unless defined $::default_infostrucs; | ||
| 269 | |||
| 270 | %::ignored=(); | ||
| 271 | $::ignored=0; | ||
| 272 | |||
| 273 | ############################################################################## | ||
| 274 | #end command line processing | ||
| 275 | |||
| 276 | # | ||
| 277 | LWP::MediaTypes::add_type("text/html" => qw(shtml)); | ||
| 278 | |||
| 279 | die 'you must define the $::links configuration variable' | ||
| 280 | unless $::links; | ||
| 281 | |||
| 282 | #if we have a communal links database then we store our links in a | ||
| 283 | #file for the link testing system to pick up. | ||
| 284 | |||
| 285 | -f $::links and not -w _ and do { | ||
| 286 | defined $::out_uri_list or print STDERR <<EOF; | ||
| 287 | Can't write to links database. Creating uri list in | ||
| 288 | ~/.link-control-links instead. | ||
| 289 | EOF | ||
| 290 | $::write_links = 0; | ||
| 291 | }; | ||
| 292 | |||
| 293 | open (URILIST, ">$::out_uri_list") if defined $::out_uri_list; | ||
| 294 | |||
| 295 | $::checklist="/tmp/extract-links.seen.tmp.$$"; | ||
| 296 | |||
| 297 | #checklist protects the system overall from recording the same | ||
| 298 | #link more than once e.g. in the urilists output and optimises | ||
| 299 | #against accessing the link database more than once for each | ||
| 300 | #link | ||
| 301 | |||
| 302 | tie %::checklist, "DB_File", $::checklist, O_RDWR|O_CREAT, 0666 | ||
| 303 | or die $!; | ||
| 304 | |||
| 305 | eval "END { unlink \$::checklist if defined \$::checklist }"; | ||
| 306 | |||
| 307 | |||
| 308 | $::write_links && do { | ||
| 309 | |||
| 310 | WWW::Link_Controller::Lock::lock($::links); | ||
| 311 | my $linkdbm; | ||
| 312 | print STDERR "opening link file $::links\n" if $::verbose; | ||
| 313 | $linkdbm = tie %::link_db, "MLDBM", $::links, O_CREAT|O_RDWR, 0666, $DB_HASH | ||
| 314 | or die "failed to open database `$::links': $!"; | ||
| 315 | |||
| 316 | $::refresh_time=$::link_db{$WWW::Link_Controller::refresh_key}; | ||
| 317 | |||
| 318 | if ( defined $::refresh_time ) { | ||
| 319 | $::refresh_time =~ m/^(?:\d+(?:\.\d*)?|\.\d+)$/ or do { | ||
| 320 | warn "invalid update time on database $::refresh_time"; | ||
| 321 | $::refresh_time=undef; | ||
| 322 | }; | ||
| 323 | $::refresh_time > $::start_time and do { | ||
| 324 | warn "update time in database in future $::refresh_time. Updating"; | ||
| 325 | $::refresh_time=undef; | ||
| 326 | }; | ||
| 327 | } | ||
| 328 | $::link_db{$WWW::Link_Controller::refresh_key}=$::start_time; | ||
| 329 | }; | ||
| 330 | |||
| 331 | die "you must define the \$::page_index configuration variable" | ||
| 332 | unless defined $::page_index; | ||
| 333 | die "you must define the \$::link_index configuration variable" | ||
| 334 | unless defined $::link_index; | ||
| 335 | |||
| 336 | |||
| 337 | $::index=new CDB_File::BiIndex::Generator | ||
| 338 | $::page_index , $::link_index; | ||
| 339 | |||
| 340 | die "you must define the \$::infostrucs configuration variable" | ||
| 341 | if $::default_infostrucs and not defined $::infostrucs ; | ||
| 342 | |||
| 343 | |||
| 344 | if ( $::default_infostrucs ) { | ||
| 345 | print STDERR "loading default infostrucs\n" unless $::silent; | ||
| 346 | WWW::Link_Controller::InfoStruc::default_infostrucs(); | ||
| 347 | } else { | ||
| 348 | print STDERR "ignoring default infostrucs\n" unless $::silent; | ||
| 349 | } | ||
| 350 | |||
| 351 | die "no infostructures to extract links from" unless @::infostrucs; | ||
| 352 | INFOS: foreach my $base_url ( @::infostrucs ) { | ||
| 353 | my $defs = $::infostrucs{$base_url}; | ||
| 354 | |||
| 355 | my $file_base=$defs->{file_base}; | ||
| 356 | defined $file_base and not -e $file_base and die | ||
| 357 | "file base $file_base doesn't exist for $base_url"; | ||
| 358 | # defined $directory and not -d $directory and die | ||
| 359 | # "base directory $directory isn't a directory for $url"; | ||
| 360 | |||
| 361 | |||
| 362 | CASE: { | ||
| 363 | my $mode=$defs->{mode}; | ||
| 364 | $mode eq "directory" && do { | ||
| 365 | die "file_base not defined for infostructure $base_url" | ||
| 366 | unless defined $file_base; | ||
| 367 | -e $file_base || do { | ||
| 368 | warn "$file_base does not exist, ignoring infostruc $base_url"; | ||
| 369 | next INFOS; | ||
| 370 | }; | ||
| 371 | |||
| 372 | -d $file_base && do { | ||
| 373 | $file_base = $file_base . "/" unless $file_base =~ m,/$,; | ||
| 374 | $base_url = $base_url . "/" unless $base_url =~ m,/$,; | ||
| 375 | }; | ||
| 376 | |||
| 377 | print STDERR <<EOF if $::verbose & 2; | ||
| 378 | About to extract links from infostructure $base_url | ||
| 379 | using $file_base as the base directory. | ||
| 380 | EOF | ||
| 381 | my $act_on_file=make_act_on_file ($base_url,$file_base,%$defs); | ||
| 382 | find ( $act_on_file, $file_base); | ||
| 383 | last; | ||
| 384 | }; | ||
| 385 | |||
| 386 | $mode eq "www" && do { | ||
| 387 | print STDERR <<EOF if $::verbose & 2; | ||
| 388 | About to extract links from infostructure $base_url | ||
| 389 | using recursive web download. | ||
| 390 | EOF | ||
| 391 | findweb ( $base_url, %$defs); | ||
| 392 | last; | ||
| 393 | }; | ||
| 394 | |||
| 395 | die "invalid mode $mode"; | ||
| 396 | |||
| 397 | } | ||
| 398 | |||
| 399 | } | ||
| 400 | print STDERR "--------------------------\n" | ||
| 401 | . "finished extracting links; building databases\n" | ||
| 402 | if $::verbose & 4; | ||
| 403 | |||
| 404 | $::index->finish(); | ||
| 405 | |||
| 406 | if ( $::ignored and not $::silent) { | ||
| 407 | print "Ignored $::ignored files of the following types\n"; | ||
| 408 | while (my ($type, $count)=each %::ignored) { | ||
| 409 | my $repeat=30 - length($type) - length($count); | ||
| 410 | $repeat=0 if $repeat < 0; | ||
| 411 | $repeat+=5; | ||
| 412 | print $type, "." x $repeat, $count, "\n"; | ||
| 413 | } | ||
| 414 | print "If $::ignored seems alot then maybe you want to send in a patch?\n"; | ||
| 415 | } | ||
| 416 | exit 0; | ||
| 417 | |||
| 418 | |||
| 419 | #act on file, designed to be called by File::Find for each | ||
| 420 | #file in the infostructure. | ||
| 421 | |||
| 422 | sub make_act_on_file ($$%){ | ||
| 423 | my $url_base=shift; | ||
| 424 | my $file_base=shift; | ||
| 425 | my %settings=@_; | ||
| 426 | |||
| 427 | my $prune_regex = $settings{prune_re}; | ||
| 428 | my $include_regex = $settings{resource_include_re}; | ||
| 429 | my $exclude_regex = $settings{resource_exclude_re}; | ||
| 430 | |||
| 431 | return sub { | ||
| 432 | |||
| 433 | -d && do { | ||
| 434 | $File::Find::prune=1 if defined $prune_regex | ||
| 435 | and $File::Find::name =~ m/$prune_regex/o; | ||
| 436 | return 1; | ||
| 437 | }; | ||
| 438 | |||
| 439 | #file name for this page | ||
| 440 | my $page_name=$File::Find::name; | ||
| 441 | #base url for this page | ||
| 442 | my $base = $File::Find::dir . '/'; | ||
| 443 | $base =~ s/^$file_base/$url_base/; | ||
| 444 | #url for this page | ||
| 445 | my $page_url="$base$_"; | ||
| 446 | |||
| 447 | return if -l; | ||
| 448 | ( defined $include_regex and not $page_url =~ m/$include_regex/ ) or | ||
| 449 | ( defined $exclude_regex and $page_url =~ m/$exclude_regex/ ) and do { | ||
| 450 | print STDERR "excluding $page_name\n" | ||
| 451 | if $::verbose & 4; | ||
| 452 | return; | ||
| 453 | }; | ||
| 454 | |||
| 455 | # FIXME: dangerous, there could be a link to an external file which | ||
| 456 | # means the external file should be included in the infostructure | ||
| 457 | |||
| 458 | my $media=guess_media_type($_); | ||
| 459 | ($media =~ /html/) or do { | ||
| 460 | print STDERR "ignoring $page_name\n" | ||
| 461 | if $::verbose & 4; | ||
| 462 | $::ignored{$media}=0 unless defined $::ignored{$media}; | ||
| 463 | $::ignored{$media}++; | ||
| 464 | $::ignored++; | ||
| 465 | return; | ||
| 466 | }; | ||
| 467 | |||
| 468 | my @linklist=(); | ||
| 469 | |||
| 470 | print STDERR "acting on $page_name, url $page_url\n" | ||
| 471 | if $::verbose & 4; | ||
| 472 | |||
| 473 | my $act_url=make_attrib_func($page_url,\@linklist, %settings); | ||
| 474 | |||
| 475 | my $p = HTML::LinkExtor->new($act_url); | ||
| 476 | |||
| 477 | unless ( eval { $p->parse_file($_) } ) { | ||
| 478 | warn $@ ; | ||
| 479 | return; | ||
| 480 | } | ||
| 481 | |||
| 482 | die "no url??" unless $page_url; | ||
| 483 | page_handler($page_url, \@linklist); | ||
| 484 | } | ||
| 485 | |||
| 486 | } | ||
| 487 | |||
| 488 | |||
| 489 | #Findweb - download pages from the internet | ||
| 490 | # | ||
| 491 | #This function starts at a url and works from there through all linked | ||
| 492 | #pages which are below that page in the URL hirearchy. | ||
| 493 | # | ||
| 494 | |||
| 495 | sub findweb ($%) { | ||
| 496 | my $url=shift; | ||
| 497 | my %settings=@_; | ||
| 498 | |||
| 499 | my $basere='^' . quotemeta ($url); | ||
| 500 | |||
| 501 | my $work_url; | ||
| 502 | |||
| 503 | #in future this should be parameterisable for each infostructure e.g. | ||
| 504 | #do we use robot exclusion rules etc. | ||
| 505 | |||
| 506 | my $tmpfile="/tmp/extract-links.visited.tmp.$$"; | ||
| 507 | unlink $tmpfile; | ||
| 508 | -e $tmpfile and die "failed to remove $tmpfile"; | ||
| 509 | |||
| 510 | #the visited hash protects us against reading in the same page more | ||
| 511 | #than once during this infostructure run. thereby preventing loops. | ||
| 512 | |||
| 513 | tie my %visited, "DB_File", $tmpfile, O_RDWR|O_CREAT, 0666 | ||
| 514 | or die $!; | ||
| 515 | |||
| 516 | my $ua = new LWP::Auth_UA; | ||
| 517 | |||
| 518 | my $worktmp="/tmp/extract-links.work.tmp.$$"; | ||
| 519 | |||
| 520 | open(my $workout, ">$worktmp") | ||
| 521 | or die "couldn't open working file $worktmp for writing"; | ||
| 522 | open(my $workin, "<$worktmp") | ||
| 523 | or die "couldn't open working $worktmp file for reading"; | ||
| 524 | autoflush $workout 1; | ||
| 525 | print $workout "$url\n" ; | ||
| 526 | |||
| 527 | PAGE: while (defined($work_url = <$workin>)) { | ||
| 528 | chomp $work_url; | ||
| 529 | |||
| 530 | defined $::exclude_regex && $work_url =~ m/$::exclude_regex/ and do { | ||
| 531 | print STDERR "skipping $work_url\n" if $::verbose & 4; | ||
| 532 | next PAGE; | ||
| 533 | }; | ||
| 534 | print STDERR "looking at $work_url\n" if $::verbose & 4; | ||
| 535 | |||
| 536 | #the following is just an optimisation which means that we don't | ||
| 537 | #download pages that we won't process and can be got rid of if you | ||
| 538 | #find that pages are being missed. | ||
| 539 | |||
| 540 | unless ( ($work_url =~ m,/$, ) #a directory.. | ||
| 541 | or ( guess_media_type($work_url) =~ | ||
| 542 | m<html|^application/octet-stream$> | ||
| 543 | #something that's could come out as html | ||
| 544 | ) | ||
| 545 | ) | ||
| 546 | { | ||
| 547 | print STDERR "non-readable looking url $work_url"; | ||
| 548 | next PAGE; | ||
| 549 | } | ||
| 550 | |||
| 551 | #1 what's the base for this file | ||
| 552 | |||
| 553 | my $rebound=1; | ||
| 554 | |||
| 555 | my %redirect=(); | ||
| 556 | |||
| 557 | while ($rebound) { | ||
| 558 | |||
| 559 | if ( $visited{$work_url} ) { | ||
| 560 | print STDERR "skipping already visited url: $work_url \n" | ||
| 561 | if $::verbose & 4; | ||
| 562 | next PAGE; | ||
| 563 | } | ||
| 564 | |||
| 565 | $visited{$work_url}=1; | ||
| 566 | |||
| 567 | unless ($work_url =~ m/$basere/ ) { # we are trapped in one area. | ||
| 568 | warn "out of bounds url $work_url"; | ||
| 569 | next PAGE; | ||
| 570 | } | ||
| 571 | |||
| 572 | print STDERR "going ahead with first visit to url: $work_url \n" | ||
| 573 | unless $::silent; | ||
| 574 | |||
| 575 | my $oldurl=$work_url; | ||
| 576 | my $request = new HTTP::Request('GET', $work_url); | ||
| 577 | |||
| 578 | my @linklist=(); | ||
| 579 | |||
| 580 | my $link_func = make_attrib_func($work_url, \@linklist, | ||
| 581 | work_url_re => $basere, | ||
| 582 | workfile => $workout, | ||
| 583 | %settings, | ||
| 584 | ); | ||
| 585 | |||
| 586 | my $extractor = HTML::LinkExtor->new($link_func); | ||
| 587 | |||
| 588 | my $accept_func=chunk_accept_func($extractor); | ||
| 589 | |||
| 590 | |||
| 591 | print STDERR "making request on $work_url\n" unless $::silent; | ||
| 592 | |||
| 593 | #download and parse .. big chunks becuase should be on local network | ||
| 594 | #so can expect fast response | ||
| 595 | |||
| 596 | my $response = $ua->simple_request($request, $accept_func, 8192) ; | ||
| 597 | |||
| 598 | if ($response->is_redirect()) { | ||
| 599 | print STDERR "got a redirect "; | ||
| 600 | my $where=$response->content_location(); | ||
| 601 | if ($rebound > 100) { #who in their right mind? | ||
| 602 | $rebound=0; #FIXME .. tell server person. | ||
| 603 | print STDERR "too many.. give up\n"; | ||
| 604 | next PAGE; | ||
| 605 | } | ||
| 606 | if ($redirect{$where}) { | ||
| 607 | $rebound=0; #FIXME .. tell server person. | ||
| 608 | print STDERR "back to where we were before.. give up\n"; | ||
| 609 | die "redirect loop found" | ||
| 610 | } | ||
| 611 | print STDERR "try again\n"; | ||
| 612 | $redirect{$where}=1; | ||
| 613 | $rebound++; | ||
| 614 | $work_url = $where; | ||
| 615 | die "try again"; | ||
| 616 | } else { | ||
| 617 | $rebound=0; | ||
| 618 | } | ||
| 619 | |||
| 620 | # FIXME redirects? | ||
| 621 | |||
| 622 | page_handler($work_url, \@linklist); | ||
| 623 | |||
| 624 | # next PAGE unless $response->is_success() or $oldurl ne $work_url; | ||
| 625 | } | ||
| 626 | } | ||
| 627 | unlink $tmpfile, $worktmp; | ||
| 628 | } | ||
| 629 | |||
| 630 | |||
| 631 | sub make_attrib_func ($$%) { | ||
| 632 | my $base_url=shift; | ||
| 633 | my $link_array=shift; | ||
| 634 | my %settings=@_; | ||
| 635 | |||
| 636 | my $fh=$settings{workfile}; | ||
| 637 | my $urlre=$settings{work_url_re}; | ||
| 638 | |||
| 639 | my $include_regex = $settings{link_include_re}; | ||
| 640 | my $exclude_regex = $settings{link_exclude_re}; | ||
| 641 | |||
| 642 | #optional file handle we will print to | ||
| 643 | # a closeure with a reference to the linklist which is | ||
| 644 | # called repeatedly by the link extractor with the link | ||
| 645 | # containing attributes from the page. | ||
| 646 | |||
| 647 | my $act_attrib= sub { | ||
| 648 | my($tag, %attr) = @_; | ||
| 649 | |||
| 650 | #this is a closure which uses | ||
| 651 | #$page_name | ||
| 652 | #$page_record | ||
| 653 | |||
| 654 | my $linkname; | ||
| 655 | my $attr; | ||
| 656 | |||
| 657 | #my $linkname=$attr{'href'}; | ||
| 658 | LINK: while (($attr, $linkname)=each %attr) { | ||
| 659 | unless ( $linkname ) { | ||
| 660 | warn "Undefined link name generated"; | ||
| 661 | next LINK; | ||
| 662 | } | ||
| 663 | print STDERR "extracted link $linkname\n" if | ||
| 664 | $::verbose & 8; | ||
| 665 | |||
| 666 | ( defined $include_regex and not $linkname =~ m/$include_regex/ ) or | ||
| 667 | ( defined $exclude_regex and $linkname =~ m/$exclude_regex/ ) and do { | ||
| 668 | print STDERR "excluding link $linkname\n" | ||
| 669 | if $::verbose & 4; | ||
| 670 | next LINK; | ||
| 671 | }; | ||
| 672 | |||
| 673 | my ( $url, $fragment ) | ||
| 674 | = WWW::Link_Controller::URL::fixup_link_url($linkname,$base_url); | ||
| 675 | |||
| 676 | next unless defined $url; | ||
| 677 | |||
| 678 | push @$link_array, $url; | ||
| 679 | |||
| 680 | link_handler ( $url, fragment => $fragment ); | ||
| 681 | |||
| 682 | #this prints the link out for use in or working URL list | ||
| 683 | #for or web recursion process. | ||
| 684 | print $fh $url . "\n" if defined $fh and $url =~ m/$urlre/; | ||
| 685 | } | ||
| 686 | }; # END definition of $act_link function | ||
| 687 | return $act_attrib | ||
| 688 | } | ||
| 689 | |||
| 690 | |||
| 691 | sub chunk_accept_func ($){ | ||
| 692 | my $extractor=shift; | ||
| 693 | #my %settings=@_; - no use for now | ||
| 694 | |||
| 695 | #acceptfunc is a closeure with the link extractor which which is | ||
| 696 | #called by the user agent with the parts of the page as they | ||
| 697 | #are downloaded | ||
| 698 | |||
| 699 | #it then calls the link extractor which in turn calls the $act_url | ||
| 700 | #function | ||
| 701 | |||
| 702 | #Redirects etc. are handled by dying immediately. This is caught by | ||
| 703 | #the user agent which then gives it back in the response and never | ||
| 704 | #calls us again. | ||
| 705 | |||
| 706 | my $func = sub { | ||
| 707 | my ($chunk, $response, $protocol) = @_; | ||
| 708 | printf STDERR "Checking response\n" if $::verbose & 8; | ||
| 709 | die "only successes implemented now" unless $response->is_success(); | ||
| 710 | unless ( $response->content_type =~ m,html$,) { | ||
| 711 | print STDERR "Wrong content type. give up.\n"; | ||
| 712 | die "can only handle html so far" ; | ||
| 713 | } | ||
| 714 | $extractor->parse($chunk); | ||
| 715 | }; | ||
| 716 | return $func; | ||
| 717 | } | ||
| 718 | |||
| 719 | |||
| 720 | sub page_handler ($$) { | ||
| 721 | my $page_url=shift; | ||
| 722 | my $link_list_ref=shift; | ||
| 723 | if (@$link_list_ref) { | ||
| 724 | print STDERR "adding links in $page_url to index\n" | ||
| 725 | if $::verbose & 8; | ||
| 726 | $::index->add_list_first($page_url,$link_list_ref); | ||
| 727 | } | ||
| 728 | } | ||
| 729 | |||
| 730 | #link_handler should be called for each valid URL found. | ||
| 731 | #it will output it to the right places etc. | ||
| 732 | |||
| 733 | sub link_handler ($%) { | ||
| 734 | my $url=shift; | ||
| 735 | |||
| 736 | |||
| 737 | my %link_info=@_; | ||
| 738 | |||
| 739 | my $fragment=$link_info{fragment}; | ||
| 740 | print STDERR "ignoring fragment $fragment\n" | ||
| 741 | if $::verbose & 16 and defined $fragment; | ||
| 742 | |||
| 743 | |||
| 744 | $::checklist{$url} && do { | ||
| 745 | print STDERR "URL: $url already seen\n" | ||
| 746 | if $::verbose & 32; | ||
| 747 | return; | ||
| 748 | }; | ||
| 749 | $::checklist{$url} = 1; | ||
| 750 | |||
| 751 | $::write_links && do { | ||
| 752 | my $link = $::link_db{$url}; | ||
| 753 | |||
| 754 | unless ( defined $link) { | ||
| 755 | print STDERR "URL: $url being added to link database\n" | ||
| 756 | if $::verbose & 16; | ||
| 757 | $link=WWW::Link->new($url); | ||
| 758 | } else { | ||
| 759 | print STDERR "URL: $url already in link database\n" | ||
| 760 | if $::verbose & 16; | ||
| 761 | $link->last_refresh($::start_time); | ||
| 762 | } | ||
| 763 | |||
| 764 | $::link_db{$url}=$link; | ||
| 765 | }; | ||
| 766 | |||
| 767 | print URILIST $url . "\n" if defined $::out_uri_list; | ||
| 768 | } | ||
| 769 | |||
| 770 | |||
| 771 | |||
| 772 | #call tree | ||
| 773 | # | ||
| 774 | # LWP::UserAgent | ||
| 775 | # calls the chunk accept function | ||
| 776 | # which calls the HTML::LinkExtor | ||
| 777 | # which calls the attrib_func | ||
| 778 | # which calls link_handler() | ||
| 779 | |||
| 780 | # File::Find | ||
| 781 | # calls the act_on file func | ||
| 782 | # which calls the HTML::LinkExtor | ||
| 783 | # which calls the attrib_func | ||
| 784 | # which calls link_handler() | ||