blib/lib/WWW/WebArchive/WaybackMachine.pm untested

filter: Statementstmt 0.0% 0 / 126 Branchbran 0.0% 0 / 70 Conditioncond 0.0% 0 / 14 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 0.0% 0 / 7 Podpod 0.0% 0 / 3 totaltotal 0.0% 0 / 228 CC 36 SCAR 58.9
CC
36
Cov
0%
restore
600.0
clean_html
72.0
mirror
42.0
CRAP 362.8
1  
2 #############################################################################
3 ## $Id: WaybackMachine.pm 6702 2006-07-25 01:43:27Z spadkins $
4 #############################################################################
5  
6 use strict;
7  
8 package WWW::WebArchive::WaybackMachine;
9  
10 use WWW::WebArchive::Agent;
11  
12 use vars qw($VERSION @ISA);
13 $VERSION = "0.50";
14 @ISA = ("WWW::WebArchive::Agent");
15  
16 use WWW::Mechanize;
17  
18 sub restore {
19 &App::sub_entry if ($App::trace);
20 my ($self, $options) = @_;
21  
22 my $dir = $options->{dir};
23 $dir = $self->{dir} if (!defined $dir);
24 $dir = "." if (!defined $dir);
25 $dir .= "/$self->{name}";
26  
27 my $url = $options->{url} || die "restore(): URL not provided";
28 $url =~ s!/$!!;
29 if ($url !~ /^[a-z]+:/) {
30 $url = "http://$url";
31 }
32 my $domain = $url;
33 $domain =~ s!^[a-z]+://!!;
34 $domain =~ s!/.*!!;
35 my $seclvl_domain = $domain;
36 if ($seclvl_domain =~ /([^\.]+\.[^\.]+)$/) {
37 $seclvl_domain = $1;
38 }
39  
40 my $verbose = $options->{verbose};
41 $verbose = $self->{verbose} if (!defined $verbose);
42 $verbose = 0 if (!defined $verbose);
43  
44 ###################################################################
45 # Initialize User Agent
46 ###################################################################
47 my $ua = WWW::Mechanize->new();
48 $ua->agent_alias("Windows IE 6");
49 $ua->stack_depth(1); # limit the number of pages we remember to 1 (one back() allowed)
50  
51 ###################################################################
52 # Search Internet Archive Wayback Machine for cached documents
53 ###################################################################
54 my (%link, @links, $link);
55 my ($done, $next_url, $link_text, $link_url);
56 my ($link_text2, $link_url2);
57 $done = 0;
58 print "Restoring [$url]\n" if ($verbose);
59 $ua->get("http://web.archive.org/web/*sr_1nr_100/$url*");
60 $self->check_status($ua);
61 while (!$done) {
62 @links = $ua->links();
63 $done = 1;
64 foreach $link (@links) {
65 $link_text = $link->text();
66 $link_url = $link->url_abs();
67 printf("> Link: %-40s %s\n", $link_text, $link_url) if ($verbose >= 3);
68 if ($link_url =~ m!^http://web.archive.org/web/.*$seclvl_domain! &&
69 $link_text =~ m!$seclvl_domain!) {
70 printf(">> Archived Document Found: http://%s\n", $link_text) if ($verbose);
71 $link{$link_text} = $link;
72 }
73  
74 if ($link_text eq "Next") {
75 $next_url = $link->url_abs();
76 }
77 }
78 if ($next_url) {
79 #print "Next: $next_url\n";
80 $ua->get($next_url);
81 $self->check_status($ua);
82 $done = 0;
83 $next_url = "";
84 }
85 }
86  
87 ###################################################################
88 # Mirror cached documents to local file system
89 ###################################################################
90 my ($action, $file);
91 foreach $link_text (sort keys %link) {
92 $link = $link{$link_text};
93 $link_url = $link->url_abs();
94  
95 if ($link_url =~ m!^http://web.archive.org/web/([^/]+)/(.*)$!) {
96 $action = $1;
97 $file = $2;
98 if ($file =~ m!/$!) {
99 print "Probably a directory index [$file] : not retrieving\n" if ($verbose >= 2);
100 }
101 elsif ($file =~ m!/[^/\\\.]+$!) {
102 print "Probably a directory index [$file] : not retrieving\n" if ($verbose >= 2);
103 }
104 elsif ($file =~ m!/\?[DMNS]=[DA]$!) {
105 print "Probably a directory index [$file] : not retrieving\n" if ($verbose >= 2);
106 }
107 else {
108 if ($action eq "*hh_") {
109 $self->mirror($ua, "http://web.archive.org/http://$file", $file, $dir, $domain);
110 #print "Getting historical versions [$link_url] ...\n" if ($verbose >= 1);
111 #$ua->get($link_url);
112 #$self->check_status($ua);
113 #if ($ua->success()) {
114 # @links = $ua->links();
115 # foreach $link (@links) {
116 # $link_text2 = $link->text();
117 # $link_url2 = $link->url_abs();
118 # if ($link_url2 =~ m!^http://web.archive.org/web/.*$domain! &&
119 # $link_text2 =~ m!$domain!) {
120 # #printf(">> Archived Document Found: http://%s\n", $link_text) if ($verbose);
121 # printf("> Link: %-40s %s\n", $link_text2, $link_url2);
122 # #$link{$link_text} = $link;
123 # }
124 # }
125 #}
126 #else {
127 # print "Can't get URL [$link_url]\n";
128 #}
129 }
130 elsif ($action =~ /^[0-9]+$/) {
131 $self->mirror($ua, $link_url, $file, $dir, $domain);
132 }
133 else {
134 print "Unknown link type [$link_url]\n";
135 }
136 }
137 }
138 else {
139 print "Unknown link type [$link_url]\n";
140 }
141 }
142  
143 &App::sub_exit() if ($App::trace);
144 }
145  
146 sub mirror {
147 &App::sub_entry if ($App::trace);
148 my ($self, $ua, $url, $file, $basedir, $domain) = @_;
149 if (! -f "$basedir/$file" || $App::options{clobber}) {
150 $ua->get($url);
151 $self->check_status($ua);
152 if ($ua->success()) {
153 my $content = $ua->content();
154 my $content_type = $ua->ct();
155 if ($content_type eq "text/html") {
156 $content = $self->clean_html($content, $file, $domain);
157 }
158 my $len = length($content);
159 $self->write_file("$basedir/$file", $content);
160 print "Wrote file [$file] ($len bytes)\n";
161 }
162 else {
163 print "Missed file [$file]\n";
164 }
165 }
166 else {
167 print "File exists [$file]\n";
168 }
169 &App::sub_exit() if ($App::trace);
170 }
171  
172 sub clean_html {
173 &App::sub_entry if ($App::trace);
174 my ($self, $html, $file, $domain) = @_;
175  
176 # Unix files. No CR's allowed.
177 $html =~ s/\r//g;
178  
179 # clean up weird additions to <BASE>. Unfortunately, this wipes out real uses of the <BASE> tag in the original doc.
180 $html =~ s#<!-- base href="[^"<>]*" -->##;
181 $html =~ s#<BASE [^<>]*>\s*##si; # the first one was put in by Internet Archive
182 $html =~ s#<(BASE [^<>]*)>\s*#<!-- $1 -->#si; # there may be a real <BASE> tag. keep in comment. all URL's must be relative.
183 #$html =~ s#<link rel="stylesheet" type="text/css" href="[^"]*/style.css">#<link rel="stylesheet" type="text/css" href="style.css">\n#;
184  
185 # clean up the spacing to get rid of extraneous lines
186 $html =~ s#<html>\s*#<html>\n#si;
187 $html =~ s#<head>\s*#<head>\n#si;
188 $html =~ s#</title>\s*#</title>\n#si;
189 $html =~ s#</head>\s*#</head>\n#;
190  
191 # remove a really odd background="foo.html" attribute from the <body>
192 $html =~ s#<body([^<>]*) background="[^"]*.html?"#<body$1#si;
193  
194 # try to rewrite web archive links (which have been made absolute)
195 if ($html =~ s#http://web.archive.org/[^"]*(http://[^"]*)#$1#g) {
196 # if we succeeded and we know the filename and domain ...
197 if ($file && $domain && $html =~ m#http://$domain#) {
198 my $reldir = ""; # compute a relative root from the filename
199 my $absdir = $file;
200 $absdir =~ s#^[^/]+##; # trim off domain part
201 $absdir =~ s#[^/]+$##; # trim off file part
202 $absdir = "/" if (!$absdir);
203 while (1) {
204 # print "Substituting [$domain$absdir] for [$reldir]\n";
205 $html =~ s#http://$domain$absdir#$reldir#g; # substitute absolute links to file in the domain with relative paths
206 last if ($absdir eq "/");
207 $absdir =~ s#[^/]+/$##; # trim off file part
208 $absdir = "/" if (!$absdir);
209 $reldir .= "../";
210 }
211 # print "Substituting [$domain] for [$reldir]\n";
212 $html =~ s#http://$domain#$reldir#g; # substitute absolute links to file in the domain with relative paths
213 }
214 }
215  
216 # get rid of a comment and some javascript added by the Internet Archive
217 $html =~ s#<!--\s*SOME\s*LINK\s*HREF[^<>]>\s*##s;
218 $html =~ s#<!-- SOME LINK HREF'S ON THIS PAGE HAVE BEEN REWRITTEN BY THE WAYBACK MACHINE\s*##s;
219 $html =~ s#OF THE INTERNET ARCHIVE IN ORDER TO PRESERVE THE TEMPORAL INTEGRITY OF THE SESSION. -->\s*##s;
220 $html =~ s#<script[^<>]*>\s*<!--\s*// FILE ARCHIVED[^>]*>\n</script>\s*##si;
221 $html =~ s#<!-- SOME FRAME SRC'S ON THIS PAGE HAVE BEEN REWRITTEN BY THE WAYBACK MACHINE\s*##s;
222 $html =~ s#<!--\s*// FILE ARCHIVED ON [^<>]>\s*##s;
223  
224 &App::sub_exit($html) if ($App::trace);
225 return($html);
226 }
227  
228 =head1 NAME
229  
230 WWW::WebArchive::WaybackMachine - An agent to retrieve files from Internet Archive's Wayback Machine (www.archive.org)
231  
232 =head1 SYNOPSIS
233  
234 NOTE: You probably want to use this module through the WWW::WebArchive API.
235 If not, it's up to you to read the code and figure out how to use this module.
236  
237 =head1 DESCRIPTION
238  
239 An agent to retrieve files from Internet Archive's Wayback Machine (www.archive.org)
240  
241 =head1 ACKNOWLEDGEMENTS
242  
243 * Author: Stephen Adkins <spadkins@gmail.com>
244 * License: This is free software. It is licensed under the same terms as Perl itself.
245  
246 =head1 SEE ALSO
247  
248 L<WWW::WebArchive::Agent>, L<WWW::WebArchive>, L<WWW::Mechanize>
249  
250 =cut
251  
252 1;
253