blib/lib/HTML/Link_Miner.pm untested

filter: Statementstmt 0.0% 0 / 49 Branchbran 0.0% 0 / 16 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 9 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 79 CC 9 SCAR 28.6
CC
9
Cov
0%
_start_tag
42.0
new
6.0
_found_link
6.0
CRAP 17.5
1 package HTML::Link_Miner;
2 $REVISION=q$Revision: 1.4 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ );
3  
4 =head1 NAME
5  
6 HTML::Link_Miner - return links in a document along with information
7  
8 =head1 SYNOPSIS
9  
10 require HTML::Link_Miner;
11 $p = HTML::Link_Miner->new(\&cb, "http://www.sn.no/");
12  
13 #this is not correct
14 sub cb {
15 my($tag, %links) = @_;
16 print "$tag @{[%links]}\n";
17 }
18 $p->parse_file("index.html");
19  
20 =head1 DESCRIPTION
21  
22 This is currently just an idea and is not implemented yet
23  
24 HTML::Link_Miner is a link extractor. The I<HTML::Link_Miner> is a
25 subclass of I<HTML::Parser>. This means that the document should be
26 given to the parser by calling the $p->parse() or $p->parse_file()
27 methods.
28  
29 The difference between HTML::LinkExtor and this module is that we try
30 to find all of the useful information we can about each link from the
31 document.
32  
33 With the HREF from C<< <A> >> tags we return the text between the
34 start and end tags.
35  
36 #With the HREF from C<< <IMG> >> tags we return any C<ALT> text.
37  
38 With other links, we return the attribute name
39  
40 =head1 INTERFACE
41  
42 call backs for each link type??
43  
44 =cut
45  
46 require HTML::Parser;
47 @ISA = qw(HTML::Parser);
48 $VERSION = sprintf("%d.%02d", q$Revision: 1.4 $ =~ /(\d+)\.(\d+)/);
49  
50 use strict;
51 use HTML::Tagset ();
52  
53 =over 4
54  
55 =item $p = HTML::LinkExtor->new([$callback[, $base]])
56  
57 The constructor takes two optional arguments. The first is a reference
58 to a callback routine. It will be called as links are found. If a
59 callback is not provided, then links are just accumulated internally
60 and can be retrieved by calling the $p->links() method.
61  
62 The $base argument is an optional base URL used to absolutize all URLs found.
63 You need to have the I<URI> module installed if you provide $base.
64  
65 The callback is called with the lowercase tag name as first argument,
66 and then all link attributes as separate key/value pairs. All
67 non-link attributes are removed.
68  
69 =cut
70  
71 sub new
72 {
73 my($class, $cb, $base) = @_;
74 my $self = $class->SUPER::new(start_h => ["_start_tag",
75 "self,tagname,attr"]);
76 $self->{extractlink_cb} = $cb;
77 if ($base) {
78 require URI;
79 $self->{extractlink_base} = URI->new($base);
80 }
81 $self;
82 }
83  
84 sub _start_tag
85 {
86 my($self, $tag, $attr) = @_;
87 return unless exists $HTML::Tagset::linkElements{$tag};
88  
89 my $base = $self->{extractlink_base};
90 my $links = $HTML::Tagset::linkElements{$tag};
91 $links = [$links] unless ref $links;
92  
93 my @links;
94 my $a;
95 for $a (@$links) {
96 next unless exists $attr->{$a};
97 push(@links, $a, $base ? URI->new($attr->{$a}, $base)->abs($base)
98 : $attr->{$a});
99 }
100 return unless @links;
101 $self->_found_link($tag, @links);
102 }
103  
104 sub _found_link
105 {
106 my $self = shift;
107 my $cb = $self->{extractlink_cb};
108 if ($cb) {
109 &$cb(@_);
110 } else {
111 push(@{$self->{'links'}}, [@_]);
112 }
113 }
114  
115 =item $p->links
116  
117 Returns a list of all links found in the document. The returned
118 values will be anonymous arrays with the follwing elements:
119  
120 [$tag, $attr => $url1, $attr2 => $url2,...]
121  
122 The $p->links method will also truncate the internal link list. This
123 means that if the method is called twice without any parsing
124 between them the second call will return an empty list.
125  
126 Also note that $p->links will always be empty if a callback routine
127 was provided when the I<HTML::LinkExtor> was created.
128  
129 =cut
130  
131 sub links
132 {
133 my $self = shift;
134 exists($self->{'links'}) ? @{delete $self->{'links'}} : ();
135 }
136  
137 # We override the parse_file() method so that we can clear the links
138 # before we start a new file.
139 sub parse_file
140 {
141 my $self = shift;
142 delete $self->{'links'};
143 $self->SUPER::parse_file(@_);
144 }
145  
146 =back
147  
148 =head1 EXAMPLE
149  
150 This is an example showing how you can extract links from a document
151 received using LWP:
152  
153 use LWP::UserAgent;
154 use HTML::LinkExtor;
155 use URI;
156  
157 $url = "http://www.sn.no/"; # for instance
158 $ua = new LWP::UserAgent;
159  
160 # Set up a callback that collect image links
161 my @imgs = ();
162 sub callback {
163 my($tag, %attr) = @_;
164 return if $tag ne 'img'; # we only look closer at <img ...>
165 push(@imgs, values %attr);
166 }
167  
168 # Make the parser. Unfortunately, we don't know the base yet
169 # (it might be diffent from $url)
170 $p = HTML::LinkExtor->new(\&callback);
171  
172 # Request document and parse it as it arrives
173 $res = $ua->request(HTTP::Request->new(GET => $url),
174 sub {$p->parse($_[0])});
175  
176 # Expand all image URLs to absolute ones
177 my $base = $res->base;
178 @imgs = map { $_ = url($_, $base)->abs; } @imgs;
179  
180 # Print them out
181 print join("\n", @imgs), "\n";
182  
183 =head1 SEE ALSO
184  
185 L<HTML::Parser>, L<HTML::Tagset>, L<LWP>, L<URI>
186  
187 =head1 COPYRIGHT
188  
189 Based on LinkExtor and Copyright 1996-2000 Gisle Aas.
190  
191 Copyright 2001 Michael De La Rue
192  
193 This library is free software; you can redistribute it and/or
194 modify it under the same terms as Perl itself.
195  
196 =cut
197  
198 1;
199