| 1 | package HTML::Link_Miner; | ||
| 2 | $REVISION=q$Revision: 1.4 $ ; $VERSION = sprintf ( "%d.%02d", $REVISION =~ /(\d+).(\d+)/ ); | ||
| 3 | |||
| 4 | =head1 NAME | ||
| 5 | |||
| 6 | HTML::Link_Miner - return links in a document along with information | ||
| 7 | |||
| 8 | =head1 SYNOPSIS | ||
| 9 | |||
| 10 | require HTML::Link_Miner; | ||
| 11 | $p = HTML::Link_Miner->new(\&cb, "http://www.sn.no/"); | ||
| 12 | |||
| 13 | #this is not correct | ||
| 14 | sub cb { | ||
| 15 | my($tag, %links) = @_; | ||
| 16 | print "$tag @{[%links]}\n"; | ||
| 17 | } | ||
| 18 | $p->parse_file("index.html"); | ||
| 19 | |||
| 20 | =head1 DESCRIPTION | ||
| 21 | |||
| 22 | This is currently just an idea and is not implemented yet | ||
| 23 | |||
| 24 | HTML::Link_Miner is a link extractor. The I<HTML::Link_Miner> is a | ||
| 25 | subclass of I<HTML::Parser>. This means that the document should be | ||
| 26 | given to the parser by calling the $p->parse() or $p->parse_file() | ||
| 27 | methods. | ||
| 28 | |||
| 29 | The difference between HTML::LinkExtor and this module is that we try | ||
| 30 | to find all of the useful information we can about each link from the | ||
| 31 | document. | ||
| 32 | |||
| 33 | With the HREF from C<< <A> >> tags we return the text between the | ||
| 34 | start and end tags. | ||
| 35 | |||
| 36 | #With the HREF from C<< <IMG> >> tags we return any C<ALT> text. | ||
| 37 | |||
| 38 | With other links, we return the attribute name | ||
| 39 | |||
| 40 | =head1 INTERFACE | ||
| 41 | |||
| 42 | call backs for each link type?? | ||
| 43 | |||
| 44 | =cut | ||
| 45 | |||
| 46 | require HTML::Parser; | ||
| 47 | @ISA = qw(HTML::Parser); | ||
| 48 | $VERSION = sprintf("%d.%02d", q$Revision: 1.4 $ =~ /(\d+)\.(\d+)/); | ||
| 49 | |||
| 50 | use strict; | ||
| 51 | use HTML::Tagset (); | ||
| 52 | |||
| 53 | =over 4 | ||
| 54 | |||
| 55 | =item $p = HTML::LinkExtor->new([$callback[, $base]]) | ||
| 56 | |||
| 57 | The constructor takes two optional arguments. The first is a reference | ||
| 58 | to a callback routine. It will be called as links are found. If a | ||
| 59 | callback is not provided, then links are just accumulated internally | ||
| 60 | and can be retrieved by calling the $p->links() method. | ||
| 61 | |||
| 62 | The $base argument is an optional base URL used to absolutize all URLs found. | ||
| 63 | You need to have the I<URI> module installed if you provide $base. | ||
| 64 | |||
| 65 | The callback is called with the lowercase tag name as first argument, | ||
| 66 | and then all link attributes as separate key/value pairs. All | ||
| 67 | non-link attributes are removed. | ||
| 68 | |||
| 69 | =cut | ||
| 70 | |||
| 71 | sub new | ||
| 72 | { | ||
| 73 | my($class, $cb, $base) = @_; | ||
| 74 | my $self = $class->SUPER::new(start_h => ["_start_tag", | ||
| 75 | "self,tagname,attr"]); | ||
| 76 | $self->{extractlink_cb} = $cb; | ||
| 77 | if ($base) { | ||
| 78 | require URI; | ||
| 79 | $self->{extractlink_base} = URI->new($base); | ||
| 80 | } | ||
| 81 | $self; | ||
| 82 | } | ||
| 83 | |||
| 84 | sub _start_tag | ||
| 85 | { | ||
| 86 | my($self, $tag, $attr) = @_; | ||
| 87 | return unless exists $HTML::Tagset::linkElements{$tag}; | ||
| 88 | |||
| 89 | my $base = $self->{extractlink_base}; | ||
| 90 | my $links = $HTML::Tagset::linkElements{$tag}; | ||
| 91 | $links = [$links] unless ref $links; | ||
| 92 | |||
| 93 | my @links; | ||
| 94 | my $a; | ||
| 95 | for $a (@$links) { | ||
| 96 | next unless exists $attr->{$a}; | ||
| 97 | push(@links, $a, $base ? URI->new($attr->{$a}, $base)->abs($base) | ||
| 98 | : $attr->{$a}); | ||
| 99 | } | ||
| 100 | return unless @links; | ||
| 101 | $self->_found_link($tag, @links); | ||
| 102 | } | ||
| 103 | |||
| 104 | sub _found_link | ||
| 105 | { | ||
| 106 | my $self = shift; | ||
| 107 | my $cb = $self->{extractlink_cb}; | ||
| 108 | if ($cb) { | ||
| 109 | &$cb(@_); | ||
| 110 | } else { | ||
| 111 | push(@{$self->{'links'}}, [@_]); | ||
| 112 | } | ||
| 113 | } | ||
| 114 | |||
| 115 | =item $p->links | ||
| 116 | |||
| 117 | Returns a list of all links found in the document. The returned | ||
| 118 | values will be anonymous arrays with the follwing elements: | ||
| 119 | |||
| 120 | [$tag, $attr => $url1, $attr2 => $url2,...] | ||
| 121 | |||
| 122 | The $p->links method will also truncate the internal link list. This | ||
| 123 | means that if the method is called twice without any parsing | ||
| 124 | between them the second call will return an empty list. | ||
| 125 | |||
| 126 | Also note that $p->links will always be empty if a callback routine | ||
| 127 | was provided when the I<HTML::LinkExtor> was created. | ||
| 128 | |||
| 129 | =cut | ||
| 130 | |||
| 131 | sub links | ||
| 132 | { | ||
| 133 | my $self = shift; | ||
| 134 | exists($self->{'links'}) ? @{delete $self->{'links'}} : (); | ||
| 135 | } | ||
| 136 | |||
| 137 | # We override the parse_file() method so that we can clear the links | ||
| 138 | # before we start a new file. | ||
| 139 | sub parse_file | ||
| 140 | { | ||
| 141 | my $self = shift; | ||
| 142 | delete $self->{'links'}; | ||
| 143 | $self->SUPER::parse_file(@_); | ||
| 144 | } | ||
| 145 | |||
| 146 | =back | ||
| 147 | |||
| 148 | =head1 EXAMPLE | ||
| 149 | |||
| 150 | This is an example showing how you can extract links from a document | ||
| 151 | received using LWP: | ||
| 152 | |||
| 153 | use LWP::UserAgent; | ||
| 154 | use HTML::LinkExtor; | ||
| 155 | use URI; | ||
| 156 | |||
| 157 | $url = "http://www.sn.no/"; # for instance | ||
| 158 | $ua = new LWP::UserAgent; | ||
| 159 | |||
| 160 | # Set up a callback that collect image links | ||
| 161 | my @imgs = (); | ||
| 162 | sub callback { | ||
| 163 | my($tag, %attr) = @_; | ||
| 164 | return if $tag ne 'img'; # we only look closer at <img ...> | ||
| 165 | push(@imgs, values %attr); | ||
| 166 | } | ||
| 167 | |||
| 168 | # Make the parser. Unfortunately, we don't know the base yet | ||
| 169 | # (it might be diffent from $url) | ||
| 170 | $p = HTML::LinkExtor->new(\&callback); | ||
| 171 | |||
| 172 | # Request document and parse it as it arrives | ||
| 173 | $res = $ua->request(HTTP::Request->new(GET => $url), | ||
| 174 | sub {$p->parse($_[0])}); | ||
| 175 | |||
| 176 | # Expand all image URLs to absolute ones | ||
| 177 | my $base = $res->base; | ||
| 178 | @imgs = map { $_ = url($_, $base)->abs; } @imgs; | ||
| 179 | |||
| 180 | # Print them out | ||
| 181 | print join("\n", @imgs), "\n"; | ||
| 182 | |||
| 183 | =head1 SEE ALSO | ||
| 184 | |||
| 185 | L<HTML::Parser>, L<HTML::Tagset>, L<LWP>, L<URI> | ||
| 186 | |||
| 187 | =head1 COPYRIGHT | ||
| 188 | |||
| 189 | Based on LinkExtor and Copyright 1996-2000 Gisle Aas. | ||
| 190 | |||
| 191 | Copyright 2001 Michael De La Rue | ||
| 192 | |||
| 193 | This library is free software; you can redistribute it and/or | ||
| 194 | modify it under the same terms as Perl itself. | ||
| 195 | |||
| 196 | =cut | ||
| 197 | |||
| 198 | 1; | ||
| 199 |