| 1 | package WWW::Leech::Parser; | ||
| 2 | use strict; | ||
| 3 | use HTML::TreeBuilder::XPath; | ||
| 4 | use Encode qw|is_utf8 encode decode|; | ||
| 5 | use utf8; | ||
| 6 | |||
| 7 | BEGIN { | ||
| 8 | use Exporter (); | ||
| 9 | use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); | ||
| 10 | $VERSION = '0.04'; | ||
| 11 | @ISA = qw(Exporter); | ||
| 12 | @EXPORT = qw(); | ||
| 13 | @EXPORT_OK = qw(); | ||
| 14 | %EXPORT_TAGS = (); | ||
| 15 | } | ||
| 16 | |||
| 17 | sub new{ | ||
| 18 | |||
| 19 | my $class = shift; | ||
| 20 | my $params = shift; | ||
| 21 | |||
| 22 | |||
| 23 | my $fields = {}; | ||
| 24 | |||
| 25 | if(ref($params->{'fields'}) eq 'ARRAY'){ | ||
| 26 | foreach( @{$params->{'fields'}} ){ | ||
| 27 | $fields->{$_->{'name'}} = $_; | ||
| 28 | } | ||
| 29 | $params->{'fields'} = $fields; | ||
| 30 | } | ||
| 31 | |||
| 32 | $params->{'current_dom'} = undef; | ||
| 33 | |||
| 34 | return bless $params, __PACKAGE__; | ||
| 35 | |||
| 36 | } | ||
| 37 | |||
| 38 | sub parseList{ | ||
| 39 | my $this = shift; | ||
| 40 | my $str = shift; | ||
| 41 | |||
| 42 | $this->{'current_dom'} = HTML::TreeBuilder::XPath->new; | ||
| 43 | $this->{'current_dom'}->parse_content($str); | ||
| 44 | |||
| 45 | my $links = []; | ||
| 46 | my $links_text = []; | ||
| 47 | foreach(@{$this->{'current_dom'}->findnodes($this->{'item:link'})}){ | ||
| 48 | push(@$links, $_->attr('href')); | ||
| 49 | push(@$links_text, $_->as_text()); | ||
| 50 | } | ||
| 51 | |||
| 52 | my $np = undef; | ||
| 53 | if($this->{'nextpage:link'}){ | ||
| 54 | $np = $this->{'current_dom'}->findnodes($this->{'nextpage:link'})->[0]; | ||
| 55 | |||
| 56 | if($np){ | ||
| 57 | $np = $np->attr('href'); | ||
| 58 | } | ||
| 59 | } | ||
| 60 | |||
| 61 | return { | ||
| 62 | 'links' => $links, | ||
| 63 | 'links_text' => $links_text, | ||
| 64 | 'next_page' => $np | ||
| 65 | } | ||
| 66 | } | ||
| 67 | |||
| 68 | sub parse{ | ||
| 69 | my $this = shift; | ||
| 70 | my $str = shift; | ||
| 71 | |||
| 72 | if(!is_utf8($str)){ | ||
| 73 | $str = decode('UTF-8',$str); | ||
| 74 | } | ||
| 75 | |||
| 76 | $this->{'current_dom'}= HTML::TreeBuilder::XPath->new; | ||
| 77 | $this->{'current_dom'}->ignore_unknown(0); | ||
| 78 | $this->{'current_dom'}->parse_content($str); | ||
| 79 | |||
| 80 | my $item = {}; | ||
| 81 | |||
| 82 | foreach my $field_name ( keys(%{$this->{'fields'}}) ){ | ||
| 83 | |||
| 84 | my $field = $this->{'fields'}->{$field_name}; | ||
| 85 | |||
| 86 | if(ref($field) ne 'HASH') { | ||
| 87 | $field = { | ||
| 88 | 'xpath' => $field | ||
| 89 | }; | ||
| 90 | } | ||
| 91 | |||
| 92 | $field->{'name'} = $field_name; | ||
| 93 | |||
| 94 | if($field->{'xpath'}){ | ||
| 95 | if(!is_utf8($field->{'xpath'})){ | ||
| 96 | $field->{'xpath'} = decode('UTF-8',$field->{'xpath'}); | ||
| 97 | } | ||
| 98 | |||
| 99 | my $type = ($field->{'type'} ? $field->{'type'} : 'text'); | ||
| 100 | my $wantarray = ( $field_name =~ /\[\]$/ ? 1 : undef ); | ||
| 101 | |||
| 102 | my $value; | ||
| 103 | |||
| 104 | if($wantarray){ | ||
| 105 | $value = []; | ||
| 106 | |||
| 107 | if($type eq 'html'){ | ||
| 108 | |||
| 109 | foreach my $n ($this->{'current_dom'}->findnodes($field->{'xpath'})){ | ||
| 110 | push(@$value,$n->as_XML); | ||
| 111 | } | ||
| 112 | |||
| 113 | } else { | ||
| 114 | $value = [$this->{'current_dom'}->findvalues($field->{'xpath'})]; | ||
| 115 | |||
| 116 | if($type eq 'unique'){ | ||
| 117 | my %u; | ||
| 118 | $u{$_} = 1 foreach(@$value); | ||
| 119 | $value = [keys(%u)]; | ||
| 120 | } | ||
| 121 | } | ||
| 122 | } else { | ||
| 123 | if($type eq 'html'){ | ||
| 124 | |||
| 125 | if([$this->{'current_dom'}->findnodes($field->{'xpath'})]->[0]){ | ||
| 126 | $value = [$this->{'current_dom'}->findnodes($field->{'xpath'})]->[0]->as_XML; | ||
| 127 | } | ||
| 128 | |||
| 129 | } | ||
| 130 | else { | ||
| 131 | $value = $this->{'current_dom'}->findvalue($field->{'xpath'}); | ||
| 132 | } | ||
| 133 | |||
| 134 | if($type eq 'int'){ | ||
| 135 | $value =~ s/[^\d]//g; | ||
| 136 | } | ||
| 137 | } | ||
| 138 | |||
| 139 | if($field->{'filter'}){ | ||
| 140 | $value = $field->{'filter'}->($value,$field); | ||
| 141 | } | ||
| 142 | |||
| 143 | $item->{$field_name} = $value; | ||
| 144 | } | ||
| 145 | } | ||
| 146 | |||
| 147 | return $item; | ||
| 148 | |||
| 149 | } | ||
| 150 | |||
| 151 | 1; | ||
| 152 | |||
| 153 | __END__ |