| 1 | package WWW::Leech::Walker; | ||
| 2 | use strict; | ||
| 3 | use LWP; | ||
| 4 | use URI::URL; | ||
| 5 | use WWW::Leech::Parser; | ||
| 6 | |||
| 7 | BEGIN { | ||
| 8 | use Exporter (); | ||
| 9 | use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); | ||
| 10 | $VERSION = '0.05'; | ||
| 11 | @ISA = qw(Exporter); | ||
| 12 | @EXPORT = qw(); | ||
| 13 | @EXPORT_OK = qw(); | ||
| 14 | %EXPORT_TAGS = (); | ||
| 15 | } | ||
| 16 | |||
| 17 | sub new{ | ||
| 18 | my $class = shift; | ||
| 19 | my $this = shift; | ||
| 20 | |||
| 21 | $this->{'url'} = url $this->{'url'}; | ||
| 22 | $this->{'state'} ||= {}; | ||
| 23 | |||
| 24 | $this->{'_stop_flag'} = undef; | ||
| 25 | |||
| 26 | $this = bless $this, __PACKAGE__; | ||
| 27 | |||
| 28 | $this->{'parser'} = new WWW::Leech::Parser($this->{'parser'}); | ||
| 29 | |||
| 30 | if(!$this->{'raw_preprocessor'}){ | ||
| 31 | $this->{'raw_preprocessor'} = sub { | ||
| 32 | return shift(); | ||
| 33 | } | ||
| 34 | } | ||
| 35 | if(!$this->{'error_handler'}){ | ||
| 36 | $this->{'error_handler'} = sub { | ||
| 37 | my $this = shift; | ||
| 38 | my $item = shift; | ||
| 39 | my $err = shift; | ||
| 40 | die $err; | ||
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | return $this; | ||
| 45 | } | ||
| 46 | |||
| 47 | sub leech{ | ||
| 48 | my $this = shift; | ||
| 49 | $this->{'_stop_flag'} = undef; | ||
| 50 | |||
| 51 | my $current_url = $this->{'url'} or die("No url provided"); | ||
| 52 | |||
| 53 | while(1){ | ||
| 54 | |||
| 55 | $this->log("Getting links list: $current_url "); | ||
| 56 | |||
| 57 | # get and parse link list | ||
| 58 | my $data = $this->_get($current_url, $this->{'post_data'}); | ||
| 59 | |||
| 60 | my $parse_result = $this->{'parser'}->parseList($data); | ||
| 61 | |||
| 62 | my $links = [map { url($_, $this->{'url'})->abs } @{$parse_result->{'links'}}]; | ||
| 63 | my $links_text = $parse_result->{'links_text'}; | ||
| 64 | |||
| 65 | $this->log("Links list length: ".scalar(@$links)); | ||
| 66 | |||
| 67 | if(scalar(@$links) == 0){ | ||
| 68 | $this->log("No items links found, check your XPath: ".$this->{'parser'}->{'item:link'}); | ||
| 69 | |||
| 70 | $this->stop(); | ||
| 71 | return; | ||
| 72 | } | ||
| 73 | |||
| 74 | # filtering if required | ||
| 75 | if( $this->{'filter'} ){ | ||
| 76 | $this->log("Filtering links"); | ||
| 77 | $links = $this->{'filter'}->($links, $this, $links_text); | ||
| 78 | $this->log("Filtered links list length: ".scalar(@$links)); | ||
| 79 | |||
| 80 | return if($this->{'_stop_flag'}); | ||
| 81 | } | ||
| 82 | |||
| 83 | # get and parse individual items | ||
| 84 | my $i = 0; | ||
| 85 | my $llength = scalar(@$links); | ||
| 86 | foreach(@$links){ | ||
| 87 | |||
| 88 | $i++; | ||
| 89 | |||
| 90 | # some log | ||
| 91 | my $ll = length($llength); | ||
| 92 | |||
| 93 | $this->log("\tGetting item (".sprintf("%0${ll}d",$i)."/$llength): $_ "); | ||
| 94 | |||
| 95 | my $str = $this->_get($_); | ||
| 96 | |||
| 97 | my $item = {'_url' => $_->as_string }; | ||
| 98 | |||
| 99 | # try to parse content | ||
| 100 | eval{ | ||
| 101 | $item = $this->{'parser'}->parse($str); | ||
| 102 | }; | ||
| 103 | |||
| 104 | if($@){ | ||
| 105 | $this->{'error_handler'}->($item,$this,$@); | ||
| 106 | } else { | ||
| 107 | $this->{'processor'}->($item,$this); | ||
| 108 | } | ||
| 109 | |||
| 110 | |||
| 111 | return if($this->{'_stop_flag'}); | ||
| 112 | } | ||
| 113 | |||
| 114 | if($parse_result->{'next_page'}){ | ||
| 115 | |||
| 116 | my $pnurl; | ||
| 117 | |||
| 118 | if($this->{'next_page_link_post_process'}){ | ||
| 119 | $pnurl = $this->{'next_page_link_post_process'}->($parse_result->{'next_page'}, $this, $current_url); | ||
| 120 | } | ||
| 121 | |||
| 122 | if($pnurl){ | ||
| 123 | $current_url = url($pnurl)->abs; | ||
| 124 | } else { | ||
| 125 | $current_url = url($parse_result->{'next_page'}, $this->{'url'})->abs; | ||
| 126 | } | ||
| 127 | |||
| 128 | return if($this->{'_stop_flag'}); | ||
| 129 | |||
| 130 | } else { | ||
| 131 | $this->stop("No next page link found"); | ||
| 132 | return; | ||
| 133 | } | ||
| 134 | } | ||
| 135 | |||
| 136 | |||
| 137 | } | ||
| 138 | |||
| 139 | sub getCurrentDOM{ | ||
| 140 | my $this = shift; | ||
| 141 | return $this->{'parser'}->{'current_dom'}; | ||
| 142 | } | ||
| 143 | |||
| 144 | sub log{ | ||
| 145 | my $this = shift; | ||
| 146 | my $message = shift; | ||
| 147 | |||
| 148 | if($this->{'logger'}){ | ||
| 149 | $this->{'logger'}->($message,$this); | ||
| 150 | } | ||
| 151 | } | ||
| 152 | |||
| 153 | sub stop{ | ||
| 154 | my $this = shift; | ||
| 155 | my $reason = shift; | ||
| 156 | |||
| 157 | $this->{'_stop_flag'} = 1; | ||
| 158 | $this->log("\tWalker stopped."); | ||
| 159 | if($reason){ | ||
| 160 | $this->log("\tReason: $reason."); | ||
| 161 | } | ||
| 162 | } | ||
| 163 | |||
| 164 | sub _get{ | ||
| 165 | my $this = shift; | ||
| 166 | my $url = shift; | ||
| 167 | my $post_data = shift; | ||
| 168 | |||
| 169 | my $req; | ||
| 170 | |||
| 171 | if( $post_data ){ | ||
| 172 | $req = new HTTP::Request('POST',$url); | ||
| 173 | $req->content_type('application/x-www-form-urlencoded'); | ||
| 174 | $req->content( $post_data ); | ||
| 175 | |||
| 176 | } else { | ||
| 177 | $req = new HTTP::Request('GET',$url); | ||
| 178 | } | ||
| 179 | |||
| 180 | |||
| 181 | my $res = $this->{'ua'}->request($req)->decoded_content(); | ||
| 182 | $res = $this->{'raw_preprocessor'}->($res); | ||
| 183 | |||
| 184 | return $res; | ||
| 185 | |||
| 186 | } | ||
| 187 | |||
| 188 | |||
| 189 | 1; | ||
| 190 | |||
| 191 | |||
| 192 | __END__ |