blib/lib/WWW/Leech/Walker.pm untested

filter: Statementstmt 0.0% 0 / 112 Branchbran 0.0% 0 / 28 Conditioncond 0.0% 0 / 4 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 0.0% 0 / 13 Podpod 0.0% 0 / 7 totaltotal 0.0% 0 / 166 CC 15 SCAR 37.8
CC
15
Cov
0%
leech
110.0
new
12.0
log
6.0
CRAP 44.0
1 package WWW::Leech::Walker;
2 use strict;
3 use LWP;
4 use URI::URL;
5 use WWW::Leech::Parser;
6  
7 BEGIN {
8 use Exporter ();
9 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
10 $VERSION = '0.05';
11 @ISA = qw(Exporter);
12 @EXPORT = qw();
13 @EXPORT_OK = qw();
14 %EXPORT_TAGS = ();
15 }
16  
17 sub new{
18 my $class = shift;
19 my $this = shift;
20  
21 $this->{'url'} = url $this->{'url'};
22 $this->{'state'} ||= {};
23
24 $this->{'_stop_flag'} = undef;
25  
26 $this = bless $this, __PACKAGE__;
27  
28 $this->{'parser'} = new WWW::Leech::Parser($this->{'parser'});
29  
30 if(!$this->{'raw_preprocessor'}){
31 $this->{'raw_preprocessor'} = sub {
32 return shift();
33 }
34 }
35 if(!$this->{'error_handler'}){
36 $this->{'error_handler'} = sub {
37 my $this = shift;
38 my $item = shift;
39 my $err = shift;
40 die $err;
41 }
42 }
43
44 return $this;
45 }
46  
47 sub leech{
48 my $this = shift;
49 $this->{'_stop_flag'} = undef;
50  
51 my $current_url = $this->{'url'} or die("No url provided");
52  
53 while(1){
54  
55 $this->log("Getting links list: $current_url ");
56  
57 # get and parse link list
58 my $data = $this->_get($current_url, $this->{'post_data'});
59  
60 my $parse_result = $this->{'parser'}->parseList($data);
61  
62 my $links = [map { url($_, $this->{'url'})->abs } @{$parse_result->{'links'}}];
63 my $links_text = $parse_result->{'links_text'};
64  
65 $this->log("Links list length: ".scalar(@$links));
66  
67 if(scalar(@$links) == 0){
68 $this->log("No items links found, check your XPath: ".$this->{'parser'}->{'item:link'});
69  
70 $this->stop();
71 return;
72 }
73  
74 # filtering if required
75 if( $this->{'filter'} ){
76 $this->log("Filtering links");
77 $links = $this->{'filter'}->($links, $this, $links_text);
78 $this->log("Filtered links list length: ".scalar(@$links));
79  
80 return if($this->{'_stop_flag'});
81 }
82  
83 # get and parse individual items
84 my $i = 0;
85 my $llength = scalar(@$links);
86 foreach(@$links){
87  
88 $i++;
89  
90 # some log
91 my $ll = length($llength);
92
93 $this->log("\tGetting item (".sprintf("%0${ll}d",$i)."/$llength): $_ ");
94  
95 my $str = $this->_get($_);
96  
97 my $item = {'_url' => $_->as_string };
98  
99 # try to parse content
100 eval{
101 $item = $this->{'parser'}->parse($str);
102 };
103  
104 if($@){
105 $this->{'error_handler'}->($item,$this,$@);
106 } else {
107 $this->{'processor'}->($item,$this);
108 }
109  
110  
111 return if($this->{'_stop_flag'});
112 }
113  
114 if($parse_result->{'next_page'}){
115  
116 my $pnurl;
117  
118 if($this->{'next_page_link_post_process'}){
119 $pnurl = $this->{'next_page_link_post_process'}->($parse_result->{'next_page'}, $this, $current_url);
120 }
121  
122 if($pnurl){
123 $current_url = url($pnurl)->abs;
124 } else {
125 $current_url = url($parse_result->{'next_page'}, $this->{'url'})->abs;
126 }
127  
128 return if($this->{'_stop_flag'});
129  
130 } else {
131 $this->stop("No next page link found");
132 return;
133 }
134 }
135  
136
137 }
138  
139 sub getCurrentDOM{
140 my $this = shift;
141 return $this->{'parser'}->{'current_dom'};
142 }
143  
144 sub log{
145 my $this = shift;
146 my $message = shift;
147  
148 if($this->{'logger'}){
149 $this->{'logger'}->($message,$this);
150 }
151 }
152  
153 sub stop{
154 my $this = shift;
155 my $reason = shift;
156  
157 $this->{'_stop_flag'} = 1;
158 $this->log("\tWalker stopped.");
159 if($reason){
160 $this->log("\tReason: $reason.");
161 }
162 }
163  
164 sub _get{
165 my $this = shift;
166 my $url = shift;
167 my $post_data = shift;
168  
169 my $req;
170  
171 if( $post_data ){
172 $req = new HTTP::Request('POST',$url);
173 $req->content_type('application/x-www-form-urlencoded');
174 $req->content( $post_data );
175  
176 } else {
177 $req = new HTTP::Request('GET',$url);
178 }
179  
180  
181 my $res = $this->{'ua'}->request($req)->decoded_content();
182 $res = $this->{'raw_preprocessor'}->($res);
183  
184 return $res;
185
186 }
187  
188  
189 1;
190  
191  
192 __END__