blib/lib/LWP/UserAgent/ProxyHopper/Base.pm

filter: Statementstmt 90.4% 19 / 21 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 7 / 7 Podpod n/a totaltotal 92.8% 26 / 28 CC 1 SCAR 0.4
CC
1
Cov
90%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
↑ Summary
1 package LWP::UserAgent::ProxyHopper::Base;
2  
3 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 our $VERSION = '0.004';
7  
8 2 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 2 ▶use Devel::TakeHashArgs;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 2 ▶use List::MoreUtils 'uniq';
Subroutine BEGIN: called CC=1 SCAR=0.0
11 2 ▶use WWW::FreeProxyListsCom;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 0 ▶use WWW::Proxy4FreeCom;
Subroutine BEGIN: called CC=1 SCAR=2.6
13 use base 'Class::Data::Accessor';
14 __PACKAGE__->mk_classaccessors(qw(
15 proxify_list
16 proxify_bad_list
17 proxify_real_bad_list
18 proxify_working_list
19 proxify_schemes
20 proxify_retries
21 proxify_debug
22 proxify_current
23 _proxify_last_load_args
24 _proxify_freeproxylists_obj
25 _proxify_proxy4free_obj
26 ));
27  
28 sub proxify_load {
29 my $self = shift;
30 get_args_as_hash(\@_, \my %args, {
31 freeproxylists => 1,
32 plan_b => 1,
33 proxy4free => 0,
34 timeout => 20,
35 debug => 0,
36 retries => 5,
37 extra_proxies => [],
38 schemes => 'http',
39 get_list_args => {
40 freeproxylists => [ ],
41 proxy4free => [ ],
42 },
43 },
44 ) or croak $@;
45  
46 $self->_proxify_last_load_args( \%args );
47  
48 my @proxies;
49  
50 if ( $args{freeproxylists} ) {
51 my $obj = $self->_proxify_freeproxylists_obj(
52 WWW::FreeProxyListsCom->new( timeout => $args{timeout} )
53 );
54  
55 my $list_ref
56 = $obj->get_list( @{$args{get_list_args}{freeproxylists}} );
57 if ( defined $list_ref ) {
58 push @proxies, map { "http://$_->{ip}:$_->{port}/" } @$list_ref;
59 }
60 else {
61 $args{debug}
62 and carp 'Failed while trying to get a proxy list from '
63 . 'http://freeproxylists.com: ' . $obj->error;
64 }
65 }
66  
67 if ( $args{proxy4free} or ( !@proxies and $args{plan_b} ) ) {
68 my $obj = $self->_proxify_proxy4free_obj(
69 WWW::Proxy4FreeCom->new( timeout => $args{timeout} )
70 );
71  
72 my $list_ref = $obj->get_list( @{$args{get_list_args}{proxy4free}} );
73  
74 if ( defined $list_ref ) {
75 push @proxies, map { "http://$_->{ip}:$_->{port}/" } @$list_ref;
76 }
77 else {
78 $args{debug}
79 and carp 'Failed while trying to get a proxy list from '
80 . 'http://proxy4free.com: ' . $obj->error;
81 }
82 }
83  
84 unshift @proxies, @{ $args{extra_proxies} };
85  
86 croak q|Don't have ANY proxy addresses :(|
87 unless @proxies;
88  
89 @proxies = uniq @proxies;
90  
91 $args{debug}
92 and carp "Got " . @proxies . " proxies in total";
93  
94 $self->proxify_retries( $args{retries} );
95 $self->proxify_schemes( $args{schemes} );
96 $self->proxify_debug( $args{debug } );
97 $self->proxify_working_list( [] );
98 $self->proxify_bad_list( [] );
99 $self->proxify_real_bad_list( [] );
100  
101 return $self->proxify_list( \@proxies );
102 }
103  
104 sub proxify_get { return shift->_proxify_try_request( 'get', \@_ ); }
105 sub proxify_post { return shift->_proxify_try_request( 'post', \@_ ); }
106 sub proxify_request { return shift->_proxify_try_request( 'request', \@_ ); }
107 sub proxify_head { return shift->_proxify_try_request( 'head', \@_ ); }
108 sub proxify_mirror { return shift->_proxify_try_request( 'mirror', \@_ ); }
109 sub proxify_simple_request {
110 return shift->_proxify_try_request( 'simple_request', \@_ );
111 }
112  
113 sub _proxify_try_request {
114 my ( $self, $req_type, $args_ref ) = @_;
115  
116 my $current_proxy = $self->_proxify_set_proxy;
117 my $tries;
118 my $max_tries = $self->proxify_retries;
119 TRY_REQ: {
120 $tries++;
121  
122 my $response = $self->$req_type( @$args_ref );
123 if ( $response->is_success ) {
124 # a lot of proxies seem to be run by this company and it will
125 # give us a 200 but display their page with timeout
126 # all we need to do is redo the request
127 if ( $response->content =~ /\Qcodeen.cs.princeton.edu">CoDeeN/ ) {
128 redo TRY_REQ;
129 }
130 elsif ( not $self->_proxify_check_success($response->content) ) {
131 push @{ $self->proxify_real_bad_list }, $current_proxy;
132 }
133  
134 push @{ $self->proxify_working_list }, $self->proxify_current;
135 return $response;
136  
137 redo TRY_REQ
138 unless $tries > $max_tries;
139  
140 return $response;
141 }
142 else {
143 $self->proxify_debug
144 and carp 'Failed on proxify_get(): '
145 . $response->status_line;
146  
147 if ( $response->status_line =~ /500.+\Q$current_proxy/
148 or $response->code == 400
149 or $response->code == 504
150 or $response->code == 502
151 ) {
152 # BAD PROXY!!! NO COOKIE!
153 push @{ $self->proxify_real_bad_list }, $current_proxy;
154 }
155 else {
156 push @{ $self->proxify_bad_list }, $current_proxy;
157 }
158 $current_proxy = $self->_proxify_set_proxy;
159  
160 redo TRY_REQ
161 unless $tries > $max_tries;
162  
163 # if we got here $response is not successfull but that might have
164 # nothing to do with proxies at all
165 return $response;
166 }
167 } # TRY_GET:{}
168 croak 'I should never get to this point. Please email this message '
169 . 'to zoffix@cpan.org. Thank you very much';
170 }
171  
172 sub _proxify_set_proxy {
173 my $self = shift;
174  
175 my $proxy = $self->proxify_current( shift @{ $self->proxify_list } );
176  
177 unless ( defined $proxy ) {
178 $self->proxify_debug
179 and carp 'proxify_list() is exhausted, trying "working" list';
180
181 $self->proxify_list( $self->proxify_working_list );
182 $self->proxify_working_list([]);
183 $proxy = $self->proxify_current( shift @{ $self->proxify_list } );
184 }
185  
186 unless ( defined $proxy ) {
187 $self->proxify_debug
188 and carp 'proxify_working_list() is exhausted, trying "bad" list';
189  
190 $self->proxify_list( $self->proxify_bad_list );
191 $self->proxify_bad_list([]);
192 $proxy = $self->proxify_current( shift @{ $self->proxify_list } );
193 }
194  
195 unless ( defined $proxy ) {
196 $self->proxify_debug
197 and carp 'lists are exhausted, trying to proxify_load now';
198  
199 $self->proxify_load( %{ $self->_proxify_last_load_args || {} });
200 $proxy = $self->proxify_current( shift @{ $self->proxify_list } );
201  
202 defined $proxy
203 or croak 'After trying so hard I still could not get any more'
204 . ' proxies to play with :(';
205 }
206  
207 $self->proxify_debug
208 and carp "Using proxy $proxy";
209
210 $self->proxy($self->proxify_schemes, $proxy );
211  
212 return $proxy;
213 }
214  
215 sub _proxify_check_success {
216 my ( $self, $content ) = @_;
217 return 1 if length $content > 4000;
218 if ( $content =~ m|\s*
219 \Qhttp/1.1 401 Unauthorized\E\s*
220 \QServer:\E\s*
221 .+?
222 \QWWW-Authenticate: Basic realm="ADSL Router \(ANNEX A\)"\E\s*
223 \QContent-Type: text/html\E\s*
224 \QConnection: close\E\s*
225 \s*
226 \Q<html>\E\s*
227 \Q<head>\E\s*
228 \Q<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-9">\E\s*
229 \Q<META http-equiv="Pragma" CONTENT="no-cache">\E\s*
230 \Q<META HTTP-EQUIV="Cache-Control" CONTENT="no-cache">\E\s*
231 \Q<meta HTTP-EQUIV="Expires" CONTENT="Mon, 06 Jan 1990 00:00:01 GMT">\E\s*
232 |xsm
233 ) {
234 return 0; # failed
235 }
236  
237 if ( $content =~ m|<title>ESPOCH Acceso denegado</title>| ) {
238 return 0; # failed
239 }
240 return 1; # success
241 }
242  
243 1;
244 __END__
↑ Summary