| 1 | package LWP::UserAgent::ProxyHopper::Base; | ||
| 2 | |||
| 3 | 2 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 2 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | |||
| 6 | our $VERSION = '0.004'; | ||
| 7 | |||
| 8 | 2 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 2 | ▶ | use Devel::TakeHashArgs; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 2 | ▶ | use List::MoreUtils 'uniq'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 11 | 2 | ▶ | use WWW::FreeProxyListsCom; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | 0 | ▶ | use WWW::Proxy4FreeCom; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 13 | use base 'Class::Data::Accessor'; | ||
| 14 | __PACKAGE__->mk_classaccessors(qw( | ||
| 15 | proxify_list | ||
| 16 | proxify_bad_list | ||
| 17 | proxify_real_bad_list | ||
| 18 | proxify_working_list | ||
| 19 | proxify_schemes | ||
| 20 | proxify_retries | ||
| 21 | proxify_debug | ||
| 22 | proxify_current | ||
| 23 | _proxify_last_load_args | ||
| 24 | _proxify_freeproxylists_obj | ||
| 25 | _proxify_proxy4free_obj | ||
| 26 | )); | ||
| 27 | |||
| 28 | sub proxify_load { | ||
| 29 | my $self = shift; | ||
| 30 | get_args_as_hash(\@_, \my %args, { | ||
| 31 | freeproxylists => 1, | ||
| 32 | plan_b => 1, | ||
| 33 | proxy4free => 0, | ||
| 34 | timeout => 20, | ||
| 35 | debug => 0, | ||
| 36 | retries => 5, | ||
| 37 | extra_proxies => [], | ||
| 38 | schemes => 'http', | ||
| 39 | get_list_args => { | ||
| 40 | freeproxylists => [ ], | ||
| 41 | proxy4free => [ ], | ||
| 42 | }, | ||
| 43 | }, | ||
| 44 | ) or croak $@; | ||
| 45 | |||
| 46 | $self->_proxify_last_load_args( \%args ); | ||
| 47 | |||
| 48 | my @proxies; | ||
| 49 | |||
| 50 | if ( $args{freeproxylists} ) { | ||
| 51 | my $obj = $self->_proxify_freeproxylists_obj( | ||
| 52 | WWW::FreeProxyListsCom->new( timeout => $args{timeout} ) | ||
| 53 | ); | ||
| 54 | |||
| 55 | my $list_ref | ||
| 56 | = $obj->get_list( @{$args{get_list_args}{freeproxylists}} ); | ||
| 57 | if ( defined $list_ref ) { | ||
| 58 | push @proxies, map { "http://$_->{ip}:$_->{port}/" } @$list_ref; | ||
| 59 | } | ||
| 60 | else { | ||
| 61 | $args{debug} | ||
| 62 | and carp 'Failed while trying to get a proxy list from ' | ||
| 63 | . 'http://freeproxylists.com: ' . $obj->error; | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | if ( $args{proxy4free} or ( !@proxies and $args{plan_b} ) ) { | ||
| 68 | my $obj = $self->_proxify_proxy4free_obj( | ||
| 69 | WWW::Proxy4FreeCom->new( timeout => $args{timeout} ) | ||
| 70 | ); | ||
| 71 | |||
| 72 | my $list_ref = $obj->get_list( @{$args{get_list_args}{proxy4free}} ); | ||
| 73 | |||
| 74 | if ( defined $list_ref ) { | ||
| 75 | push @proxies, map { "http://$_->{ip}:$_->{port}/" } @$list_ref; | ||
| 76 | } | ||
| 77 | else { | ||
| 78 | $args{debug} | ||
| 79 | and carp 'Failed while trying to get a proxy list from ' | ||
| 80 | . 'http://proxy4free.com: ' . $obj->error; | ||
| 81 | } | ||
| 82 | } | ||
| 83 | |||
| 84 | unshift @proxies, @{ $args{extra_proxies} }; | ||
| 85 | |||
| 86 | croak q|Don't have ANY proxy addresses :(| | ||
| 87 | unless @proxies; | ||
| 88 | |||
| 89 | @proxies = uniq @proxies; | ||
| 90 | |||
| 91 | $args{debug} | ||
| 92 | and carp "Got " . @proxies . " proxies in total"; | ||
| 93 | |||
| 94 | $self->proxify_retries( $args{retries} ); | ||
| 95 | $self->proxify_schemes( $args{schemes} ); | ||
| 96 | $self->proxify_debug( $args{debug } ); | ||
| 97 | $self->proxify_working_list( [] ); | ||
| 98 | $self->proxify_bad_list( [] ); | ||
| 99 | $self->proxify_real_bad_list( [] ); | ||
| 100 | |||
| 101 | return $self->proxify_list( \@proxies ); | ||
| 102 | } | ||
| 103 | |||
| 104 | sub proxify_get { return shift->_proxify_try_request( 'get', \@_ ); } | ||
| 105 | sub proxify_post { return shift->_proxify_try_request( 'post', \@_ ); } | ||
| 106 | sub proxify_request { return shift->_proxify_try_request( 'request', \@_ ); } | ||
| 107 | sub proxify_head { return shift->_proxify_try_request( 'head', \@_ ); } | ||
| 108 | sub proxify_mirror { return shift->_proxify_try_request( 'mirror', \@_ ); } | ||
| 109 | sub proxify_simple_request { | ||
| 110 | return shift->_proxify_try_request( 'simple_request', \@_ ); | ||
| 111 | } | ||
| 112 | |||
| 113 | sub _proxify_try_request { | ||
| 114 | my ( $self, $req_type, $args_ref ) = @_; | ||
| 115 | |||
| 116 | my $current_proxy = $self->_proxify_set_proxy; | ||
| 117 | my $tries; | ||
| 118 | my $max_tries = $self->proxify_retries; | ||
| 119 | TRY_REQ: { | ||
| 120 | $tries++; | ||
| 121 | |||
| 122 | my $response = $self->$req_type( @$args_ref ); | ||
| 123 | if ( $response->is_success ) { | ||
| 124 | # a lot of proxies seem to be run by this company and it will | ||
| 125 | # give us a 200 but display their page with timeout | ||
| 126 | # all we need to do is redo the request | ||
| 127 | if ( $response->content =~ /\Qcodeen.cs.princeton.edu">CoDeeN/ ) { | ||
| 128 | redo TRY_REQ; | ||
| 129 | } | ||
| 130 | elsif ( not $self->_proxify_check_success($response->content) ) { | ||
| 131 | push @{ $self->proxify_real_bad_list }, $current_proxy; | ||
| 132 | } | ||
| 133 | |||
| 134 | push @{ $self->proxify_working_list }, $self->proxify_current; | ||
| 135 | return $response; | ||
| 136 | |||
| 137 | redo TRY_REQ | ||
| 138 | unless $tries > $max_tries; | ||
| 139 | |||
| 140 | return $response; | ||
| 141 | } | ||
| 142 | else { | ||
| 143 | $self->proxify_debug | ||
| 144 | and carp 'Failed on proxify_get(): ' | ||
| 145 | . $response->status_line; | ||
| 146 | |||
| 147 | if ( $response->status_line =~ /500.+\Q$current_proxy/ | ||
| 148 | or $response->code == 400 | ||
| 149 | or $response->code == 504 | ||
| 150 | or $response->code == 502 | ||
| 151 | ) { | ||
| 152 | # BAD PROXY!!! NO COOKIE! | ||
| 153 | push @{ $self->proxify_real_bad_list }, $current_proxy; | ||
| 154 | } | ||
| 155 | else { | ||
| 156 | push @{ $self->proxify_bad_list }, $current_proxy; | ||
| 157 | } | ||
| 158 | $current_proxy = $self->_proxify_set_proxy; | ||
| 159 | |||
| 160 | redo TRY_REQ | ||
| 161 | unless $tries > $max_tries; | ||
| 162 | |||
| 163 | # if we got here $response is not successfull but that might have | ||
| 164 | # nothing to do with proxies at all | ||
| 165 | return $response; | ||
| 166 | } | ||
| 167 | } # TRY_GET:{} | ||
| 168 | croak 'I should never get to this point. Please email this message ' | ||
| 169 | . 'to zoffix@cpan.org. Thank you very much'; | ||
| 170 | } | ||
| 171 | |||
| 172 | sub _proxify_set_proxy { | ||
| 173 | my $self = shift; | ||
| 174 | |||
| 175 | my $proxy = $self->proxify_current( shift @{ $self->proxify_list } ); | ||
| 176 | |||
| 177 | unless ( defined $proxy ) { | ||
| 178 | $self->proxify_debug | ||
| 179 | and carp 'proxify_list() is exhausted, trying "working" list'; | ||
| 180 | |||
| 181 | $self->proxify_list( $self->proxify_working_list ); | ||
| 182 | $self->proxify_working_list([]); | ||
| 183 | $proxy = $self->proxify_current( shift @{ $self->proxify_list } ); | ||
| 184 | } | ||
| 185 | |||
| 186 | unless ( defined $proxy ) { | ||
| 187 | $self->proxify_debug | ||
| 188 | and carp 'proxify_working_list() is exhausted, trying "bad" list'; | ||
| 189 | |||
| 190 | $self->proxify_list( $self->proxify_bad_list ); | ||
| 191 | $self->proxify_bad_list([]); | ||
| 192 | $proxy = $self->proxify_current( shift @{ $self->proxify_list } ); | ||
| 193 | } | ||
| 194 | |||
| 195 | unless ( defined $proxy ) { | ||
| 196 | $self->proxify_debug | ||
| 197 | and carp 'lists are exhausted, trying to proxify_load now'; | ||
| 198 | |||
| 199 | $self->proxify_load( %{ $self->_proxify_last_load_args || {} }); | ||
| 200 | $proxy = $self->proxify_current( shift @{ $self->proxify_list } ); | ||
| 201 | |||
| 202 | defined $proxy | ||
| 203 | or croak 'After trying so hard I still could not get any more' | ||
| 204 | . ' proxies to play with :('; | ||
| 205 | } | ||
| 206 | |||
| 207 | $self->proxify_debug | ||
| 208 | and carp "Using proxy $proxy"; | ||
| 209 | |||
| 210 | $self->proxy($self->proxify_schemes, $proxy ); | ||
| 211 | |||
| 212 | return $proxy; | ||
| 213 | } | ||
| 214 | |||
| 215 | sub _proxify_check_success { | ||
| 216 | my ( $self, $content ) = @_; | ||
| 217 | return 1 if length $content > 4000; | ||
| 218 | if ( $content =~ m|\s* | ||
| 219 | \Qhttp/1.1 401 Unauthorized\E\s* | ||
| 220 | \QServer:\E\s* | ||
| 221 | .+? | ||
| 222 | \QWWW-Authenticate: Basic realm="ADSL Router \(ANNEX A\)"\E\s* | ||
| 223 | \QContent-Type: text/html\E\s* | ||
| 224 | \QConnection: close\E\s* | ||
| 225 | \s* | ||
| 226 | \Q<html>\E\s* | ||
| 227 | \Q<head>\E\s* | ||
| 228 | \Q<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-9">\E\s* | ||
| 229 | \Q<META http-equiv="Pragma" CONTENT="no-cache">\E\s* | ||
| 230 | \Q<META HTTP-EQUIV="Cache-Control" CONTENT="no-cache">\E\s* | ||
| 231 | \Q<meta HTTP-EQUIV="Expires" CONTENT="Mon, 06 Jan 1990 00:00:01 GMT">\E\s* | ||
| 232 | |xsm | ||
| 233 | ) { | ||
| 234 | return 0; # failed | ||
| 235 | } | ||
| 236 | |||
| 237 | if ( $content =~ m|<title>ESPOCH Acceso denegado</title>| ) { | ||
| 238 | return 0; # failed | ||
| 239 | } | ||
| 240 | return 1; # success | ||
| 241 | } | ||
| 242 | |||
| 243 | 1; | ||
| 244 | __END__ | ||