| 1 | # $Id: /mirror/gungho/lib/Gungho/Engine/POE.pm 39017 2008-01-16T16:05:45.674472Z lestrrat $ | ||
| 2 | # | ||
| 3 | # Copyright (c) 2007 Daisuke Maki <daisuke@endeworks.jp> | ||
| 4 | # All rights reserved. | ||
| 5 | |||
| 6 | package Gungho::Engine::POE; | ||
| 7 | use strict; | ||
| 8 | use warnings; | ||
| 9 | use base qw(Gungho::Engine); | ||
| 10 | use POE; | ||
| 11 | use POE::Component::Client::Keepalive; | ||
| 12 | use POE::Component::Client::HTTP; | ||
| 13 | |||
| 14 | __PACKAGE__->mk_accessors($_) for qw(alias loop_alarm loop_delay resolver clients); | ||
| 15 | |||
| 16 | use constant DEBUG => 0; | ||
| 17 | use constant UserAgentAlias => 'Gungho_Engine_POE_UserAgent_Alias'; | ||
| 18 | use constant DnsResolverAlias => 'Gungho_Engine_POE_DnsResolver_Alias'; | ||
| 19 | use constant SKIP_DECODE_CONTENT => | ||
| 20 | exists $ENV{GUNGHO_ENGINE_POE_SKIP_DECODE_CONTENT} ? $ENV{GUNGHO_ENGINE_POE_SKIP_DECODE_CONTENT} : 1; | ||
| 21 | use constant FORCE_ENCODE_CONTENT => | ||
| 22 | $ENV{GUNGHO_ENGINE_POE_FORCE_ENCODE_CONTENT} && ! SKIP_DECODE_CONTENT; | ||
| 23 | |||
| 24 | BEGIN | ||
| 25 | { | ||
| 26 | if (SKIP_DECODE_CONTENT) { | ||
| 27 | # PoCo::Client::HTTP workaround for blindly decoding content for us | ||
| 28 | # when encountering Contentn-Encoding | ||
| 29 | eval sprintf(<<' EOCODE', 'HTTP::Response'); | ||
| 30 | no warnings 'redefine'; | ||
| 31 | package %s; | ||
| 32 | sub HTTP::Response::decoded_content { | ||
| 33 | my ($self, %%opt) = @_; | ||
| 34 | my $caller = (caller(2))[3]; | ||
| 35 | |||
| 36 | if ($caller eq 'POE::Component::Client::HTTP::Request::return_response') { | ||
| 37 | $opt{charset} = 'none'; | ||
| 38 | } | ||
| 39 | $self->SUPER::decoded_content(%%opt); | ||
| 40 | } | ||
| 41 | EOCODE | ||
| 42 | } | ||
| 43 | } | ||
| 44 | |||
| 45 | sub setup | ||
| 46 | { | ||
| 47 | my $self = shift; | ||
| 48 | $self->alias('MainComp'); | ||
| 49 | $self->loop_delay( $self->config->{loop_delay} ) if $self->config->{loop_delay}; | ||
| 50 | $self->next::method(@_); | ||
| 51 | } | ||
| 52 | |||
| 53 | sub run | ||
| 54 | { | ||
| 55 | my ($self, $c) = @_; | ||
| 56 | |||
| 57 | my %config = %{ $self->config || {} }; | ||
| 58 | |||
| 59 | my $keepalive_config = delete $config{keepalive} || {}; | ||
| 60 | |||
| 61 | { | ||
| 62 | my %defaults = ( | ||
| 63 | keep_alive => 10, | ||
| 64 | max_open => 200, | ||
| 65 | max_per_host => 5, | ||
| 66 | timeout => 10 | ||
| 67 | ); | ||
| 68 | while (my($key, $value) = each %defaults) { | ||
| 69 | if (! defined $keepalive_config->{$key}) { | ||
| 70 | $keepalive_config->{$key} = $value; | ||
| 71 | } | ||
| 72 | } | ||
| 73 | } | ||
| 74 | |||
| 75 | my $keepalive = POE::Component::Client::Keepalive->new(%$keepalive_config); | ||
| 76 | |||
| 77 | my $client_config = delete $config{client} || {}; | ||
| 78 | foreach my $key (keys %$client_config) { | ||
| 79 | if ($key =~ /^[a-z]/) { # ah, need to make this CamelCase | ||
| 80 | my $camel = ucfirst($key); | ||
| 81 | $camel =~ s/_(\w)/uc($1)/ge; | ||
| 82 | $client_config->{$camel} = delete $client_config->{$key}; | ||
| 83 | } | ||
| 84 | } | ||
| 85 | |||
| 86 | # Starting from 0.09002, we accept that there are environments where | ||
| 87 | # DNS resolution is NOT necessary. This turns out to be a problem when | ||
| 88 | # going through, for example, a misconfigured proxy. | ||
| 89 | # | ||
| 90 | # Here, we detect if one of the following is true: | ||
| 91 | # 1) The user has explicitly disable DNS resolution via dns.disable = 1 | ||
| 92 | # 2) The user has requested the use of a proxy via engine.client.proxy | ||
| 93 | # 3) The user has implicitly requested the use of a proxy via | ||
| 94 | # $ENV{HTTP_PROXY} | ||
| 95 | my $dns_config = delete $config{dns} || {}; | ||
| 96 | unless ($dns_config->{disable} || $client_config->{Proxy} || $client_config->{proxy} || $ENV{HTTP_PROXY}) { | ||
| 97 | foreach my $key (keys %$dns_config) { | ||
| 98 | if ($key =~ /^[a-z]/) { # ah, need to make this CamelCase | ||
| 99 | my $camel = ucfirst($key); | ||
| 100 | $camel =~ s/_(\w)/uc($1)/ge; | ||
| 101 | $dns_config->{$camel} = delete $dns_config->{$key}; | ||
| 102 | } | ||
| 103 | } | ||
| 104 | my $resolver = POE::Component::Client::DNS->spawn( | ||
| 105 | %$dns_config, | ||
| 106 | Alias => &DnsResolverAlias, | ||
| 107 | ); | ||
| 108 | $self->resolver($resolver); | ||
| 109 | } | ||
| 110 | |||
| 111 | # Oh, guess what. We will create as many clients as we were requested, | ||
| 112 | # just so that PoCo::Client::HTTP doesn't stall on us (as of | ||
| 113 | # PoCo::Client::HTTP 0.82, PoCo::Client::HTTP tended to get filled up | ||
| 114 | # pretty quickcly) | ||
| 115 | $self->clients( [] ); | ||
| 116 | my $spawn = delete $client_config->{Spawn} || 2; | ||
| 117 | if ($spawn < 1) { $spawn = 2 } | ||
| 118 | for my $i ( 1 .. $spawn ) { | ||
| 119 | my $alias = join('-', &UserAgentAlias, $i); | ||
| 120 | push @{ $self->clients }, $alias; | ||
| 121 | POE::Component::Client::HTTP->spawn( | ||
| 122 | FollowRedirects => 1, | ||
| 123 | Agent => $c->user_agent, | ||
| 124 | Timeout => 60, | ||
| 125 | %$client_config, | ||
| 126 | Alias => $alias, | ||
| 127 | ConnectionManager => $keepalive, | ||
| 128 | ); | ||
| 129 | } | ||
| 130 | |||
| 131 | POE::Session->create( | ||
| 132 | heap => { CONTEXT => $c }, | ||
| 133 | object_states => [ | ||
| 134 | $self => { | ||
| 135 | _start => '_poe_session_start', | ||
| 136 | _stop => '_poe_session_stop', | ||
| 137 | map { ($_ => "_poe_$_") } | ||
| 138 | qw(session_loop start_request handle_response got_dns_response shutdown) | ||
| 139 | } | ||
| 140 | ] | ||
| 141 | ); | ||
| 142 | |||
| 143 | POE::Kernel->run() if | ||
| 144 | ! exists $config{ kernel_start } || $config{ kernel_start }; | ||
| 145 | } | ||
| 146 | |||
| 147 | sub stop | ||
| 148 | { | ||
| 149 | my ($self, $c) = @_; | ||
| 150 | POE::Kernel->post($self->alias, 'shutdown'); | ||
| 151 | } | ||
| 152 | |||
| 153 | sub _poe_shutdown | ||
| 154 | { | ||
| 155 | my ($self, $kernel, $heap) = @_[OBJECT, KERNEL, HEAP]; | ||
| 156 | my $clients = $self->clients; | ||
| 157 | foreach my $client (@$clients) { | ||
| 158 | $kernel->post($client, 'shutdown'); | ||
| 159 | } | ||
| 160 | my $c = $heap->{CONTEXT}; | ||
| 161 | $c->is_running(0); | ||
| 162 | } | ||
| 163 | |||
| 164 | sub _poe_session_start | ||
| 165 | { | ||
| 166 | $_[KERNEL]->alias_set( $_[OBJECT]->alias ); | ||
| 167 | $_[KERNEL]->yield('session_loop'); | ||
| 168 | } | ||
| 169 | |||
| 170 | sub _poe_session_stop | ||
| 171 | { | ||
| 172 | $_[KERNEL]->alias_remove( $_[OBJECT]->alias ); | ||
| 173 | eval { | ||
| 174 | $_[KERNEL]->post($_, 'shutdown') for @{ $_[OBJECT]->clients } | ||
| 175 | }; | ||
| 176 | } | ||
| 177 | |||
| 178 | sub _poe_session_loop | ||
| 179 | { | ||
| 180 | my ($self, $kernel, $heap) = @_[OBJECT, KERNEL, HEAP]; | ||
| 181 | |||
| 182 | my $c = $heap->{CONTEXT}; | ||
| 183 | |||
| 184 | if (! $c->is_running) { | ||
| 185 | $c->log->debug("is_running = 0, waiting for other queued states to finish...\n"); | ||
| 186 | return; | ||
| 187 | } | ||
| 188 | |||
| 189 | $c->dispatch_requests(); | ||
| 190 | |||
| 191 | my $alarm_id = $self->loop_alarm; | ||
| 192 | if ($alarm_id) { | ||
| 193 | $kernel->alarm_remove( $alarm_id ); | ||
| 194 | } | ||
| 195 | my $delay = $self->loop_delay; | ||
| 196 | if (! defined $delay || $delay <= 0) { | ||
| 197 | $delay = 1; | ||
| 198 | $self->loop_delay($delay); | ||
| 199 | } | ||
| 200 | $self->loop_alarm($kernel->alarm_set('session_loop', time() + $delay)); | ||
| 201 | |||
| 202 | $c->notify('engine.end_loop'); | ||
| 203 | } | ||
| 204 | |||
| 205 | sub send_request | ||
| 206 | { | ||
| 207 | my ($self, $c, $request) = @_; | ||
| 208 | POE::Kernel->post($self->alias, 'start_request', $request); | ||
| 209 | return 1; | ||
| 210 | } | ||
| 211 | |||
| 212 | sub _poe_start_request | ||
| 213 | { | ||
| 214 | my ($kernel, $self, $heap, $request) = @_[KERNEL, OBJECT, HEAP, ARG0]; | ||
| 215 | my $c = $heap->{CONTEXT}; | ||
| 216 | |||
| 217 | # check if this request requires a DNS resolution | ||
| 218 | if ($c->engine->resolver && $request->requires_name_lookup()) { | ||
| 219 | my $dns_response = $c->engine->resolver->resolve( | ||
| 220 | event => "got_dns_response", | ||
| 221 | host => $request->uri->host, | ||
| 222 | context => { request => $request } | ||
| 223 | ); | ||
| 224 | # PoCo::Client::DNS may resolve DNS immediately | ||
| 225 | if ($dns_response) { | ||
| 226 | $kernel->yield('got_dns_response', $dns_response); | ||
| 227 | } | ||
| 228 | return; | ||
| 229 | } | ||
| 230 | |||
| 231 | $request->uri->host($request->notes('resolved_ip')) | ||
| 232 | if $request->notes('resolved_ip'); | ||
| 233 | |||
| 234 | if (! $c->request_is_allowed($request)) { | ||
| 235 | # For whatever reason, the request was not allowed | ||
| 236 | return; | ||
| 237 | } | ||
| 238 | |||
| 239 | $c->notify('engine.send_request', { request => $request }); | ||
| 240 | |||
| 241 | if (DEBUG) { | ||
| 242 | my $uri = $request->uri->clone; | ||
| 243 | $uri->host( $request->notes('original_host') ) if $request->notes('original_host'); | ||
| 244 | $c->log->info("Going to fetch $uri"); | ||
| 245 | } | ||
| 246 | |||
| 247 | # Choose a random client | ||
| 248 | my @clients = @{ $self->clients }; | ||
| 249 | my $client = $clients[ rand @clients ]; | ||
| 250 | POE::Kernel->post($client, 'request', 'handle_response', $request); | ||
| 251 | } | ||
| 252 | |||
| 253 | sub _poe_got_dns_response | ||
| 254 | { | ||
| 255 | my ($kernel, $response) = @_[KERNEL, ARG0]; | ||
| 256 | |||
| 257 | $_[OBJECT]->handle_dns_response( | ||
| 258 | $_[HEAP]->{CONTEXT}, | ||
| 259 | $response->{context}->{request}, # original request | ||
| 260 | $response->{response}, # DNS response | ||
| 261 | ); | ||
| 262 | } | ||
| 263 | |||
| 264 | sub _poe_handle_response | ||
| 265 | { | ||
| 266 | my ($kernel, $heap, $req_packet, $res_packet) = @_[ KERNEL, HEAP, ARG0, ARG1 ]; | ||
| 267 | |||
| 268 | my $c = $heap->{CONTEXT}; | ||
| 269 | |||
| 270 | my $req = $req_packet->[0]; | ||
| 271 | my $res = $res_packet->[0]; | ||
| 272 | |||
| 273 | if (my $host = $req->notes('original_host')) { | ||
| 274 | # Put it back | ||
| 275 | $req->uri->host($host); | ||
| 276 | } | ||
| 277 | if (DEBUG) { | ||
| 278 | $c->log->info("Received " . $req->uri); | ||
| 279 | } | ||
| 280 | |||
| 281 | # Work around POE doing too much for us. | ||
| 282 | if (FORCE_ENCODE_CONTENT && $POE::Component::Client::HTTP::VERSION # Hide from CPAN | ||
| 283 | >= 0.80) | ||
| 284 | { | ||
| 285 | if ($res->content_encoding) { | ||
| 286 | my @ct = $res->content_type; | ||
| 287 | if ((shift @ct) =~ /^text\//) { | ||
| 288 | foreach my $ct (@ct) { | ||
| 289 | next unless $ct =~ /charset=((?!utf-?8).+)$/; | ||
| 290 | my $enc = $1; | ||
| 291 | require Encode; | ||
| 292 | $res->content( Encode::encode($enc, $res->content) ); | ||
| 293 | last; | ||
| 294 | } | ||
| 295 | } | ||
| 296 | } | ||
| 297 | } | ||
| 298 | |||
| 299 | $c->notify('engine.handle_response', { request => $req, response => $res }); | ||
| 300 | |||
| 301 | # Do we support auth challenge ? | ||
| 302 | my $code = $c->can('check_authentication_challenge'); | ||
| 303 | if ( $code ) { | ||
| 304 | # return if auth has taken care of the response | ||
| 305 | return if $code->($c, $req, $res); | ||
| 306 | } | ||
| 307 | |||
| 308 | $c->handle_response($req, $c->prepare_response($res) ); | ||
| 309 | |||
| 310 | $kernel->yield('session_loop'); | ||
| 311 | } | ||
| 312 | |||
| 313 | 1; | ||
| 314 | |||
| 315 | __END__ |