| 1 | # $Id: /mirror/perl/GunghoX-DeepCrawler/trunk/lib/GunghoX/DeepCrawler/Worker.pm 39021 2008-01-16T16:36:40.809573Z daisuke $ | ||
| 2 | |||
| 3 | package GunghoX::DeepCrawler::Worker; | ||
| 4 | use strict; | ||
| 5 | use warnings; | ||
| 6 | use base qw(Class::Accessor::Fast); | ||
| 7 | use POE; | ||
| 8 | use Gungho; | ||
| 9 | use Gungho::Request; | ||
| 10 | use Swarmage::Task; | ||
| 11 | |||
| 12 | __PACKAGE__->mk_accessors($_) for qw(worker url); | ||
| 13 | |||
| 14 | sub new | ||
| 15 | { | ||
| 16 | my $class = shift; | ||
| 17 | my $self = $class->SUPER::new({ @_ }); | ||
| 18 | |||
| 19 | Gungho->run( $self->_gen_config() ); | ||
| 20 | $self; | ||
| 21 | } | ||
| 22 | |||
| 23 | sub _gen_config | ||
| 24 | { | ||
| 25 | my $self = shift; | ||
| 26 | |||
| 27 | # Setup the sessions here. | ||
| 28 | { | ||
| 29 | engine => { | ||
| 30 | module => 'POE', | ||
| 31 | config => { | ||
| 32 | kernel_start => 0, | ||
| 33 | loop_delay => 120 | ||
| 34 | } | ||
| 35 | }, | ||
| 36 | follow_links => { | ||
| 37 | parsers => [ | ||
| 38 | { | ||
| 39 | module => 'HTML', | ||
| 40 | config => { | ||
| 41 | merge_rule => 'ALL', | ||
| 42 | filters => [ | ||
| 43 | { | ||
| 44 | module => 'Strip', | ||
| 45 | } | ||
| 46 | ], | ||
| 47 | rules => [ | ||
| 48 | { | ||
| 49 | module => 'HTML::SelectedTags', | ||
| 50 | config => { | ||
| 51 | tags => [ 'a' ] | ||
| 52 | } | ||
| 53 | }, | ||
| 54 | { | ||
| 55 | module => 'Fresh', | ||
| 56 | config => { | ||
| 57 | storage => { | ||
| 58 | module => "Cache", | ||
| 59 | config => { | ||
| 60 | cache => { | ||
| 61 | module => "Cache::Memcached::LibMemcached", | ||
| 62 | config => { | ||
| 63 | hashref => 1, | ||
| 64 | servers => [ qw(127.0.0.1:11211) ] | ||
| 65 | } | ||
| 66 | } | ||
| 67 | } | ||
| 68 | } | ||
| 69 | } | ||
| 70 | }, | ||
| 71 | { | ||
| 72 | module => 'URI', | ||
| 73 | config => { | ||
| 74 | match => [ | ||
| 75 | { | ||
| 76 | action => 'FOLLOW_ALLOW', | ||
| 77 | action_nomatch => 'FOLLOW_DENY', | ||
| 78 | url => $self->url, | ||
| 79 | } | ||
| 80 | ] | ||
| 81 | } | ||
| 82 | } | ||
| 83 | ] | ||
| 84 | } | ||
| 85 | } | ||
| 86 | ] | ||
| 87 | }, | ||
| 88 | provider => { | ||
| 89 | module => '+GunghoX::DeepCrawler::Provider', | ||
| 90 | config => { | ||
| 91 | worker => $self->worker, | ||
| 92 | queue => $self->worker->queue | ||
| 93 | } | ||
| 94 | }, | ||
| 95 | components => [ qw( | ||
| 96 | +GunghoX::FollowLinks | ||
| 97 | Throttle::Domain | ||
| 98 | ) ], | ||
| 99 | throttle => { | ||
| 100 | domain => { | ||
| 101 | max_items => 10, | ||
| 102 | interval => 5 | ||
| 103 | } | ||
| 104 | }, | ||
| 105 | handler => sub { | ||
| 106 | my ($self, $c, $req, $res) = @_; | ||
| 107 | |||
| 108 | warn $req->uri; | ||
| 109 | my $spec = $req->notes('spec'); | ||
| 110 | my $task = $req->notes('task'); | ||
| 111 | $spec->{task} = $task; | ||
| 112 | |||
| 113 | $c->follow_links($res); | ||
| 114 | $poe_kernel->post($spec->{session}, $spec->{event}, $spec); | ||
| 115 | } | ||
| 116 | }; | ||
| 117 | } | ||
| 118 | |||
| 119 | sub work | ||
| 120 | { | ||
| 121 | my ($self, $spec, $task) = @_; | ||
| 122 | |||
| 123 | my $req = $task->data; | ||
| 124 | $req->notes(task => $task); | ||
| 125 | $req->notes(spec => $spec); | ||
| 126 | Gungho->send_request( $req ); | ||
| 127 | } | ||
| 128 | |||
| 129 | 1; |