DeepCrawler/Worker.pm untested

filter: Statementstmt 0.0% 0 / 49 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 10 Podpod 0.0% 0 / 3 totaltotal 0.0% 0 / 62 CC 1 SCAR 6.9
CC
1
Cov
0%
new
2.0
_gen_config
2.0
work
2.0
CRAP 2.0
1 # $Id: /mirror/perl/GunghoX-DeepCrawler/trunk/lib/GunghoX/DeepCrawler/Worker.pm 39021 2008-01-16T16:36:40.809573Z daisuke $
2  
3 package GunghoX::DeepCrawler::Worker;
4 use strict;
5 use warnings;
6 use base qw(Class::Accessor::Fast);
7 use POE;
8 use Gungho;
9 use Gungho::Request;
10 use Swarmage::Task;
11  
12 __PACKAGE__->mk_accessors($_) for qw(worker url);
13  
14 sub new
15 {
16 my $class = shift;
17 my $self = $class->SUPER::new({ @_ });
18  
19 Gungho->run( $self->_gen_config() );
20 $self;
21 }
22  
23 sub _gen_config
24 {
25 my $self = shift;
26  
27 # Setup the sessions here.
28 {
29 engine => {
30 module => 'POE',
31 config => {
32 kernel_start => 0,
33 loop_delay => 120
34 }
35 },
36 follow_links => {
37 parsers => [
38 {
39 module => 'HTML',
40 config => {
41 merge_rule => 'ALL',
42 filters => [
43 {
44 module => 'Strip',
45 }
46 ],
47 rules => [
48 {
49 module => 'HTML::SelectedTags',
50 config => {
51 tags => [ 'a' ]
52 }
53 },
54 {
55 module => 'Fresh',
56 config => {
57 storage => {
58 module => "Cache",
59 config => {
60 cache => {
61 module => "Cache::Memcached::LibMemcached",
62 config => {
63 hashref => 1,
64 servers => [ qw(127.0.0.1:11211) ]
65 }
66 }
67 }
68 }
69 }
70 },
71 {
72 module => 'URI',
73 config => {
74 match => [
75 {
76 action => 'FOLLOW_ALLOW',
77 action_nomatch => 'FOLLOW_DENY',
78 url => $self->url,
79 }
80 ]
81 }
82 }
83 ]
84 }
85 }
86 ]
87 },
88 provider => {
89 module => '+GunghoX::DeepCrawler::Provider',
90 config => {
91 worker => $self->worker,
92 queue => $self->worker->queue
93 }
94 },
95 components => [ qw(
96 +GunghoX::FollowLinks
97 Throttle::Domain
98 ) ],
99 throttle => {
100 domain => {
101 max_items => 10,
102 interval => 5
103 }
104 },
105 handler => sub {
106 my ($self, $c, $req, $res) = @_;
107  
108 warn $req->uri;
109 my $spec = $req->notes('spec');
110 my $task = $req->notes('task');
111 $spec->{task} = $task;
112  
113 $c->follow_links($res);
114 $poe_kernel->post($spec->{session}, $spec->{event}, $spec);
115 }
116 };
117 }
118  
119 sub work
120 {
121 my ($self, $spec, $task) = @_;
122  
123 my $req = $task->data;
124 $req->notes(task => $task);
125 $req->notes(spec => $spec);
126 Gungho->send_request( $req );
127 }
128  
129 1;