line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package WWW::GoKGS; |
2
|
9
|
|
|
9
|
|
1143918
|
use 5.008_009; |
|
9
|
|
|
|
|
58
|
|
|
9
|
|
|
|
|
3139
|
|
3
|
9
|
|
|
9
|
|
138
|
use strict; |
|
9
|
|
|
|
|
19
|
|
|
9
|
|
|
|
|
437
|
|
4
|
9
|
|
|
9
|
|
48
|
use warnings; |
|
9
|
|
|
|
|
21
|
|
|
9
|
|
|
|
|
317
|
|
5
|
9
|
|
|
9
|
|
50
|
use Carp qw/croak/; |
|
9
|
|
|
|
|
29
|
|
|
9
|
|
|
|
|
830
|
|
6
|
9
|
|
|
9
|
|
9012
|
use HTML::TreeBuilder::XPath; |
|
9
|
|
|
|
|
864551
|
|
|
9
|
|
|
|
|
140
|
|
7
|
9
|
|
|
9
|
|
10370
|
use LWP::RobotUA; |
|
9
|
|
|
|
|
730656
|
|
|
9
|
|
|
|
|
344
|
|
8
|
9
|
|
|
9
|
|
116
|
use URI; |
|
9
|
|
|
|
|
21
|
|
|
9
|
|
|
|
|
310
|
|
9
|
9
|
|
|
9
|
|
7604
|
use WWW::GoKGS::Scraper::GameArchives; |
|
9
|
|
|
|
|
201
|
|
|
9
|
|
|
|
|
149
|
|
10
|
9
|
|
|
9
|
|
12443
|
use WWW::GoKGS::Scraper::Top100; |
|
9
|
|
|
|
|
41
|
|
|
9
|
|
|
|
|
6085
|
|
11
|
9
|
|
|
9
|
|
5650
|
use WWW::GoKGS::Scraper::TournList; |
|
9
|
|
|
|
|
27
|
|
|
9
|
|
|
|
|
101
|
|
12
|
9
|
|
|
9
|
|
5421
|
use WWW::GoKGS::Scraper::TournInfo; |
|
9
|
|
|
|
|
26
|
|
|
9
|
|
|
|
|
131
|
|
13
|
9
|
|
|
9
|
|
7641
|
use WWW::GoKGS::Scraper::TournEntrants; |
|
9
|
|
|
|
|
25
|
|
|
9
|
|
|
|
|
157
|
|
14
|
9
|
|
|
9
|
|
7284
|
use WWW::GoKGS::Scraper::TournGames; |
|
9
|
|
|
|
|
22
|
|
|
9
|
|
|
|
|
112
|
|
15
|
9
|
|
|
9
|
|
5427
|
use WWW::GoKGS::Scraper::TzList; |
|
9
|
|
|
|
|
312
|
|
|
9
|
|
|
|
|
106
|
|
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
our $VERSION = '0.21'; |
18
|
|
|
|
|
|
|
|
19
|
7
|
|
|
7
|
|
14
|
sub _tree_builder_class { 'HTML::TreeBuilder::XPath' } |
20
|
|
|
|
|
|
|
|
21
|
|
|
|
|
|
|
sub new { |
22
|
1
|
|
|
1
|
1
|
1715
|
my $class = shift; |
23
|
1
|
50
|
|
|
|
8
|
my %args = @_ == 1 ? %{$_[0]} : @_; |
|
0
|
|
|
|
|
0
|
|
24
|
1
|
|
|
|
|
3
|
my $self = bless {}, $class; |
25
|
|
|
|
|
|
|
|
26
|
1
|
|
|
|
|
3
|
for my $key (qw/user_agent/) { |
27
|
1
|
50
|
|
|
|
6
|
$self->{$key} = $args{$key} if exists $args{$key}; |
28
|
|
|
|
|
|
|
} |
29
|
|
|
|
|
|
|
|
30
|
1
|
|
|
|
|
5
|
$self->init( \%args ); |
31
|
|
|
|
|
|
|
|
32
|
1
|
|
|
|
|
5
|
$self; |
33
|
|
|
|
|
|
|
} |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
sub init { |
36
|
1
|
|
|
1
|
0
|
2
|
my ( $self, $args ) = @_; |
37
|
|
|
|
|
|
|
|
38
|
1
|
50
|
|
|
|
10
|
unless ( exists $self->{user_agent} ) { |
39
|
1
|
|
|
|
|
2
|
my $class = ref $self; |
40
|
|
|
|
|
|
|
|
41
|
1
|
|
33
|
|
|
33
|
$self->user_agent( |
42
|
|
|
|
|
|
|
LWP::RobotUA->new( |
43
|
|
|
|
|
|
|
agent => $args->{agent} || "$class/" . $class->VERSION, |
44
|
|
|
|
|
|
|
from => $args->{from}, |
45
|
|
|
|
|
|
|
cookie_jar => $args->{cookie_jar}, |
46
|
|
|
|
|
|
|
) |
47
|
|
|
|
|
|
|
); |
48
|
|
|
|
|
|
|
} |
49
|
|
|
|
|
|
|
|
50
|
1
|
|
|
|
|
7
|
return; |
51
|
|
|
|
|
|
|
} |
52
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
sub user_agent { |
54
|
12
|
|
|
12
|
1
|
14804
|
my $self = shift; |
55
|
12
|
100
|
|
|
|
32
|
$self->{user_agent} = shift if @_; |
56
|
12
|
|
|
|
|
123
|
$self->{user_agent}; |
57
|
|
|
|
|
|
|
} |
58
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
sub agent { |
60
|
1
|
|
|
1
|
1
|
754
|
my ( $self, @args ) = @_; |
61
|
1
|
|
|
|
|
4
|
$self->user_agent->agent( @args ); |
62
|
|
|
|
|
|
|
} |
63
|
|
|
|
|
|
|
|
64
|
|
|
|
|
|
|
sub from { |
65
|
1
|
|
|
1
|
1
|
4
|
my ( $self, @args ) = @_; |
66
|
1
|
|
|
|
|
4
|
$self->user_agent->default_header( 'From', @args ); |
67
|
|
|
|
|
|
|
} |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
sub cookie_jar { |
70
|
1
|
|
|
1
|
1
|
553
|
my ( $self, @args ) = @_; |
71
|
1
|
|
|
|
|
4
|
$self->user_agent->cookie_jar( @args ); |
72
|
|
|
|
|
|
|
} |
73
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
sub get { |
75
|
0
|
|
|
0
|
1
|
0
|
my ( $self, @args ) = @_; |
76
|
0
|
|
|
|
|
0
|
$self->user_agent->get( @args ); |
77
|
|
|
|
|
|
|
} |
78
|
|
|
|
|
|
|
|
79
|
|
|
|
|
|
|
sub _scrapers { |
80
|
20
|
|
|
20
|
|
22
|
my $self = shift; |
81
|
20
|
|
66
|
|
|
160
|
$self->{_scrapers} ||= $self->__build_scrapers; |
82
|
|
|
|
|
|
|
} |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
BEGIN { # install scrapers |
85
|
9
|
|
|
9
|
|
90
|
my %scrapers = ( |
86
|
|
|
|
|
|
|
game_archives => 'WWW::GoKGS::Scraper::GameArchives', |
87
|
|
|
|
|
|
|
top_100 => 'WWW::GoKGS::Scraper::Top100', |
88
|
|
|
|
|
|
|
tourn_list => 'WWW::GoKGS::Scraper::TournList', |
89
|
|
|
|
|
|
|
tourn_info => 'WWW::GoKGS::Scraper::TournInfo', |
90
|
|
|
|
|
|
|
tourn_entrants => 'WWW::GoKGS::Scraper::TournEntrants', |
91
|
|
|
|
|
|
|
tourn_games => 'WWW::GoKGS::Scraper::TournGames', |
92
|
|
|
|
|
|
|
tz_list => 'WWW::GoKGS::Scraper::TzList', |
93
|
|
|
|
|
|
|
); |
94
|
|
|
|
|
|
|
|
95
|
9
|
|
|
|
|
31
|
my %paths; |
96
|
9
|
|
|
|
|
81
|
while ( my ($method, $class) = each %scrapers ) { |
97
|
63
|
|
|
|
|
735
|
my $path = $paths{$class} = $class->build_uri->path; |
98
|
63
|
|
|
11
|
|
3235
|
my $body = sub { $_[0]->get_scraper($path) }; |
|
11
|
|
|
|
|
33
|
|
99
|
9
|
|
|
9
|
|
6815
|
no strict 'refs'; |
|
9
|
|
|
|
|
21
|
|
|
9
|
|
|
|
|
1134
|
|
100
|
63
|
|
|
|
|
6268
|
*$method = $body; |
101
|
|
|
|
|
|
|
} |
102
|
|
|
|
|
|
|
|
103
|
|
|
|
|
|
|
sub __build_scrapers { |
104
|
1
|
|
|
1
|
|
3
|
my $self = shift; |
105
|
1
|
|
|
|
|
2
|
my $class = ref $self; |
106
|
|
|
|
|
|
|
|
107
|
1
|
|
|
|
|
2
|
my %_scrapers; |
108
|
1
|
|
|
|
|
7
|
while ( my ($scraper, $path) = each %paths ) { |
109
|
7
|
|
|
|
|
28
|
$_scrapers{$path} = $scraper->new( |
110
|
|
|
|
|
|
|
_tree_builder_class => $class->_tree_builder_class, |
111
|
|
|
|
|
|
|
user_agent => $self->user_agent, |
112
|
|
|
|
|
|
|
); |
113
|
|
|
|
|
|
|
} |
114
|
|
|
|
|
|
|
|
115
|
1
|
|
|
|
|
8
|
\%_scrapers; |
116
|
|
|
|
|
|
|
} |
117
|
|
|
|
|
|
|
} |
118
|
|
|
|
|
|
|
|
119
|
|
|
|
|
|
|
sub get_scraper { |
120
|
18
|
|
|
18
|
1
|
946
|
my ( $self, $path ) = @_; |
121
|
18
|
|
|
|
|
34
|
$self->_scrapers->{$path}; |
122
|
|
|
|
|
|
|
} |
123
|
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
sub each_scraper { |
125
|
2
|
|
|
2
|
1
|
85
|
my ( $self, $code ) = @_; |
126
|
2
|
|
|
|
|
5
|
my %scrapers = %{ $self->_scrapers }; |
|
2
|
|
|
|
|
6
|
|
127
|
|
|
|
|
|
|
|
128
|
2
|
100
|
|
|
|
31
|
croak 'Not a CODE reference' unless ref $code eq 'CODE'; |
129
|
|
|
|
|
|
|
|
130
|
1
|
|
|
|
|
5
|
while ( my ($path, $scraper) = each %scrapers ) { |
131
|
7
|
|
|
|
|
2784
|
$code->( $path => $scraper ); |
132
|
|
|
|
|
|
|
} |
133
|
|
|
|
|
|
|
|
134
|
1
|
|
|
|
|
521
|
return; |
135
|
|
|
|
|
|
|
} |
136
|
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
sub can_scrape { |
138
|
6
|
|
|
6
|
1
|
13
|
my $self = shift; |
139
|
6
|
|
|
|
|
16
|
my $uri = $self->_build_uri( shift ); |
140
|
6
|
|
66
|
|
|
23
|
my $path = $uri =~ m{^http://www\.gokgs\.com(?::80)?/} && $uri->path; |
141
|
6
|
100
|
|
|
|
138
|
$path ? $self->get_scraper( $path ) : undef; |
142
|
|
|
|
|
|
|
} |
143
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
sub scrape { |
145
|
1
|
|
|
1
|
1
|
42
|
my ( $self, $arg ) = @_; |
146
|
1
|
|
|
|
|
5
|
my $uri = $self->_build_uri( $arg ); |
147
|
1
|
|
|
|
|
5
|
my $scraper = $self->can_scrape( $uri ); |
148
|
1
|
50
|
|
|
|
19
|
croak "Don't know how to scrape '$arg'" unless $scraper; |
149
|
0
|
|
|
|
|
0
|
$scraper->scrape( $self->get($uri), $uri ); |
150
|
|
|
|
|
|
|
} |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
sub _build_uri { |
153
|
7
|
|
|
7
|
|
9
|
my $self = shift; |
154
|
7
|
|
|
|
|
30
|
my $uri = URI->new( shift ); |
155
|
7
|
100
|
|
|
|
596
|
$uri->scheme( 'http' ) unless $uri->scheme; |
156
|
7
|
100
|
|
|
|
714
|
$uri->authority( 'www.gokgs.com' ) unless $uri->authority; |
157
|
7
|
|
|
|
|
261
|
$uri; |
158
|
|
|
|
|
|
|
} |
159
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
1; |
161
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
__END__ |