line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package WWW::GoKGS::Scraper::GameArchives; |
2
|
9
|
|
|
9
|
|
858
|
use strict; |
|
9
|
|
|
|
|
21
|
|
|
9
|
|
|
|
|
544
|
|
3
|
9
|
|
|
9
|
|
57
|
use warnings; |
|
9
|
|
|
|
|
20
|
|
|
9
|
|
|
|
|
361
|
|
4
|
9
|
|
|
9
|
|
11930
|
use parent qw/WWW::GoKGS::Scraper/; |
|
9
|
|
|
|
|
2897
|
|
|
9
|
|
|
|
|
52
|
|
5
|
9
|
|
|
9
|
|
7269
|
use WWW::GoKGS::Scraper::Declare; |
|
9
|
|
|
|
|
34
|
|
|
9
|
|
|
|
|
104
|
|
6
|
9
|
|
|
9
|
|
9134
|
use WWW::GoKGS::Scraper::Filters qw/game_result datetime/; |
|
9
|
|
|
|
|
29
|
|
|
9
|
|
|
|
|
12187
|
|
7
|
|
|
|
|
|
|
|
8
|
10
|
|
|
10
|
1
|
117
|
sub base_uri { 'http://www.gokgs.com/gameArchives.jsp' } |
9
|
|
|
|
|
|
|
|
10
|
|
|
|
|
|
|
sub __build_scraper { |
11
|
1
|
|
|
1
|
|
2
|
my $self = shift; |
12
|
|
|
|
|
|
|
|
13
|
1
|
|
|
|
|
2
|
my $month2num = do { |
14
|
1
|
|
|
|
|
71
|
my @months = qw( Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec ); |
15
|
1
|
|
|
|
|
2
|
my %month2num; @month2num{ @months } = ( 1..12 ); |
|
1
|
|
|
|
|
9
|
|
16
|
1
|
|
|
0
|
|
13
|
sub { $month2num{$_[0]} }; |
|
0
|
|
|
|
|
0
|
|
17
|
|
|
|
|
|
|
}; |
18
|
|
|
|
|
|
|
|
19
|
|
|
|
|
|
|
my %user = ( |
20
|
0
|
|
|
0
|
|
0
|
name => [ 'TEXT', sub { s/ \[[^\]]+\]$// } ], |
21
|
1
|
0
|
|
0
|
|
8
|
rank => [ 'TEXT', sub { m/ \[([^\]]+)\]$/ ? $1 : undef } ], |
|
0
|
|
|
|
|
0
|
|
22
|
|
|
|
|
|
|
uri => '@href', |
23
|
|
|
|
|
|
|
); |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
my $game = scraper { |
26
|
0
|
|
|
0
|
|
0
|
process '//td[1]/a', 'sgf_uri' => '@href'; |
27
|
0
|
|
|
|
|
0
|
process '//td[2]/a', 'white[]' => \%user; |
28
|
0
|
|
|
|
|
0
|
process '//td[3]/a', 'black[]' => \%user; |
29
|
0
|
|
|
|
|
0
|
process '//td[3]', 'maybe_setup' => 'TEXT'; |
30
|
0
|
|
|
|
|
0
|
process '//td[4]', 'setup' => 'TEXT'; |
31
|
0
|
|
|
|
|
0
|
process '//td[5]', 'start_time' => 'TEXT'; |
32
|
0
|
|
|
|
|
0
|
process '//td[6]', 'type' => 'TEXT'; |
33
|
0
|
|
|
|
|
0
|
process '//td[7]', 'result' => 'TEXT'; |
34
|
0
|
|
|
|
|
0
|
process '//td[8]', 'tag' => 'TEXT'; |
35
|
1
|
|
|
|
|
28
|
}; |
36
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
my $calendar = scraper { |
38
|
0
|
|
|
0
|
|
0
|
process 'td', 'year' => 'TEXT'; |
39
|
|
|
|
|
|
|
process '//following-sibling::td[not(@colspan)]', |
40
|
|
|
|
|
|
|
'month[]' => scraper { |
41
|
0
|
|
|
|
|
0
|
process '.', 'month' => [ 'TEXT', $month2num ]; |
42
|
0
|
|
|
|
|
0
|
process 'a', 'uri' => '@href'; }; |
|
0
|
|
|
|
|
0
|
|
43
|
1
|
|
|
|
|
12
|
}; |
44
|
|
|
|
|
|
|
|
45
|
|
|
|
|
|
|
scraper { |
46
|
0
|
|
|
0
|
|
|
process '//a[@href="tzList.jsp"]', 'time_zone' => 'TEXT'; |
47
|
0
|
|
|
|
|
|
process '//table[tr/th/text()="Viewable?"]//following-sibling::tr', |
48
|
|
|
|
|
|
|
'games[]' => $game; |
49
|
0
|
|
|
|
|
|
process '//a[contains(@href, ".zip")]', 'zip_uri' => '@href'; |
50
|
0
|
|
|
|
|
|
process '//a[contains(@href, ".tar.gz")]', 'tgz_uri' => '@href'; |
51
|
0
|
|
|
|
|
|
process '//table[colgroup]//following-sibling::tr', |
52
|
|
|
|
|
|
|
'calendar[]' => $calendar; |
53
|
1
|
|
|
|
|
16
|
}; |
54
|
|
|
|
|
|
|
} |
55
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
sub scrape { |
57
|
0
|
|
|
0
|
1
|
|
my ( $self, @args ) = @_; |
58
|
0
|
|
|
|
|
|
my $result = $self->SUPER::scrape( @args ); |
59
|
|
|
|
|
|
|
|
60
|
0
|
0
|
|
|
|
|
return $result unless $result->{calendar}; |
61
|
|
|
|
|
|
|
|
62
|
0
|
|
|
|
|
|
my @calendar; |
63
|
0
|
|
|
|
|
|
for my $calendar ( @{$result->{calendar}} ) { |
|
0
|
|
|
|
|
|
|
64
|
0
|
|
|
|
|
|
for my $month ( @{$calendar->{month}} ) { |
|
0
|
|
|
|
|
|
|
65
|
0
|
|
|
|
|
|
$month->{year} = $calendar->{year}; |
66
|
0
|
|
|
|
|
|
push @calendar, $month; |
67
|
|
|
|
|
|
|
} |
68
|
|
|
|
|
|
|
} |
69
|
|
|
|
|
|
|
|
70
|
0
|
0
|
0
|
|
|
|
if ( @calendar == 1 and $calendar[0]{year} == 1970 ) { # KGS's bug |
71
|
0
|
|
|
|
|
|
delete $result->{calendar}; |
72
|
|
|
|
|
|
|
} |
73
|
|
|
|
|
|
|
else { |
74
|
0
|
|
|
|
|
|
$result->{calendar} = \@calendar; |
75
|
|
|
|
|
|
|
} |
76
|
|
|
|
|
|
|
|
77
|
0
|
0
|
|
|
|
|
return $result unless $result->{games}; |
78
|
|
|
|
|
|
|
|
79
|
0
|
|
|
|
|
|
for my $game ( @{$result->{games}} ) { |
|
0
|
|
|
|
|
|
|
80
|
0
|
0
|
|
|
|
|
next if exists $game->{black}; |
81
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
# | |
83
|
0
|
|
|
|
|
|
my $users = $game->{white}; |
84
|
0
|
0
|
|
|
|
|
if ( @$users == 1 ) { # Type: Demonstration |
|
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
85
|
0
|
|
|
|
|
|
$game->{owner} = $users->[0]; |
86
|
|
|
|
|
|
|
} |
87
|
|
|
|
|
|
|
elsif ( @$users == 3 ) { # Type: Review |
88
|
0
|
|
|
|
|
|
$game->{owner} = $users->[0]; |
89
|
0
|
|
|
|
|
|
$game->{white} = [ $users->[1] ]; |
90
|
0
|
|
|
|
|
|
$game->{black} = [ $users->[2] ]; |
91
|
|
|
|
|
|
|
} |
92
|
|
|
|
|
|
|
elsif ( @$users == 5 ) { # Type: Rengo Review |
93
|
0
|
|
|
|
|
|
$game->{owner} = $users->[0]; |
94
|
0
|
|
|
|
|
|
$game->{white} = [ @{$users}[1,2] ]; |
|
0
|
|
|
|
|
|
|
95
|
0
|
|
|
|
|
|
$game->{black} = [ @{$users}[3,4] ]; |
|
0
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
} |
97
|
|
|
|
|
|
|
else { |
98
|
0
|
|
|
|
|
|
die 'Oops! Something went wrong'; |
99
|
|
|
|
|
|
|
} |
100
|
|
|
|
|
|
|
|
101
|
0
|
0
|
|
|
|
|
$game->{tag} = $game->{result} if exists $game->{result}; |
102
|
0
|
|
|
|
|
|
$game->{result} = $game->{type}; |
103
|
0
|
|
|
|
|
|
$game->{type} = $game->{start_time}; |
104
|
0
|
|
|
|
|
|
$game->{start_time} = $game->{setup}; |
105
|
0
|
|
|
|
|
|
$game->{setup} = $game->{maybe_setup}; |
106
|
|
|
|
|
|
|
} |
107
|
|
|
|
|
|
|
continue { |
108
|
0
|
|
|
|
|
|
$game->{start_time} = datetime( $game->{start_time} ); |
109
|
0
|
|
|
|
|
|
$game->{result} = game_result( $game->{result} ); |
110
|
0
|
|
|
|
|
|
$game->{setup} =~ /^(\d+)\x{d7}\d+ (?:H(\d+))?$/; |
111
|
0
|
|
|
|
|
|
$game->{board_size} = $1; |
112
|
0
|
0
|
|
|
|
|
$game->{handicap} = $2 if defined $2; |
113
|
|
|
|
|
|
|
|
114
|
0
|
0
|
0
|
|
|
|
for my $user ( |
|
0
|
|
|
|
|
|
|
115
|
|
|
|
|
|
|
$game->{owner} || (), |
116
|
0
|
0
|
|
|
|
|
@{ $game->{black} || [] }, |
117
|
|
|
|
|
|
|
@{ $game->{white} || [] }, |
118
|
|
|
|
|
|
|
) { |
119
|
0
|
0
|
|
|
|
|
delete $user->{rank} unless $user->{rank}; |
120
|
|
|
|
|
|
|
} |
121
|
|
|
|
|
|
|
|
122
|
0
|
|
|
|
|
|
delete $game->{setup}; |
123
|
0
|
|
|
|
|
|
delete $game->{maybe_setup}; |
124
|
|
|
|
|
|
|
} |
125
|
|
|
|
|
|
|
|
126
|
0
|
|
|
|
|
|
$result; |
127
|
|
|
|
|
|
|
} |
128
|
|
|
|
|
|
|
|
129
|
|
|
|
|
|
|
1; |
130
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
__END__ |