line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Geo::Coder::Many::Scheduler::Selective; |
2
|
|
|
|
|
|
|
|
3
|
2
|
|
|
2
|
|
10
|
use strict; |
|
2
|
|
|
|
|
2
|
|
|
2
|
|
|
|
|
66
|
|
4
|
2
|
|
|
2
|
|
8
|
use warnings; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
62
|
|
5
|
2
|
|
|
2
|
|
10
|
use Time::HiRes qw( gettimeofday ); |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
18
|
|
6
|
2
|
|
|
2
|
|
382
|
use List::Util qw( min max ); |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
154
|
|
7
|
2
|
|
|
2
|
|
9
|
use Carp; |
|
2
|
|
|
|
|
5
|
|
|
2
|
|
|
|
|
164
|
|
8
|
|
|
|
|
|
|
|
9
|
2
|
|
|
2
|
|
12
|
use base 'Geo::Coder::Many::Scheduler'; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
1415
|
|
10
|
|
|
|
|
|
|
|
11
|
|
|
|
|
|
|
our $VERSION = '0.01'; |
12
|
|
|
|
|
|
|
|
13
|
|
|
|
|
|
|
=head1 NAME |
14
|
|
|
|
|
|
|
|
15
|
|
|
|
|
|
|
Geo::Coder::Many::Scheduler::Selective - Scheduler that times out bad geocoders |
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
=head1 DESCRIPTION |
18
|
|
|
|
|
|
|
|
19
|
|
|
|
|
|
|
This scheduler wraps another scheduler, and provides facilities for disabling |
20
|
|
|
|
|
|
|
various geocoders based on external feedback (e.g. limit exceeded messages) |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
In particular, if a geocoder returns an error, it is disabled for a timeout |
23
|
|
|
|
|
|
|
period. This period increases exponentially upon each successive consecutive |
24
|
|
|
|
|
|
|
failure. |
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
=head1 METHODS |
27
|
|
|
|
|
|
|
|
28
|
|
|
|
|
|
|
=head2 new |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
Constructs and returns an instance of the class. |
31
|
|
|
|
|
|
|
Takes a reference to an array of {name, weight} hashrefs, and the name of a |
32
|
|
|
|
|
|
|
scheduler class to wrap (e.g. Geo::Coder::Many::Scheduler::OrderedList) |
33
|
|
|
|
|
|
|
|
34
|
|
|
|
|
|
|
=cut |
35
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
sub new { |
37
|
210
|
|
|
210
|
1
|
333
|
my $class = shift; |
38
|
210
|
|
|
|
|
254
|
my $ra_geocoders = shift; |
39
|
210
|
|
|
|
|
293
|
my $scheduler = shift; |
40
|
|
|
|
|
|
|
|
41
|
210
|
50
|
|
|
|
451
|
unless (defined $scheduler) { |
42
|
0
|
|
|
|
|
0
|
croak "Selective scheduler needs to wrap an ordinary scheduler.\n"; |
43
|
|
|
|
|
|
|
} |
44
|
|
|
|
|
|
|
|
45
|
210
|
|
|
|
|
322
|
my $self = { }; |
46
|
210
|
|
|
|
|
763
|
$self->{ scheduler } = $scheduler->new( $ra_geocoders ); |
47
|
210
|
|
|
|
|
454
|
$self->{ geocoder_meta } = { }; |
48
|
|
|
|
|
|
|
|
49
|
|
|
|
|
|
|
# Length of first timeout in seconds |
50
|
210
|
|
|
|
|
394
|
$self->{ base_timeout } = 1; |
51
|
|
|
|
|
|
|
|
52
|
|
|
|
|
|
|
# Timeout multiplies by this upon each successive failure |
53
|
210
|
|
|
|
|
404
|
$self->{ timeout_multiplier } = 1.5; |
54
|
|
|
|
|
|
|
|
55
|
210
|
|
|
|
|
569
|
bless $self, $class; |
56
|
|
|
|
|
|
|
|
57
|
210
|
|
|
|
|
366
|
for my $rh_geocoder (@$ra_geocoders) { |
58
|
315
|
|
|
|
|
819
|
$self->_clear_timeout($rh_geocoder->{name}); |
59
|
|
|
|
|
|
|
} |
60
|
|
|
|
|
|
|
|
61
|
|
|
|
|
|
|
|
62
|
210
|
|
|
|
|
797
|
return $self; |
63
|
|
|
|
|
|
|
} |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
=head2 reset_available |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
Wrapper method - passes the reset on to the wrapped scheduler. |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
=cut |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
sub reset_available { |
72
|
1050
|
|
|
1050
|
1
|
1283
|
my $self = shift; |
73
|
1050
|
|
|
|
|
3014
|
$self->{scheduler}->reset_available(); |
74
|
1050
|
|
|
|
|
1859
|
return; |
75
|
|
|
|
|
|
|
} |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
=head2 get_next_unique |
78
|
|
|
|
|
|
|
|
79
|
|
|
|
|
|
|
Retrieves the next geocoder from the internal scheduler, but skipping over |
80
|
|
|
|
|
|
|
it if it isn't acceptable (e.g. being timed out) |
81
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
=cut |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
sub get_next_unique { |
85
|
1293
|
|
|
1293
|
1
|
1488
|
my $self = shift; |
86
|
|
|
|
|
|
|
|
87
|
1293
|
|
|
|
|
1435
|
my $acceptable = 0; |
88
|
1293
|
|
|
|
|
2591
|
my $t = gettimeofday(); |
89
|
1293
|
|
|
|
|
1408
|
my $geocoder; |
90
|
1293
|
|
|
|
|
2702
|
while (!$acceptable) { |
91
|
2986
|
|
|
|
|
9100
|
$geocoder = $self->{scheduler}->get_next_unique(); |
92
|
2986
|
100
|
66
|
|
|
12071
|
if (!defined $geocoder || $geocoder eq '') {return;} |
|
901
|
|
|
|
|
2080
|
|
93
|
2085
|
|
|
|
|
3918
|
my $rh_meta = $self->{geocoder_meta}->{$geocoder}; |
94
|
2085
|
100
|
|
|
|
6319
|
if ($t >= $rh_meta->{timeout_end}) { |
95
|
392
|
|
|
|
|
1058
|
$acceptable = 1; |
96
|
|
|
|
|
|
|
} |
97
|
|
|
|
|
|
|
} |
98
|
392
|
|
|
|
|
1078
|
return $geocoder; |
99
|
|
|
|
|
|
|
} |
100
|
|
|
|
|
|
|
|
101
|
|
|
|
|
|
|
=head2 next_available |
102
|
|
|
|
|
|
|
|
103
|
|
|
|
|
|
|
Returns undef if there are no more geocoders that will become available. |
104
|
|
|
|
|
|
|
Otherwise it returns the time remaining until the earliest timeout-end arrives. |
105
|
|
|
|
|
|
|
|
106
|
|
|
|
|
|
|
=cut |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
sub next_available { |
109
|
1504
|
|
|
1504
|
1
|
1815
|
my $self = shift; |
110
|
1504
|
100
|
|
|
|
4907
|
return if (!defined $self->{scheduler}->next_available()); |
111
|
2744
|
|
|
|
|
9477
|
my $first_time = |
112
|
|
|
|
|
|
|
min( |
113
|
|
|
|
|
|
|
map { |
114
|
1372
|
|
|
|
|
3972
|
$_->{timeout_end}; |
115
|
1372
|
|
|
|
|
1998
|
} values %{$self->{geocoder_meta}} |
116
|
|
|
|
|
|
|
) - gettimeofday(); |
117
|
1372
|
|
|
|
|
4655
|
return max 0, $first_time; |
118
|
|
|
|
|
|
|
} |
119
|
|
|
|
|
|
|
|
120
|
|
|
|
|
|
|
=head2 process_feedback |
121
|
|
|
|
|
|
|
|
122
|
|
|
|
|
|
|
Recieves feedback about geocoders, and sets/clears timeouts appropriately. |
123
|
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
=cut |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
sub process_feedback { |
127
|
392
|
|
|
392
|
1
|
1032
|
my ($self, $geocoder, $rh_feedback) = @_; |
128
|
|
|
|
|
|
|
|
129
|
392
|
100
|
|
|
|
890
|
if ( $rh_feedback->{response_code} != 200 ) { |
130
|
210
|
|
|
|
|
506
|
$self->_increase_timeout($geocoder); |
131
|
|
|
|
|
|
|
} |
132
|
|
|
|
|
|
|
else { |
133
|
182
|
|
|
|
|
376
|
$self->_clear_timeout($geocoder); |
134
|
|
|
|
|
|
|
} |
135
|
392
|
|
|
|
|
1270
|
return; |
136
|
|
|
|
|
|
|
} |
137
|
|
|
|
|
|
|
|
138
|
|
|
|
|
|
|
=head1 INTERNAL METHODS |
139
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
=head2 _increase_timeout |
141
|
|
|
|
|
|
|
|
142
|
|
|
|
|
|
|
Increases the timeout for the specified geocoder, according to the base_timeout |
143
|
|
|
|
|
|
|
and timeout_multiplier instance variables. |
144
|
|
|
|
|
|
|
|
145
|
|
|
|
|
|
|
=cut |
146
|
|
|
|
|
|
|
|
147
|
|
|
|
|
|
|
sub _increase_timeout { |
148
|
210
|
|
|
210
|
|
346
|
my ($self, $geocoder) = @_; |
149
|
210
|
|
|
|
|
487
|
my $rh_meta = $self->{geocoder_meta}->{$geocoder}; |
150
|
210
|
|
|
|
|
331
|
$rh_meta->{timeout_count} += 1; |
151
|
210
|
|
|
|
|
398
|
my $timeout_count = $rh_meta->{timeout_count}; |
152
|
|
|
|
|
|
|
|
153
|
210
|
|
|
|
|
336
|
my $base_timeout = $self->{base_timeout}; |
154
|
210
|
|
|
|
|
456
|
my $timeout_multiplier = $self->{timeout_multiplier}; |
155
|
|
|
|
|
|
|
|
156
|
210
|
|
|
|
|
677
|
my $timeout_length = |
157
|
|
|
|
|
|
|
$base_timeout * ($timeout_multiplier ** $timeout_count); |
158
|
|
|
|
|
|
|
|
159
|
210
|
|
|
|
|
623
|
$rh_meta->{timeout_end} = gettimeofday() + $timeout_length; |
160
|
210
|
|
|
|
|
586
|
return $timeout_length; |
161
|
|
|
|
|
|
|
}; |
162
|
|
|
|
|
|
|
|
163
|
|
|
|
|
|
|
=head2 _clear_timeout |
164
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
Clears the timeout for the specified geocoder. |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
=cut |
168
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
sub _clear_timeout { |
170
|
497
|
|
|
497
|
|
692
|
my ($self, $geocoder) = @_; |
171
|
497
|
|
|
|
|
1835
|
$self->{geocoder_meta}->{$geocoder} = { |
172
|
|
|
|
|
|
|
timeout_count => 0, |
173
|
|
|
|
|
|
|
timeout_end => 0 |
174
|
|
|
|
|
|
|
}; |
175
|
497
|
|
|
|
|
1862
|
return; |
176
|
|
|
|
|
|
|
}; |
177
|
|
|
|
|
|
|
|
178
|
|
|
|
|
|
|
1; |
179
|
|
|
|
|
|
|
|
180
|
|
|
|
|
|
|
__END__ |