line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
# HTTP connection to non-pool backend nodes (probably fast event-based webservers) |
2
|
|
|
|
|
|
|
# |
3
|
|
|
|
|
|
|
# Copyright 2004, Danga Interactive, Inc. |
4
|
|
|
|
|
|
|
# Copyright 2005-2007, Six Apart, Ltd. |
5
|
|
|
|
|
|
|
# |
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
package Perlbal::ReproxyManager; |
8
|
22
|
|
|
22
|
|
131
|
use strict; |
|
22
|
|
|
|
|
51
|
|
|
22
|
|
|
|
|
2048
|
|
9
|
22
|
|
|
22
|
|
138
|
use warnings; |
|
22
|
|
|
|
|
47
|
|
|
22
|
|
|
|
|
812
|
|
10
|
22
|
|
|
22
|
|
115
|
no warnings qw(deprecated); |
|
22
|
|
|
|
|
49
|
|
|
22
|
|
|
|
|
53453
|
|
11
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
# class storage to store 'host:ip' => $service objects, for making |
13
|
|
|
|
|
|
|
# reproxies use a service that you can then track |
14
|
|
|
|
|
|
|
our $ReproxySelf; |
15
|
|
|
|
|
|
|
our %ReproxyConnecting; # ( host:ip => $backend ); keeps track of outstanding connections to backend that |
16
|
|
|
|
|
|
|
# are in the connecting state |
17
|
|
|
|
|
|
|
our %ReproxyBored; # ( host:ip => [ $backend, ... ] ); list of our bored backends |
18
|
|
|
|
|
|
|
our %ReproxyQueues; # ( host:ip => [ $clientproxy, ... ] ); queued up requests for this backend |
19
|
|
|
|
|
|
|
our %ReproxyBackends; # ( host:ip => [ $backend, ... ] ); array of backends we have connected |
20
|
|
|
|
|
|
|
our %ReproxyMax; # ( host:ip => int ); maximum number of connections to have open at any one time |
21
|
|
|
|
|
|
|
our $ReproxyGlobalMax; # int; the global cap used if no per-host cap is specified |
22
|
|
|
|
|
|
|
our $NoSpawn = 0; # bool; when set, spawn_backend immediately returns without running |
23
|
|
|
|
|
|
|
our $LastCleanup = 0; # int; time we last ran our cleanup logic (FIXME: temp hack) |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
Perlbal::track_var("rep_connecting", \%ReproxyConnecting); |
26
|
|
|
|
|
|
|
Perlbal::track_var("rep_bored", \%ReproxyBored); |
27
|
|
|
|
|
|
|
Perlbal::track_var("rep_queues", \%ReproxyQueues); |
28
|
|
|
|
|
|
|
Perlbal::track_var("rep_backends", \%ReproxyBackends); |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
# singleton new function; returns us if we exist, else creates us |
31
|
|
|
|
|
|
|
sub get { |
32
|
17
|
100
|
|
17
|
0
|
63
|
return $ReproxySelf if $ReproxySelf; |
33
|
|
|
|
|
|
|
|
34
|
|
|
|
|
|
|
# doesn't exist, so create it and return it |
35
|
1
|
|
|
|
|
2
|
my $class = shift; |
36
|
1
|
|
|
|
|
3
|
my $self = {}; |
37
|
1
|
|
|
|
|
2
|
bless $self, $class; |
38
|
1
|
|
|
|
|
3
|
return $ReproxySelf = $self; |
39
|
|
|
|
|
|
|
} |
40
|
|
|
|
|
|
|
|
41
|
|
|
|
|
|
|
# given (clientproxy, primary_res_hdrs), initiate proceedings to process a |
42
|
|
|
|
|
|
|
# request for a reproxy resource |
43
|
|
|
|
|
|
|
sub do_reproxy { |
44
|
17
|
|
|
17
|
0
|
371
|
my Perlbal::ReproxyManager $self = Perlbal::ReproxyManager->get; # singleton |
45
|
17
|
|
|
|
|
34
|
my Perlbal::ClientProxy $cp = $_[0]; |
46
|
17
|
50
|
33
|
|
|
175
|
return undef unless $self && $cp; |
47
|
|
|
|
|
|
|
|
48
|
|
|
|
|
|
|
# get data we use |
49
|
17
|
|
|
|
|
39
|
my $datref = $cp->{reproxy_uris}->[0]; |
50
|
17
|
|
|
|
|
210
|
my $ipport = "$datref->[0]:$datref->[1]"; |
51
|
17
|
|
100
|
|
|
28
|
push @{$ReproxyQueues{$ipport} ||= []}, $cp; |
|
17
|
|
|
|
|
73
|
|
52
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
# see if we should do cleanup (FIXME: temp hack) |
54
|
17
|
|
|
|
|
32
|
my $now = time(); |
55
|
17
|
100
|
|
|
|
61
|
if ($LastCleanup < $now - 5) { |
56
|
|
|
|
|
|
|
# remove closed backends from our array. this is O(n) but n is small |
57
|
|
|
|
|
|
|
# and we're paranoid that just keeping a count would get corrupt over |
58
|
|
|
|
|
|
|
# time. also removes the backends that have clients that are closed. |
59
|
1
|
|
0
|
|
|
8
|
@{$ReproxyBackends{$ipport}} = grep { |
|
0
|
|
|
|
|
0
|
|
60
|
1
|
|
|
|
|
3
|
! $_->{closed} && (! $_->{client} || ! $_->{client}->{closed}) |
61
|
1
|
|
|
|
|
1
|
} @{$ReproxyBackends{$ipport}}; |
62
|
|
|
|
|
|
|
|
63
|
1
|
|
|
|
|
3
|
$LastCleanup = $now; |
64
|
|
|
|
|
|
|
} |
65
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
# now start a new backend |
67
|
17
|
|
|
|
|
153
|
$self->spawn_backend($ipport); |
68
|
17
|
|
|
|
|
62
|
return 1; |
69
|
|
|
|
|
|
|
} |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
# part of the reportto interface; this is called when a backend is unable to establish |
72
|
|
|
|
|
|
|
# a connection with a backend. we simply try the next uri. |
73
|
|
|
|
|
|
|
sub note_bad_backend_connect { |
74
|
1
|
|
|
1
|
0
|
3
|
my Perlbal::ReproxyManager $self = $_[0]; |
75
|
1
|
|
|
|
|
3
|
my Perlbal::BackendHTTP $be = $_[1]; |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
# decrement counts and undef connecting backend |
78
|
1
|
|
|
|
|
3
|
$ReproxyConnecting{$be->{ipport}} = undef; |
79
|
|
|
|
|
|
|
|
80
|
|
|
|
|
|
|
# if nobody waiting, doesn't matter if we couldn't get to this backend |
81
|
1
|
50
|
|
|
|
3
|
return unless @{$ReproxyQueues{$be->{ipport}} || []}; |
|
1
|
50
|
|
|
|
7
|
|
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
# if we still have some connected backends then ignore this bad connection attempt |
84
|
1
|
50
|
|
|
|
2
|
return if scalar @{$ReproxyBackends{$be->{ipport}} || []}; |
|
1
|
50
|
|
|
|
10
|
|
85
|
|
|
|
|
|
|
|
86
|
|
|
|
|
|
|
# at this point, we have no connected backends, and our connecting one failed |
87
|
|
|
|
|
|
|
# so we want to tell all of the waiting clients to try their next uri, because |
88
|
|
|
|
|
|
|
# this host is down. |
89
|
1
|
|
|
|
|
2
|
while (my Perlbal::ClientProxy $cp = shift @{$ReproxyQueues{$be->{ipport}}}) { |
|
2
|
|
|
|
|
10
|
|
90
|
1
|
|
|
|
|
5
|
$cp->try_next_uri; |
91
|
|
|
|
|
|
|
} |
92
|
1
|
|
|
|
|
3
|
return 1; |
93
|
|
|
|
|
|
|
} |
94
|
|
|
|
|
|
|
|
95
|
|
|
|
|
|
|
# called by a backend when it's ready for a request |
96
|
|
|
|
|
|
|
sub register_boredom { |
97
|
31
|
|
|
31
|
0
|
65
|
my Perlbal::ReproxyManager $self = $_[0]; |
98
|
31
|
|
|
|
|
47
|
my Perlbal::BackendHTTP $be = $_[1]; |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
# if this backend was connecting |
101
|
31
|
|
|
|
|
81
|
my $ipport = $be->{ipport}; |
102
|
31
|
100
|
66
|
|
|
146
|
if ($ReproxyConnecting{$ipport} && $ReproxyConnecting{$ipport} == $be) { |
103
|
2
|
|
|
|
|
6
|
$ReproxyConnecting{$ipport} = undef; |
104
|
2
|
|
50
|
|
|
7
|
$ReproxyBackends{$ipport} ||= []; |
105
|
2
|
|
|
|
|
4
|
push @{$ReproxyBackends{$ipport}}, $be; |
|
2
|
|
|
|
|
6
|
|
106
|
|
|
|
|
|
|
} |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
# sometimes a backend is closed but it tries to register with us anyway... ignore it |
109
|
|
|
|
|
|
|
# but since this might have been our only one, spawn another |
110
|
31
|
50
|
|
|
|
108
|
if ($be->{closed}) { |
111
|
0
|
|
|
|
|
0
|
$self->spawn_backend($ipport); |
112
|
0
|
|
|
|
|
0
|
return; |
113
|
|
|
|
|
|
|
} |
114
|
|
|
|
|
|
|
|
115
|
|
|
|
|
|
|
# find some clients to use |
116
|
31
|
50
|
|
|
|
47
|
while (my Perlbal::ClientProxy $cp = shift @{$ReproxyQueues{$ipport} || []}) { |
|
31
|
|
|
|
|
158
|
|
117
|
|
|
|
|
|
|
# safety checks |
118
|
16
|
50
|
|
|
|
43
|
next if $cp->{closed}; |
119
|
|
|
|
|
|
|
|
120
|
|
|
|
|
|
|
# give backend to client |
121
|
16
|
|
|
|
|
82
|
$cp->use_reproxy_backend($be); |
122
|
16
|
|
|
|
|
549
|
return; |
123
|
|
|
|
|
|
|
} |
124
|
|
|
|
|
|
|
|
125
|
|
|
|
|
|
|
# no clients if we get here, so push onto bored backend list |
126
|
15
|
|
100
|
|
|
24
|
push @{$ReproxyBored{$ipport} ||= []}, $be; |
|
15
|
|
|
|
|
72
|
|
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
# clean up the front of our list if we can (see docs above) |
129
|
15
|
50
|
|
|
|
50
|
if (my Perlbal::BackendHTTP $bbe = $ReproxyBored{$ipport}->[0]) { |
130
|
15
|
50
|
|
|
|
66
|
if ($bbe->{alive_time} < time() - 5) { |
131
|
0
|
|
|
|
|
0
|
$NoSpawn = 1; |
132
|
0
|
|
|
|
|
0
|
$bbe->close('have_newer_bored'); |
133
|
0
|
|
|
|
|
0
|
shift @{$ReproxyBored{$ipport}}; |
|
0
|
|
|
|
|
0
|
|
134
|
0
|
|
|
|
|
0
|
$NoSpawn = 0; |
135
|
|
|
|
|
|
|
} |
136
|
|
|
|
|
|
|
} |
137
|
15
|
|
|
|
|
38
|
return 0; |
138
|
|
|
|
|
|
|
} |
139
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
# backend closed, decrease counts, etc |
141
|
|
|
|
|
|
|
sub note_backend_close { |
142
|
2
|
|
|
2
|
0
|
4
|
my Perlbal::ReproxyManager $self = $_[0]; |
143
|
2
|
|
|
|
|
3
|
my Perlbal::BackendHTTP $be = $_[1]; |
144
|
|
|
|
|
|
|
|
145
|
|
|
|
|
|
|
# remove closed backends from our array. this is O(n) but n is small |
146
|
|
|
|
|
|
|
# and we're paranoid that just keeping a count would get corrupt over |
147
|
|
|
|
|
|
|
# time. |
148
|
2
|
|
|
|
|
7
|
@{$ReproxyBackends{$be->{ipport}}} = grep { |
|
1
|
|
|
|
|
5
|
|
149
|
2
|
|
|
|
|
9
|
! $_->{closed} |
150
|
2
|
|
|
|
|
4
|
} @{$ReproxyBackends{$be->{ipport}}}; |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
# spawn more if needed |
153
|
2
|
|
|
|
|
7
|
$self->spawn_backend($be->{ipport}); |
154
|
|
|
|
|
|
|
} |
155
|
|
|
|
|
|
|
|
156
|
|
|
|
|
|
|
sub spawn_backend { |
157
|
19
|
50
|
|
19
|
0
|
51
|
return if $NoSpawn; |
158
|
|
|
|
|
|
|
|
159
|
19
|
|
|
|
|
31
|
my Perlbal::ReproxyManager $self = $_[0]; |
160
|
19
|
|
|
|
|
27
|
my $ipport = $_[1]; |
161
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
# if we're already connecting, we don't want to spawn another one |
163
|
19
|
50
|
|
|
|
63
|
if (my Perlbal::BackendHTTP $be = $ReproxyConnecting{$ipport}) { |
164
|
|
|
|
|
|
|
# see if this one is too old? |
165
|
0
|
0
|
|
|
|
0
|
if ($be->{create_time} < (time() - 5)) { # older than 5 seconds? |
166
|
0
|
|
|
|
|
0
|
$self->note_bad_backend_connect($be); |
167
|
0
|
|
|
|
|
0
|
$be->close("connection_timeout"); |
168
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
# we return here instead of spawning because closing the backend calls |
170
|
|
|
|
|
|
|
# note_backend_close which will call spawn_backend again, and at that |
171
|
|
|
|
|
|
|
# point we won't have a pending connection and can spawn |
172
|
0
|
|
|
|
|
0
|
return; |
173
|
|
|
|
|
|
|
} else { |
174
|
|
|
|
|
|
|
# don't spawn more if we're already connecting |
175
|
0
|
|
|
|
|
0
|
return; |
176
|
|
|
|
|
|
|
} |
177
|
|
|
|
|
|
|
} |
178
|
|
|
|
|
|
|
|
179
|
|
|
|
|
|
|
# if nobody waiting, don't spawn extra connections |
180
|
19
|
50
|
|
|
|
89
|
return unless @{$ReproxyQueues{$ipport} || []}; |
|
19
|
100
|
|
|
|
90
|
|
181
|
|
|
|
|
|
|
|
182
|
|
|
|
|
|
|
# don't spawn if we have a bored one already |
183
|
17
|
100
|
|
|
|
33
|
while (my Perlbal::BackendHTTP $bbe = pop @{$ReproxyBored{$ipport} || []}) { |
|
17
|
|
|
|
|
104
|
|
184
|
|
|
|
|
|
|
|
185
|
|
|
|
|
|
|
# don't use keep-alive connections if we know the server's |
186
|
|
|
|
|
|
|
# just about to kill the connection for being idle |
187
|
14
|
|
|
|
|
22
|
my $now = time(); |
188
|
14
|
50
|
33
|
|
|
159
|
if ($bbe->{disconnect_at} && $now + 2 > $bbe->{disconnect_at} || |
|
|
|
33
|
|
|
|
|
189
|
|
|
|
|
|
|
$bbe->{alive_time} < $now - 5) |
190
|
|
|
|
|
|
|
{ |
191
|
0
|
|
|
|
|
0
|
$NoSpawn = 1; |
192
|
0
|
|
|
|
|
0
|
$bbe->close("too_close_disconnect"); |
193
|
0
|
|
|
|
|
0
|
$NoSpawn = 0; |
194
|
0
|
|
|
|
|
0
|
next; |
195
|
|
|
|
|
|
|
} |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
# it's good, give it to someone |
198
|
14
|
|
|
|
|
61
|
$self->register_boredom($bbe); |
199
|
14
|
|
|
|
|
37
|
return; |
200
|
|
|
|
|
|
|
} |
201
|
|
|
|
|
|
|
|
202
|
|
|
|
|
|
|
# see if we have too many already? |
203
|
3
|
|
50
|
|
|
21
|
my $max = $ReproxyMax{$ipport} || $ReproxyGlobalMax || 0; |
204
|
3
|
100
|
|
|
|
4
|
my $count = scalar @{$ReproxyBackends{$ipport} || []}; |
|
3
|
|
|
|
|
12
|
|
205
|
3
|
50
|
33
|
|
|
10
|
return if $max && ($count >= $max); |
206
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
# start one connecting and enqueue |
208
|
3
|
50
|
|
|
|
45
|
my $be = Perlbal::BackendHTTP->new(undef, split(/:/, $ipport), { reportto => $self }) |
209
|
|
|
|
|
|
|
or return 0; |
210
|
3
|
|
|
|
|
14
|
$ReproxyConnecting{$ipport} = $be; |
211
|
|
|
|
|
|
|
} |
212
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
sub backend_response_received { |
214
|
16
|
|
|
16
|
0
|
33
|
my Perlbal::ReproxyManager $self = $_[0]; |
215
|
16
|
|
|
|
|
29
|
my Perlbal::BackendHTTP $be = $_[1]; |
216
|
16
|
|
|
|
|
38
|
my Perlbal::ClientProxy $cp = $be->{client}; |
217
|
|
|
|
|
|
|
|
218
|
|
|
|
|
|
|
# if no client, close backend and return 1 |
219
|
16
|
50
|
|
|
|
45
|
unless ($cp) { |
220
|
0
|
|
|
|
|
0
|
$be->close("lost_client"); |
221
|
0
|
|
|
|
|
0
|
return 1; |
222
|
|
|
|
|
|
|
} |
223
|
|
|
|
|
|
|
|
224
|
|
|
|
|
|
|
# pass on to client |
225
|
16
|
|
|
|
|
79
|
return $cp->backend_response_received($be); |
226
|
|
|
|
|
|
|
} |
227
|
|
|
|
|
|
|
|
228
|
|
|
|
|
|
|
sub dump_state { |
229
|
0
|
|
|
0
|
0
|
|
my $out = shift; |
230
|
0
|
0
|
|
|
|
|
return unless $out; |
231
|
|
|
|
|
|
|
|
232
|
|
|
|
|
|
|
# spits out what we have connecting |
233
|
0
|
|
|
|
|
|
while (my ($hostip, $dat) = each %ReproxyConnecting) { |
234
|
0
|
0
|
|
|
|
|
$out->("connecting $hostip 1") if defined $dat; |
235
|
|
|
|
|
|
|
} |
236
|
0
|
|
|
|
|
|
while (my ($hostip, $dat) = each %ReproxyBored) { |
237
|
0
|
|
|
|
|
|
$out->("bored $hostip " . scalar(@$dat)); |
238
|
|
|
|
|
|
|
} |
239
|
0
|
|
|
|
|
|
while (my ($hostip, $dat) = each %ReproxyQueues) { |
240
|
0
|
|
|
|
|
|
$out->("clients_queued $hostip " . scalar(@$dat)); |
241
|
|
|
|
|
|
|
} |
242
|
0
|
|
|
|
|
|
while (my ($hostip, $dat) = each %ReproxyBackends) { |
243
|
0
|
|
|
|
|
|
$out->("backends $hostip " . scalar(@$dat)); |
244
|
0
|
|
|
|
|
|
foreach my $be (@$dat) { |
245
|
0
|
|
|
|
|
|
$out->("... " . $be->as_string); |
246
|
|
|
|
|
|
|
} |
247
|
|
|
|
|
|
|
} |
248
|
0
|
|
|
|
|
|
while (my ($hostip, $dat) = each %ReproxyMax) { |
249
|
0
|
|
|
|
|
|
$out->("SERVER max_reproxy_connections($hostip) = $dat"); |
250
|
|
|
|
|
|
|
} |
251
|
0
|
|
0
|
|
|
|
$out->("SERVER max_reproxy_connections = " . ($ReproxyGlobalMax || 0)); |
252
|
0
|
|
|
|
|
|
$out->('.'); |
253
|
|
|
|
|
|
|
} |
254
|
|
|
|
|
|
|
|
255
|
|
|
|
|
|
|
1; |