Yenta/Store/Distrib.pm untested

filter: Statementstmt 0.0% 0 / 200 Branchbran 0.0% 0 / 70 Conditioncond 0.0% 0 / 12 Subroutinesub 0.0% 0 / 21 Podpod 0.0% 0 / 12 totaltotal 0.0% 0 / 315 CC 36 SCAR 32.0
CC
36
Cov
0%
_start_next
56.0
_onload
42.0
new
30.0
CRAP 24.6
1 # -*- perl -*-
2  
3 # Copyright (c) 2009 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2009-Apr-01 18:56 (EDT)
6 # Function: distribute data to other peers
7 #
8 # $Id: Distrib.pm,v 1.19 2011/01/12 19:13:29 jaw Exp $
9  
10 package AC::Yenta::Store::Distrib;
11 use AC::Yenta::Kibitz::Store::Client;
12 use AC::Yenta::Debug 'distrib';
13 use AC::Yenta::Config;
14 use AC::Yenta::Protocol;
15 use AC::Yenta::Stats;
16 use AC::Yenta::MySelf;
17 use AC::Misc;
18 use AC::DC::Sched;
19 use strict;
20  
21 my $MAXHOP = 10;
22 my $MAXFARSEE = 2;
23 my $MAXNEARSEE = 3;
24 my $FARSENDS = 1;
25 my $NEARSENDS = 2;
26 my $MAXUNDERWAY = 64;
27  
28 my $msgid = $$;
29 my @DIST;
30  
31 AC::DC::Sched->new(
32 info => 'distribution',
33 freq => 5,
34 func => \&AC::Yenta::Store::Distrib::periodic,
35 );
36  
37 sub new {
38 my $class = shift;
39 my $req = shift;
40 my $cont = shift;
41  
42 return if $req->{hop} >= $MAXHOP;
43 return if $req->{expire} < $^T;
44  
45 my $sender = $req->{sender};
46 my $sendat = AC::Yenta::Status->peer($sender);
47  
48 my $me = bless {
49 info => "$req->{datum}{map}/$req->{datum}{key}/$req->{datum}{version}",
50 map => $req->{datum}{map},
51 req => $req,
52 content => $cont,
53 # we tune the distribution algorithm based on where it came from:
54 faraway => (my_datacenter() ne $sendat->{datacenter}),
55  
56 farseen => 0,
57 nearseen => 0,
58 farsend => [],
59 nearsend => [],
60 ordershift => 4,
61 }, $class;
62  
63 debug("distributing $me->{info}");
64 inc_stat( 'dist_requests' );
65 inc_stat( 'dist_requests_faraway' ) if $me->{faraway};
66  
67  
68 $me->_init_strategy($sender);
69  
70 # RSN - check load
71 my $max = conf_value('distrib_max') || $MAXUNDERWAY;
72 if( @DIST < $max ){
73 $me->_start_next();
74 }
75 push @DIST, $me;
76  
77 return $me;
78 }
79  
80 # periodically, go through and restart or expire
81 sub periodic {
82  
83 my @keep;
84 my $max = conf_value('distrib_max') || $MAXUNDERWAY;
85  
86 my $chance = (@DIST > $max) ? ($max / @DIST) : 1;
87  
88 for my $r (@DIST){
89 # debug("periodic $r->{info}");
90 next if $^T > $r->{req}{expire};
91  
92 if( (rand() <= $chance) && (AC::DC::IO->underway() <= 2 * $max) ){
93 my $keep = $r->_start_next();
94 push @keep, $r if $keep;
95 }else{
96 push @keep, $r;
97 }
98 }
99  
100 @DIST = @keep;
101 }
102  
103 ################################################################
104 # determine distribution strategy
105 # - if we recvd it from faraway, we will send it to other datacenters, and randomly in the same datacenter
106 # - otherwise we send it in the same datacenter, in an orderly fashion
107 # RSN - find an strategy with faster convergence + less duplication
108  
109 sub _init_strategy {
110 my $me = shift;
111 my $sender = shift;
112  
113 my $here = my_datacenter();
114 my $dcs = AC::Yenta::Status->datacenters();
115 my $sendat = AC::Yenta::Status->peer($sender);
116 my(@far, @near);
117  
118 for my $dc (keys %$dcs){
119 if( $dc eq $here ){
120 push @near, grep { $_ ne $sender } $me->_compat_peers_in_dc($dc);
121 }else{
122 next if $dc eq $sendat->{datacenter};
123  
124 push @far, {
125 dc => $dc,
126 id => [ $me->_compat_peers_in_dc($dc) ],
127 };
128 }
129 }
130  
131 if( $me->{faraway} ){
132 $me->{nearsend} = shuffle(\@near);
133 $me->{farsend} = shuffle(\@far);
134 }else{
135 $me->{nearsend} = _orderly(\@near);
136 }
137 }
138  
139 # which yentas can do something with the update?
140 sub _compat_peers_in_dc {
141 my $me = shift;
142 my $dc = shift;
143  
144 my $env = conf_value('environment');
145 my $dcs = AC::Yenta::Status->datacenters();
146 my $map = $me->{map};
147 my @id;
148  
149 for my $id (keys %{$dcs->{$dc}}){
150 my $pd = AC::Yenta::Status->peer($id);
151  
152 next unless $pd->{subsystem} eq 'yenta';
153 next unless $pd->{environment} eq $env;
154 next unless grep {$map eq $_} @{ $pd->{map} };
155 push @id, $id;
156 }
157 return @id;
158 }
159  
160 sub _start_far {
161 my $me = shift;
162  
163 my $d = shift @{ $me->{farsend} };
164 return unless $d;
165  
166 # randomly pick one server in chosen dc
167 my @id = grep {
168 my $x = AC::Yenta::Status->peer($_);
169 ($x->{status} == 200) ? 1 : 0;
170 } @{$d->{id}};
171 return unless @id;
172  
173 my $id = $id[ rand(@id) ];
174 debug("sending $me->{info} to far site $id in $d->{dc}");
175 $me->_start_peer( $id, 1 );
176 inc_stat('dist_send_far');
177 inc_stat('dist_send_total');
178 return 1;
179 }
180  
181 sub _start_near {
182 my $me = shift;
183  
184 my $id = shift @{ $me->{nearsend} };
185 return unless $id;
186 debug("sending $me->{info} to nearby site $id");
187 $me->_start_peer( $id, 0 );
188 inc_stat('dist_send_near');
189 inc_stat('dist_send_total');
190 return 1;
191 }
192  
193 sub _start_next {
194 my $me = shift;
195  
196 my $sent;
197  
198 # pick next peers
199 # start clients
200  
201 if( $me->{faraway} ){
202 if( $me->{farseen} < $MAXFARSEE ){
203 for (1 .. $FARSENDS){
204 $sent ++ if $me->_start_far();
205 }
206 }
207 if( $me->{nearseen} < $MAXNEARSEE ){
208 for (1 .. $NEARSENDS){
209 $sent ++ if $me->_start_near();
210 }
211 }
212 }else{
213 $sent ++ if $me->_start_near();
214 }
215  
216 return $sent;
217 }
218  
219 sub _start_one {
220 my $me = shift;
221 my $far = shift;
222  
223 if( $far ){
224 return if $me->{farseen} >= $MAXFARSEE;
225 $me->_start_far();
226 }else{
227 return if $me->{nearseen} >= $MAXNEARSEE;
228 $me->_start_near();
229 }
230 }
231  
232 sub _start_peer {
233 my $me = shift;
234 my $id = shift;
235 my $far = shift;
236  
237 my $pd = AC::Yenta::Status->peer($id);
238 my $addr = $pd->{ip}; # array of nat ip info
239  
240 my $enc = use_encryption($pd);
241 my $ect = '';
242 my $proto = AC::Yenta::Protocol->new( secret => conf_value('secret') );
243 $ect = $enc ? $proto->encrypt(undef, ${$me->{content}}) : ${$me->{content}} if $me->{content};
244  
245 # build request
246 my $request = $proto->encode_request( {
247 type => 'yenta_distrib',
248 msgidno => $msgid++,
249 want_reply => 1,
250 data_encrypted => $enc,
251 content_encrypted => $enc,
252 }, {
253 sender => AC::Yenta::Status->my_server_id(),
254 hop => $me->{req}{hop} + 1,
255 expire => $me->{req}{expire},
256 datum => $me->{req}{datum},
257 }, \$ect );
258  
259 # connect + send
260 my $io = AC::Yenta::Kibitz::Store::Client->new($addr, undef,
261 $request . $ect,
262 info => "distrib $me->{info} to $id",
263 );
264  
265 if( $io ){
266 $io->set_callback('load', \&_onload, $me, $id, $far);
267 $io->set_callback('error', \&_onerror, $me, $id, $far);
268 $io->start();
269 }else{
270 debug("start client failed");
271 }
272 }
273  
274 sub _onload {
275 my $io = shift;
276 my $evt = shift;
277 my $me = shift;
278 my $id = shift;
279 my $far = shift;
280  
281 debug("dist finish $me->{info} with $id => $evt->{data}{haveit}");
282  
283 if( $evt->{data}{haveit} ){
284 if( $far ){
285 $me->{farseen} ++;
286 inc_stat('dist_send_far_seen');
287 }else{
288 $me->{nearseen} ++;
289 inc_stat('dist_send_near_seen');
290 }
291 }
292  
293 if( !$me->{faraway} && !$far ){
294 # orderly distribution. hop away.
295 if( $evt->{data}{haveit} ){
296 shift @{$me->{nearsend}};
297 }else{
298 my $n = $me->{ordershift};
299 $n = @{$me->{nearsend}} / 2 if $n > @{$me->{nearsend}} / 2;
300 shift @{$me->{nearsend}} for (1 .. $n);
301 $me->{ordershift} *= 2;
302 }
303 }
304  
305 $me->_start_one($far);
306 }
307  
308 sub _onerror {
309 my $io = shift;
310 my $evt = shift;
311 my $me = shift;
312 my $id = shift;
313 my $far = shift;
314  
315 verbose("error distributing $me->{info} to $id");
316 # don't need to track anything
317  
318 $me->_start_one($far);
319 }
320  
321 sub _orderly {
322 my $peers = shift;
323  
324 my $myself = AC::Yenta::Status->my_server_id();
325 my @p = sort {$a cmp $b} @$peers;
326  
327 my @left = grep { $_ lt $myself } @p;
328 my @right = grep { $_ gt $myself } @p;
329  
330 @p = (@right, @left);
331 return \@p;
332 }
333  
334 1;