Yenta/Status.pm untested

filter: Statementstmt 0.0% 0 / 242 Branchbran 0.0% 0 / 78 Conditioncond 0.0% 0 / 35 MC/DCmcdc 0.0% 0 / 22 Subroutinesub 0.0% 0 / 30 Podpod 0.0% 0 / 19 totaltotal 0.0% 0 / 426 CC 40 SCAR 38.2
CC
40
Cov
0%
update
156.0
_random_peer
56.0
save_status
56.0
CRAP 45.4
1 # -*- perl -*-
2  
3 # Copyright (c) 2009 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2009-Apr-02 11:16 (EDT)
6 # Function: track status of peers
7 #
8 # $Id$
9  
10 package AC::Yenta::Status;
11 use AC::Yenta::Kibitz::Status;
12 use AC::Yenta::Debug 'status';
13 use AC::Yenta::Config;
14 use AC::Yenta::MySelf;
15 use AC::Dumper;
16 use AC::Misc;
17 use Sys::Hostname;
18 use JSON;
19 use Socket;
20 require 'AC/protobuf/yenta_status.pl';
21 use strict;
22  
23 my $KEEPDOWN = 1800; # keep data about down servers for how long?
24 my $KEEPLOST = 600; # keep data about servers we have not heard about for how long?
25 my $SAVEMAX = 1800; # do not save if older than
26  
27 my $PORT;
28  
29 our $DATA = bless {
30 allpeer => {}, # yenta_status
31 sceptical => {},
32 mappeer => {}, # {map} => { id => id }
33 peermap => {}, # {id} => @map
34 datacenter => {}, # {dc} => { id => id }
35 peertype => {}, # {ss} => { id => id }
36 };
37  
38 sub init {
39 my $port = shift;
40  
41 $PORT = $port;
42  
43 AC::DC::Sched->new(
44 info => 'kibitz status',
45 freq => (conf_value('time_status_kibitz') || 5),
46 func => \&periodic,
47 );
48 AC::DC::Sched->new(
49 info => 'save status',
50 freq => (conf_value('time_status_save') || 5),
51 func => \&save_status,
52 );
53 }
54  
55 # start up a client every so often
56 sub periodic {
57  
58 # clean up down or lost peers
59 for my $id ( keys %{$DATA->{allpeer}} ){
60 my $p = $DATA->{allpeer}{$id};
61 next unless $p;
62  
63 next if $p->{status} == 200 && $p->{timestamp} > $^T - $KEEPLOST;
64 _maybe_remove( $id );
65 }
66  
67 # randomly pick a peer
68 my($id, $ip, $port) = _random_peer();
69 return unless $id;
70  
71 # start a client
72 debug("starting status kibitz client to $id");
73  
74 my $c = AC::Yenta::Kibitz::Status::Client->new( $ip, $port,
75 info => "status client: $id",
76 status_peer => $id,
77 );
78 return __PACKAGE__->isdown($id) unless $c;
79  
80 $c->start();
81 }
82  
83 sub _random_peer {
84  
85 my $here = my_datacenter();
86  
87 # sceptical
88 my @scept = values %{$DATA->{sceptical}};
89  
90 my @all = map { $DATA->{allpeer}{$_} } keys %{$DATA->{peertype}{yenta}};
91 my @old = grep { $_->{timestamp} < $^T - $KEEPLOST *.75 } @all;
92 my @local = grep { $_->{datacenter} eq $here } @all; # this datacenter
93 my @away = grep { $_->{datacenter} ne $here } @all; # not this datacenter
94  
95 # first check anything sceptical
96 my @peer = @scept;
97  
98 # then (maybe) something about to expire
99 @peer = @old unless @peer || int rand(5);
100  
101 # then (maybe) something far away
102 @peer = @away unless @peer || int rand(5);
103  
104 # then something local
105 @peer = @local unless @peer;
106  
107 # last resort
108 @peer = @all unless @peer;
109  
110 # sometimes use the seed, in case there was a network split
111 if( @peer && int(rand(@all+1)) ){
112 my $p = $peer[ rand(@peer) ];
113 debug("using peer $p->{server_id}");
114 return ($p->{server_id}, $p->{ip}, undef);
115 }
116  
117 # seed peer
118 my $seed = conf_value('seedpeer');
119 my $p = $seed->[ rand(@$seed) ];
120 my ($ip, $port) = split /:/, $p;
121 $port ||= my_port();
122  
123 # don't talk to self. any of my addrs.
124 my $ipinfo = my_network_info();
125 for my $i (@$ipinfo){
126 return if $ip eq $i->{ipa} && $port == $PORT;
127 }
128  
129 return("seed/$ip:$port", $ip, $port);
130 }
131  
132 # server list for save file
133 sub server_list {
134 my $type = shift;
135  
136 ($type, my $where) = split m|/|, $type;
137 # where - no longer used
138 $where ||= my_datacenter();
139  
140 my @peer = keys %{ $DATA->{peertype}{$type} };
141 return unless @peer;
142  
143 # nothing too old
144 @peer = grep { $DATA->{allpeer}{$_}{lastup} > $^T - $SAVEMAX } @peer;
145 return unless @peer;
146  
147 return map { $DATA->{allpeer}{$_} } @peer;
148 }
149  
150 # save a list of peers, in case I crash, and for others to use
151 sub save_status {
152  
153 my $save = conf_value('savestatus');
154 my $here = my_datacenter();
155  
156 # also save locally running services
157 my @mon = AC::Yenta::Monitor::export();
158  
159 for my $s ( @$save ){
160 my $file = $s->{file};
161 my $types = $s->{type};
162  
163 my @peer;
164 for my $type (@$types){
165 push @peer, server_list($type);
166  
167 for my $m (@mon){
168 push @peer, $m if $m->{subsystem} eq $type;
169 }
170 }
171  
172 next unless @peer;
173  
174 debug("saving peer status file");
175 unless( open(FILE, ">$file.tmp") ){
176 problem("cannot open save file '$file.tmp': $!");
177 return;
178 }
179  
180 for my $pd (@peer){
181 # only save best addr in save file
182 my($ip, $port) = AC::Yenta::IO::TCP::Client->use_addr_port( $pd->{ip} );
183  
184 my $data = {
185 id => $pd->{server_id},
186 addr => $ip,
187 port => int($port),
188 status => int($pd->{status}),
189 subsystem => $pd->{subsystem},
190 environment => $pd->{environment},
191 sort_metric => int($pd->{sort_metric}),
192 capacity_metric => int($pd->{capacity_metric}),
193 datacenter => $pd->{datacenter},
194 is_local => ($here eq $pd->{datacenter} ? 1 : 0),
195 };
196 if( $pd->{subsystem} eq 'yenta' ){
197 $data->{map} = $pd->{map};
198 }
199  
200 print FILE encode_json( $data ), "\n";
201 }
202  
203 close FILE;
204 unless( rename("$file.tmp", $file) ){
205 problem("cannot rename save file '$file': $!");
206 }
207  
208 }
209 }
210  
211 ################################################################
212 # diagnostic reports
213 sub report {
214  
215 my $res;
216  
217 for my $v (AC::Yenta::Kibitz::Status::_myself(), AC::Yenta::Monitor::export(), values %{$DATA->{allpeer}} ){
218 my $id = sprintf '%-28s', $v->{server_id};
219 my $metric = int( $v->{sort_metric} );
220 $res .= "$id $v->{hostname}\t$v->{datacenter}\t$v->{subsystem}\t$v->{environment}\t$v->{status}\t$metric\n";
221 }
222  
223 return $res;
224 }
225  
226 sub report_long {
227  
228 my $res;
229  
230 for my $v (AC::Yenta::Kibitz::Status::_myself(), AC::Yenta::Monitor::export(), values %{$DATA->{allpeer}} ){
231 $res .= dumper( $v ) . "\n\n";
232 }
233 return $res;
234 }
235 ################################################################
236  
237 sub my_port { $PORT }
238  
239  
240 sub my_instance_id {
241 my $class = shift;
242 return my_server_id() . sprintf('/%04x', $$);
243 }
244  
245 sub peer {
246 my $class = shift;
247 my $id = shift;
248  
249 return $DATA->{allpeer}{$id};
250 }
251  
252 sub allpeers {
253 my $class = shift;
254  
255 # idown sets status to 0 (below), skip such
256 return grep { $_->{status} } values %{$DATA->{allpeer}};
257 }
258  
259 sub mappeers {
260 my $class = shift;
261 my $map = shift;
262  
263 return keys %{ $DATA->{mappeer}{$map} };
264 }
265  
266 sub datacenters {
267 my $class = shift;
268  
269 return $DATA->{datacenter};
270 }
271 ################################################################
272  
273 sub _remove {
274 my $id = shift;
275  
276 my $ss = $DATA->{allpeer}{$id}{subsystem};
277 delete $DATA->{peertype}{$ss}{$id} if $ss;
278  
279 my $dc = $DATA->{allpeer}{$id}{datacenter};
280 delete $DATA->{datacenter}{$dc}{$id} if $dc;
281  
282 verbose("deleting peer: $id");
283 delete $DATA->{allpeer}{$id};
284  
285 # remove map info
286 for my $map ( @{$DATA->{peermap}{$id}} ){
287 delete $DATA->{mappeer}{$map}{$id};
288 }
289 delete $DATA->{peermap}{$id};
290  
291 # delete its monitored items
292 for my $p (keys %{$DATA->{allpeer}}){
293 next unless $DATA->{allpeer}{$p}{via} eq $id;
294 _remove($p);
295 }
296 }
297  
298 sub _maybe_remove {
299 my $id = shift;
300  
301 my $d = $DATA->{allpeer}{$id};
302  
303 if( ($^T - $d->{lastup} > $KEEPDOWN) || ($^T - $d->{timestamp} > $KEEPLOST) ){
304  
305 _remove($id);
306 }
307 }
308  
309 sub isdown {
310 my $class = shift;
311 my $id = shift;
312  
313 debug("marking peer '$id' as down");
314  
315 if( ! $DATA->{allpeer}{$id} ){
316 return unless $DATA->{sceptical}{$id};
317 # we know it is down, and want to kibbitz this fact
318 $DATA->{allpeer}{$id} = $DATA->{sceptical}{$id};
319 }
320  
321 delete $DATA->{sceptical}{$id};
322  
323 if( $DATA->{allpeer}{$id} ){
324 $DATA->{allpeer}{$id}{timestamp} = $^T;
325 $DATA->{allpeer}{$id}{status} = 0;
326 $DATA->{allpeer}{$id}{path} = my_server_id();
327 }
328 _maybe_remove( $id );
329 }
330  
331 ################################################################
332  
333 sub _env_ok {
334 my $class = shift;
335 my $id = shift;
336 my $up = shift;
337  
338 # if( $up->{environment} ne conf_value('environment') ){
339 # verbose("ignoring update from $id - wrong env: $up->{environment}");
340 # return;
341 # }
342 return 1;
343 }
344  
345 sub update_sceptical {
346 my $class = shift;
347 my $id = shift; # ->server_id
348 my $up = shift;
349 my $io = shift;
350  
351 return unless $class->_env_ok($id, $up);
352  
353 if( $DATA->{allpeer}{$id} ){
354 # already known
355 delete $DATA->{sceptical}{$id};
356 return;
357 }
358  
359 debug("rcvd update (sceptical) about $id from $io->{peerip}");
360  
361 # only accept updates from the server itself
362 # no 3rd party updates. no misconfigured serevrs.
363 problem("server misconfigured $id != $io->{peerip}")
364 unless grep { inet_atoi($io->{peerip}) == $_->{ipv4} } @{$up->{ip}};
365  
366 $up->{id} = $id;
367 delete $up->{lastup};
368 $DATA->{sceptical}{$id} = $up;
369 }
370  
371 sub update {
372 my $class = shift;
373 my $id = shift; # -> server_id
374 my $up = shift;
375  
376 return unless $class->_env_ok($id, $up);
377  
378 # only keep it if it is relatively fresh, and valid
379 return unless $up->{timestamp} > $^T - $KEEPLOST;
380 return unless $up->{status};
381  
382 delete $DATA->{sceptical}{$id};
383  
384 $up->{id} = $id;
385 my $previnfo = $DATA->{allpeer}{$id};
386 verbose("discovered new peer: $id ($up->{hostname})") unless $previnfo;
387  
388 # only keep it if it is newer than what we have
389 return if $previnfo && $up->{timestamp} <= $previnfo->{timestamp};
390  
391 $up->{path} .= ' ' . my_server_id();
392  
393 # debug("updating $id => $up->{status} => " . dumper($up));
394 debug("updating $id => $up->{status}");
395  
396 $DATA->{allpeer}{$id} = $up;
397  
398 if( $up->{status} != 200 ){
399 _maybe_remove( $id );
400 return ;
401 }
402  
403 # update datacenter info
404 unless( $DATA->{datacenter}{$up->{datacenter}}{$id} ){
405 my $pdc = $previnfo->{datacenter};
406 delete $DATA->{datacenter}{$pdc}{$id} if $pdc;
407 $DATA->{datacenter}{$up->{datacenter}}{$id} = $id;
408 }
409  
410 # update subsystem info
411 unless( $DATA->{peertype}{$up->{subsystem}}{$id} ){
412 my $ss = $previnfo->{subsystem};
413 delete $DATA->{peertype}{$ss}{$id} if $ss;
414 $DATA->{peertype}{$up->{subsystem}}{$id} = $id;
415 }
416  
417 # update map info
418 $DATA->{peermap}{$id} ||= [];
419 $up->{map} ||= [];
420 my @curmap = @{$DATA->{peermap}{$id}};
421 my @newmap = sort @{$up->{map}};
422  
423 return if "@curmap" eq "@newmap"; # unchanged
424  
425 # what do we need to add/remove
426 my (%remove, %add);
427 @remove{@curmap} = @curmap;
428 @add{@newmap} = @newmap;
429 delete $remove{$_} for @newmap;
430 delete $add{$_} for @curmap;
431  
432 for my $map (keys %remove){
433 debug("removing $map from $id");
434 delete $DATA->{mappeer}{$map}{$id};
435 }
436 for my $map (keys %add){
437 debug("adding $map to $id");
438 $DATA->{mappeer}{$map}{$id} = $id;
439 }
440 $DATA->{peermap}{$id} = \@newmap;
441 }
442