Yenta/Monitor.pm untested

filter: Statementstmt 0.0% 0 / 96 Branchbran 0.0% 0 / 26 Conditioncond 0.0% 0 / 5 MC/DCmcdc 0.0% 0 / 3 Subroutinesub 0.0% 0 / 16 Podpod 0.0% 0 / 6 totaltotal 0.0% 0 / 152 CC 14 SCAR 27.2
CC
14
Cov
0%
isdown
30.0
_hb_ip_info
30.0
update
20.0
CRAP 15.2
1 # -*- perl -*-
2  
3 # Copyright (c) 2009 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2009-May-12 11:03 (EDT)
6 # Function: monitor related processes
7 #
8 # $Id$
9  
10 # we periodically check the heartbeats of various processes (dancrs, scriblers, ...)
11 # and kibitz their info around the network
12  
13 package AC::Yenta::Monitor;
14 use AC::Yenta::Debug 'monitor';
15 use AC::Yenta::Config;
16 use AC::Yenta::Monitor::Client;
17 use AC::Misc;
18 use AC::Yenta::MySelf;
19  
20 use Sys::Hostname;
21 use Socket;
22 use strict;
23  
24 require 'AC/protobuf/yenta_status.pl';
25 require 'AC/protobuf/heartbeat.pl';
26  
27 my $FREQ = 2;
28 my $OLD_DOWN = 30;
29 my $OLD_KEEP = 1800;
30  
31 my %MON; # by 'id' (from config file)
32  
33 sub init {
34  
35 AC::DC::Sched->new(
36 info => 'monitor',
37 freq => $FREQ,
38 func => \&periodic,
39 );
40 }
41  
42 sub periodic {
43  
44 my $mon = conf_value('monitor');
45  
46 # clean up old data
47 for my $id (keys %MON){
48 isdown($id, 0) if $MON{$id}{lastup} < $^T - $OLD_DOWN;
49 }
50  
51 # start monitoring (send heartbeat request)
52 for my $m (@$mon){
53 my $ip = $m->{ipa};
54 my $port = $m->{port};
55 my $id = "$ip:$port";
56 debug("start monitoring $id");
57  
58 my $ok = AC::Yenta::Monitor::Client->new( $ip, $port,
59 info => "monitor client: $id",
60 monitor_peer => $id,
61 );
62  
63 isdown($id, 0) unless $ok;
64 }
65 }
66  
67 # data for kibitzing around
68 # array of ACPYentaStatus
69 sub export {
70  
71 my @d;
72 my $here = my_datacenter();
73 my $self = my_server_id();
74  
75 for my $v (values %MON){
76  
77 push @d, {
78 id => $v->{id}, # from config, typically localhost:port
79 datacenter => $here,
80 via => $self,
81 hostname => $v->{hostname},
82 subsystem => $v->{subsystem},
83 environment => $v->{environment},
84 status => $v->{status_code},
85 timestamp => $v->{timestamp},
86 lastup => $v->{lastup},
87 sort_metric => $v->{sort_metric},
88 capacity_metric => $v->{capacity_metric},
89 server_id => $v->{server_id},
90 instance_id => $v->{server_id},
91 ip => $v->{ip},
92 path => '.',
93 };
94 }
95 return @d;
96 }
97  
98 sub isdown {
99 my $id = shift;
100 my $code = shift;
101  
102 my $d = $MON{$id};
103 return unless $d;
104  
105 # require 2 polls to fail
106 return unless $^T - $d->{lastup} >= 2 * $FREQ;
107  
108 $code = 0 if $code == 200;
109 $d->{status_code} = $code || 0;
110 $d->{timestamp} = $^T;
111  
112 debug("monitor $id is down");
113  
114 if( $d->{lastup} < $^T - $OLD_KEEP ){
115 debug("monitor $id down too long. removing from report");
116 delete $MON{$id};
117 }
118 }
119  
120 sub update {
121 my $id = shift;
122 my $gb = shift; # ACPHeartbeat
123  
124 my $up;
125 eval {
126 $up = ACPHeartBeat->decode( $gb );
127 $up->{id} = $id;
128 };
129 if(my $e = $@){
130 problem("cannot decode hb data: $e");
131 return isdown($id, 0);
132 }
133 unless( $up->{status_code} == 200 ){
134 return isdown($id, $up->{status_code});
135 }
136 return isdown($id, 0) unless $^T - $up->{timestamp} < $OLD_DOWN;
137  
138 debug("monitor $id is up");
139 $up->{lastup} = $^T;
140 $up->{downcount} = 0;
141  
142 _hb_ip_info( $up, $MON{$id} );
143 $MON{$id} = $up;
144 }
145  
146 sub _hb_ip_info {
147 my $up = shift;
148 my $old = shift;
149  
150 my $ip;
151  
152 $ip = $old->{ip} if ($old->{process_id} == $up->{process_id}) && ($old->{server_id} eq $up->{server_id});
153  
154 unless( $ip ){
155 my $port = $up->{port};
156 unless( $port ){
157 # use monitored port (id is from config)
158 (undef, $port) = split /:/, $up->{id};
159 }
160  
161 if( $up->{ip} ){
162 $ip = [ { ipv4 => $up->{ip}, port => $port, natdom => undef } ];
163 }else{
164 my $mynat = my_network_info();
165  
166 for my $i ( @$mynat ){
167 push @$ip, { ipv4 => $i->{ipi}, port => $port, natdom => $i->{natdom} };
168 }
169 }
170 }
171  
172 $up->{ip} = $ip;
173 }
174  
175 1;