| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2010 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2010-Jan-22 15:14 (EST) | ||
| 6 | # Function: | ||
| 7 | # | ||
| 8 | # $Id: Peers.pm,v 1.1 2010/11/01 18:41:59 jaw Exp $ | ||
| 9 | |||
| 10 | package AC::MrGamoo::Kibitz::Peers; | ||
| 11 | use AC::MrGamoo::Debug 'kibitz_peers'; | ||
| 12 | use AC::MrGamoo::About; | ||
| 13 | use AC::MrGamoo::MySelf; | ||
| 14 | use AC::MrGamoo::Config; | ||
| 15 | use AC::DC::Sched; | ||
| 16 | use AC::Misc; | ||
| 17 | use AC::Import; | ||
| 18 | use JSON; | ||
| 19 | use strict; | ||
| 20 | |||
| 21 | our @EXPORT = qw(pick_best_addr_for_peer peer_list_all get_peer_by_id); | ||
| 22 | |||
| 23 | my $KEEPDOWN = 300; # keep data about down servers for how long? | ||
| 24 | my $KEEPLOST = 600; # keep data about servers we have not heard about for how long? | ||
| 25 | |||
| 26 | my %SCEPTICAL; | ||
| 27 | my %ALLPEER; | ||
| 28 | my %MAYBEDOWN; | ||
| 29 | my $natdom; | ||
| 30 | my $natinit; | ||
| 31 | |||
| 32 | AC::DC::Sched->new( | ||
| 33 | info => 'kibitz status', | ||
| 34 | freq => (conf_value('time_status_kibitz') || 5), | ||
| 35 | func => \&periodic, | ||
| 36 | ); | ||
| 37 | |||
| 38 | ################################################################ | ||
| 39 | |||
| 40 | sub periodic { | ||
| 41 | |||
| 42 | # clean up down or lost peers | ||
| 43 | for my $id ( keys %ALLPEER ){ | ||
| 44 | my $p = $ALLPEER{$id}; | ||
| 45 | next unless $p; | ||
| 46 | |||
| 47 | next if $p->{status} == 200 && $p->{timestamp} > $^T - $KEEPLOST; | ||
| 48 | _maybe_remove( $id ); | ||
| 49 | } | ||
| 50 | |||
| 51 | _kibitz_with_random_peer(); | ||
| 52 | |||
| 53 | } | ||
| 54 | |||
| 55 | ################################################################ | ||
| 56 | |||
| 57 | sub update_sceptical { | ||
| 58 | my $class = shift; | ||
| 59 | my $up = shift; | ||
| 60 | |||
| 61 | return unless _update_ok($up); | ||
| 62 | my $id = $up->{server_id}; | ||
| 63 | return if $ALLPEER{$id}; | ||
| 64 | debug("recvd update (sceptical) from $id"); | ||
| 65 | $SCEPTICAL{$id} = $up; | ||
| 66 | } | ||
| 67 | |||
| 68 | sub update { | ||
| 69 | my $class = shift; | ||
| 70 | my $up = shift; | ||
| 71 | |||
| 72 | return unless _update_ok($up); | ||
| 73 | my $id = $up->{server_id}; | ||
| 74 | |||
| 75 | |||
| 76 | my $previnfo = $ALLPEER{$id}; | ||
| 77 | # only keep it if it is newer than what we have | ||
| 78 | return if $previnfo && $up->{timestamp} <= $previnfo->{timestamp}; | ||
| 79 | # only keep it if it is relatively fresh | ||
| 80 | return unless $up->{timestamp} > $^T - $KEEPDOWN; | ||
| 81 | |||
| 82 | $up->{path} .= ' ' . my_server_id(); | ||
| 83 | |||
| 84 | if( $previnfo ){ | ||
| 85 | verbose("marking peer $id as up") if $up->{status} == 200 && $previnfo->{status} != 200; | ||
| 86 | }else{ | ||
| 87 | verbose("discovered new peer: $id ($up->{hostname})"); | ||
| 88 | } | ||
| 89 | |||
| 90 | $ALLPEER{$id} = $up; | ||
| 91 | delete $SCEPTICAL{$id}; | ||
| 92 | |||
| 93 | if( $up->{status} != 200 ){ | ||
| 94 | _maybe_remove( $id ); | ||
| 95 | return ; | ||
| 96 | } | ||
| 97 | } | ||
| 98 | |||
| 99 | sub seems_ok { | ||
| 100 | my $class = shift; | ||
| 101 | my $id = shift; | ||
| 102 | |||
| 103 | delete $MAYBEDOWN{$id}; | ||
| 104 | } | ||
| 105 | |||
| 106 | # require 2 failures before declaring it down | ||
| 107 | sub maybe_down { | ||
| 108 | my $class = shift; | ||
| 109 | my $id = shift; | ||
| 110 | my $why = shift; | ||
| 111 | |||
| 112 | if( $MAYBEDOWN{$id} ){ | ||
| 113 | delete $MAYBEDOWN{$id}; | ||
| 114 | $class->isdown($id, $why); | ||
| 115 | return; | ||
| 116 | } | ||
| 117 | |||
| 118 | return $class->isdown($id, $why) unless $ALLPEER{$id}; | ||
| 119 | return $class->isdown($id, $why) unless $ALLPEER{$id}{status} == 200; | ||
| 120 | |||
| 121 | debug("peer '$id' might be down"); | ||
| 122 | $MAYBEDOWN{$id} = $ALLPEER{$id}; | ||
| 123 | } | ||
| 124 | |||
| 125 | sub isdown { | ||
| 126 | my $class = shift; | ||
| 127 | my $id = shift; | ||
| 128 | my $why = shift; | ||
| 129 | |||
| 130 | debug("peer '$id' is down"); | ||
| 131 | |||
| 132 | delete $SCEPTICAL{$id} if $SCEPTICAL{$id}; | ||
| 133 | return unless $ALLPEER{$id}; | ||
| 134 | |||
| 135 | verbose("marking peer $id as down $why") if $ALLPEER{$id}{status} == 200; | ||
| 136 | $ALLPEER{$id}{timestamp} = $^T; | ||
| 137 | $ALLPEER{$id}{status} = 0; | ||
| 138 | |||
| 139 | _maybe_remove( $id ); | ||
| 140 | } | ||
| 141 | |||
| 142 | sub peer_list_all { | ||
| 143 | |||
| 144 | return [ AC::MrGamoo::Kibitz->about_myself(), values %ALLPEER ]; | ||
| 145 | } | ||
| 146 | |||
| 147 | sub response { | ||
| 148 | return peer_list_all(); | ||
| 149 | } | ||
| 150 | |||
| 151 | sub get_peer_by_id { | ||
| 152 | my $id = shift; | ||
| 153 | |||
| 154 | return $ALLPEER{$id} if $ALLPEER{$id}; | ||
| 155 | return AC::MrGamoo::Kibitz->about_myself() if $id eq my_server_id(); | ||
| 156 | return ; | ||
| 157 | } | ||
| 158 | |||
| 159 | sub report { | ||
| 160 | |||
| 161 | my $all = peer_list_all(); | ||
| 162 | my $txt; | ||
| 163 | for my $p (@$all){ | ||
| 164 | my $lu = $^T - $p->{lastup}; | ||
| 165 | my $lh = $^T - $p->{timestamp}; | ||
| 166 | |||
| 167 | $txt .= sprintf("%-30s %s %s %s %3d %7.2f %d %d\n", | ||
| 168 | $p->{server_id}, $p->{subsystem}, $p->{environment}, | ||
| 169 | $p->{datacenter}, $p->{status}, $p->{sort_metric}, | ||
| 170 | $lu, $lh, | ||
| 171 | ); | ||
| 172 | } | ||
| 173 | return $txt; | ||
| 174 | } | ||
| 175 | |||
| 176 | sub report_json { | ||
| 177 | |||
| 178 | my $all = peer_list_all(); | ||
| 179 | my @fields = qw(hostname environment subsystem datacenter server_id status sort_metric); | ||
| 180 | |||
| 181 | return encode_json( [ map { | ||
| 182 | my %x; | ||
| 183 | @x{@fields} = @{$_}{@fields}; | ||
| 184 | $x{ip} = [ | ||
| 185 | map { { | ||
| 186 | ipv4 => inet_itoa($_->{ipv4}), | ||
| 187 | port => $_->{port}, | ||
| 188 | natdom => $_->{natdom}, | ||
| 189 | } } @{$_->{ip}} | ||
| 190 | ]; | ||
| 191 | \%x; | ||
| 192 | } @$all ] ) . "\n"; | ||
| 193 | } | ||
| 194 | |||
| 195 | ################################################################ | ||
| 196 | |||
| 197 | sub _update_ok { | ||
| 198 | my $up = shift; | ||
| 199 | |||
| 200 | my $myself = my_server_id(); | ||
| 201 | return if $up->{via} eq $myself; | ||
| 202 | return if $up->{server_id} eq $myself; | ||
| 203 | return if $up->{environment} ne conf_value('environment'); | ||
| 204 | return if $up->{subsystem} ne 'mrgamoo'; | ||
| 205 | return 1; | ||
| 206 | } | ||
| 207 | |||
| 208 | sub _remove { | ||
| 209 | my $id = shift; | ||
| 210 | |||
| 211 | my $d = $ALLPEER{$id}; | ||
| 212 | my $lu = $^T - $d->{lastup}; | ||
| 213 | my $lh = $^T - $d->{timestamp}; | ||
| 214 | verbose("deleting peer: $id ($lu, $lh)"); | ||
| 215 | delete $ALLPEER{$id}; | ||
| 216 | } | ||
| 217 | |||
| 218 | sub _maybe_remove { | ||
| 219 | my $id = shift; | ||
| 220 | |||
| 221 | my $d = $ALLPEER{$id}; | ||
| 222 | |||
| 223 | if( ($^T - $d->{lastup} > $KEEPDOWN) || ($^T - $d->{timestamp} > $KEEPLOST) ){ | ||
| 224 | _remove($id); | ||
| 225 | } | ||
| 226 | } | ||
| 227 | |||
| 228 | ################################################################ | ||
| 229 | |||
| 230 | # pick best ip addr from array (ACPIPPort) | ||
| 231 | sub pick_best_addr_for_peer { | ||
| 232 | my $ipinfo = shift; | ||
| 233 | |||
| 234 | _nat_init() unless $natinit; | ||
| 235 | |||
| 236 | my $public; | ||
| 237 | my $private; | ||
| 238 | |||
| 239 | for my $i ( @$ipinfo ){ | ||
| 240 | $public = $i unless $i->{natdom}; | ||
| 241 | $private = $i if $i->{natdom} eq $natdom; | ||
| 242 | } | ||
| 243 | |||
| 244 | # prefer private addr if available (cheaper) | ||
| 245 | my $prefer = $private || $public; | ||
| 246 | return unless $prefer; | ||
| 247 | |||
| 248 | return ( inet_itoa($prefer->{ipv4}), $prefer->{port} ); | ||
| 249 | } | ||
| 250 | |||
| 251 | sub _nat_init { | ||
| 252 | |||
| 253 | # determine my local NAT domain | ||
| 254 | my $nat = my_network_info(); | ||
| 255 | |||
| 256 | for my $i (@$nat){ | ||
| 257 | $natdom ||= $i->{natdom}; | ||
| 258 | } | ||
| 259 | $natinit = 1; | ||
| 260 | } | ||
| 261 | |||
| 262 | |||
| 263 | ################################################################ | ||
| 264 | |||
| 265 | sub _random_peer { | ||
| 266 | |||
| 267 | my @peer; | ||
| 268 | |||
| 269 | # might be down? try again. | ||
| 270 | @peer = values %MAYBEDOWN; | ||
| 271 | |||
| 272 | # sceptical | ||
| 273 | @peer = values %SCEPTICAL unless @peer; | ||
| 274 | |||
| 275 | # known peer | ||
| 276 | unless(@peer){ | ||
| 277 | @peer = values %ALLPEER; | ||
| 278 | # sometimes, randomly, use the seed peers | ||
| 279 | @peer = () unless int rand(@peer+1); | ||
| 280 | } | ||
| 281 | |||
| 282 | if( @peer ){ | ||
| 283 | my $p = $peer[ rand(@peer) ]; | ||
| 284 | debug("using peer $p->{server_id}"); | ||
| 285 | return ($p->{server_id}, pick_best_addr_for_peer($p->{ip})); | ||
| 286 | } | ||
| 287 | |||
| 288 | # seed peer | ||
| 289 | my $seed = conf_value('seedpeer'); | ||
| 290 | my $p = $seed->[ rand(@$seed) ]; | ||
| 291 | my ($ip, $port) = split /:/, $p; | ||
| 292 | $port ||= my_port(); | ||
| 293 | |||
| 294 | # don't talk to self. any of my addrs. | ||
| 295 | my $ipinfo = my_network_info(); | ||
| 296 | for my $i (@$ipinfo){ | ||
| 297 | return if $ip eq $i->{ipa} && $port == my_port(); | ||
| 298 | } | ||
| 299 | |||
| 300 | debug("using seedpeer"); | ||
| 301 | return("seed/$ip:$port", $ip, $port); | ||
| 302 | } | ||
| 303 | |||
| 304 | sub _kibitz_with_random_peer { | ||
| 305 | |||
| 306 | my( $id, $addr, $port ) = _random_peer(); | ||
| 307 | return unless $id; | ||
| 308 | debug("starting status kibitz client to $id"); | ||
| 309 | |||
| 310 | my $ok = AC::MrGamoo::Kibitz::Client->new( $addr, $port, | ||
| 311 | info => "status client: $id", | ||
| 312 | status_peer => $id, | ||
| 313 | ); | ||
| 314 | __PACKAGE__->maybe_down($id, 'connect') unless $ok; | ||
| 315 | } | ||
| 316 | |||
| 317 | 1; |