| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2009 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2009-Apr-01 18:56 (EDT) | ||
| 6 | # Function: distribute data to other peers | ||
| 7 | # | ||
| 8 | # $Id$ | ||
| 9 | |||
| 10 | package AC::Yenta::Store::Distrib; | ||
| 11 | use AC::Yenta::Kibitz::Store::Client; | ||
| 12 | use AC::Yenta::Debug 'distrib'; | ||
| 13 | use AC::Yenta::Config; | ||
| 14 | use AC::Yenta::Protocol; | ||
| 15 | use AC::Yenta::Stats; | ||
| 16 | use AC::Yenta::MySelf; | ||
| 17 | use AC::Misc; | ||
| 18 | use AC::DC::Sched; | ||
| 19 | use strict; | ||
| 20 | |||
| 21 | my $MAXHOP = 10; | ||
| 22 | my $MAXFARSEE = 2; | ||
| 23 | my $MAXNEARSEE = 3; | ||
| 24 | my $FARSENDS = 1; | ||
| 25 | my $NEARSENDS = 2; | ||
| 26 | my $MAXUNDERWAY = 64; | ||
| 27 | |||
| 28 | my $msgid = $$; | ||
| 29 | my @DIST; | ||
| 30 | |||
| 31 | AC::DC::Sched->new( | ||
| 32 | info => 'distribution', | ||
| 33 | freq => 5, | ||
| 34 | func => \&AC::Yenta::Store::Distrib::periodic, | ||
| 35 | ); | ||
| 36 | |||
| 37 | sub new { | ||
| 38 | my $class = shift; | ||
| 39 | my $req = shift; | ||
| 40 | my $cont = shift; | ||
| 41 | |||
| 42 | return if $req->{hop} >= $MAXHOP; | ||
| 43 | return if $req->{expire} < $^T; | ||
| 44 | |||
| 45 | my $sender = $req->{sender}; | ||
| 46 | my $sendat = AC::Yenta::Status->peer($sender); | ||
| 47 | |||
| 48 | my $me = bless { | ||
| 49 | info => "$req->{datum}{map}/$req->{datum}{key}/$req->{datum}{version}", | ||
| 50 | map => $req->{datum}{map}, | ||
| 51 | req => $req, | ||
| 52 | content => $cont, | ||
| 53 | # we tune the distribution algorithm based on where it came from: | ||
| 54 | faraway => (my_datacenter() ne $sendat->{datacenter}), | ||
| 55 | |||
| 56 | farseen => 0, | ||
| 57 | nearseen => 0, | ||
| 58 | farsend => [], | ||
| 59 | nearsend => [], | ||
| 60 | ordershift => 4, | ||
| 61 | }, $class; | ||
| 62 | |||
| 63 | debug("distributing $me->{info}"); | ||
| 64 | inc_stat( 'dist_requests' ); | ||
| 65 | inc_stat( 'dist_requests_faraway' ) if $me->{faraway}; | ||
| 66 | |||
| 67 | |||
| 68 | $me->_init_strategy($sender); | ||
| 69 | |||
| 70 | # RSN - check load | ||
| 71 | my $max = conf_value('distrib_max') || $MAXUNDERWAY; | ||
| 72 | if( @DIST < $max ){ | ||
| 73 | $me->_start_next(); | ||
| 74 | } | ||
| 75 | push @DIST, $me; | ||
| 76 | |||
| 77 | return $me; | ||
| 78 | } | ||
| 79 | |||
| 80 | # periodically, go through and restart or expire | ||
| 81 | sub periodic { | ||
| 82 | |||
| 83 | my @keep; | ||
| 84 | my $max = conf_value('distrib_max') || $MAXUNDERWAY; | ||
| 85 | |||
| 86 | my $chance = (@DIST > $max) ? ($max / @DIST) : 1; | ||
| 87 | |||
| 88 | for my $r (@DIST){ | ||
| 89 | # debug("periodic $r->{info}"); | ||
| 90 | next if $^T > $r->{req}{expire}; | ||
| 91 | |||
| 92 | if( (rand() <= $chance) && (AC::DC::IO->underway() <= 2 * $max) ){ | ||
| 93 | my $keep = $r->_start_next(); | ||
| 94 | push @keep, $r if $keep; | ||
| 95 | }else{ | ||
| 96 | push @keep, $r; | ||
| 97 | } | ||
| 98 | } | ||
| 99 | |||
| 100 | @DIST = @keep; | ||
| 101 | } | ||
| 102 | |||
| 103 | ################################################################ | ||
| 104 | # determine distribution strategy | ||
| 105 | # - if we recvd it from faraway, we will send it to other datacenters, and randomly in the same datacenter | ||
| 106 | # - otherwise we send it in the same datacenter, in an orderly fashion | ||
| 107 | # RSN - find an strategy with faster convergence + less duplication | ||
| 108 | |||
| 109 | sub _init_strategy { | ||
| 110 | my $me = shift; | ||
| 111 | my $sender = shift; | ||
| 112 | |||
| 113 | my $here = my_datacenter(); | ||
| 114 | my $dcs = AC::Yenta::Status->datacenters(); | ||
| 115 | my $sendat = AC::Yenta::Status->peer($sender); | ||
| 116 | my(@far, @near); | ||
| 117 | |||
| 118 | for my $dc (keys %$dcs){ | ||
| 119 | if( $dc eq $here ){ | ||
| 120 | push @near, grep { $_ ne $sender } $me->_compat_peers_in_dc($dc); | ||
| 121 | }else{ | ||
| 122 | next if $dc eq $sendat->{datacenter}; | ||
| 123 | |||
| 124 | push @far, { | ||
| 125 | dc => $dc, | ||
| 126 | id => [ $me->_compat_peers_in_dc($dc) ], | ||
| 127 | }; | ||
| 128 | } | ||
| 129 | } | ||
| 130 | |||
| 131 | if( $me->{faraway} ){ | ||
| 132 | $me->{nearsend} = shuffle(\@near); | ||
| 133 | $me->{farsend} = shuffle(\@far); | ||
| 134 | }else{ | ||
| 135 | $me->{nearsend} = _orderly(\@near); | ||
| 136 | } | ||
| 137 | } | ||
| 138 | |||
| 139 | # which yentas can do something with the update? | ||
| 140 | sub _compat_peers_in_dc { | ||
| 141 | my $me = shift; | ||
| 142 | my $dc = shift; | ||
| 143 | |||
| 144 | my $env = conf_value('environment'); | ||
| 145 | my $dcs = AC::Yenta::Status->datacenters(); | ||
| 146 | my $map = $me->{map}; | ||
| 147 | my @id; | ||
| 148 | |||
| 149 | for my $id (keys %{$dcs->{$dc}}){ | ||
| 150 | my $pd = AC::Yenta::Status->peer($id); | ||
| 151 | |||
| 152 | next unless $pd->{subsystem} eq 'yenta'; | ||
| 153 | next unless $pd->{environment} eq $env; | ||
| 154 | next unless grep {$map eq $_} @{ $pd->{map} }; | ||
| 155 | push @id, $id; | ||
| 156 | } | ||
| 157 | return @id; | ||
| 158 | } | ||
| 159 | |||
| 160 | sub _start_far { | ||
| 161 | my $me = shift; | ||
| 162 | |||
| 163 | my $d = shift @{ $me->{farsend} }; | ||
| 164 | return unless $d; | ||
| 165 | |||
| 166 | # randomly pick one server in chosen dc | ||
| 167 | my @id = grep { | ||
| 168 | my $x = AC::Yenta::Status->peer($_); | ||
| 169 | ($x->{status} == 200) ? 1 : 0; | ||
| 170 | } @{$d->{id}}; | ||
| 171 | return unless @id; | ||
| 172 | |||
| 173 | my $id = $id[ rand(@id) ]; | ||
| 174 | debug("sending $me->{info} to far site $id in $d->{dc}"); | ||
| 175 | $me->_start_peer( $id, 1 ); | ||
| 176 | inc_stat('dist_send_far'); | ||
| 177 | inc_stat('dist_send_total'); | ||
| 178 | return 1; | ||
| 179 | } | ||
| 180 | |||
| 181 | sub _start_near { | ||
| 182 | my $me = shift; | ||
| 183 | |||
| 184 | my $id = shift @{ $me->{nearsend} }; | ||
| 185 | return unless $id; | ||
| 186 | debug("sending $me->{info} to nearby site $id"); | ||
| 187 | $me->_start_peer( $id, 0 ); | ||
| 188 | inc_stat('dist_send_near'); | ||
| 189 | inc_stat('dist_send_total'); | ||
| 190 | return 1; | ||
| 191 | } | ||
| 192 | |||
| 193 | sub _start_next { | ||
| 194 | my $me = shift; | ||
| 195 | |||
| 196 | my $sent; | ||
| 197 | |||
| 198 | # pick next peers | ||
| 199 | # start clients | ||
| 200 | |||
| 201 | if( $me->{faraway} ){ | ||
| 202 | if( $me->{farseen} < $MAXFARSEE ){ | ||
| 203 | for (1 .. $FARSENDS){ | ||
| 204 | $sent ++ if $me->_start_far(); | ||
| 205 | } | ||
| 206 | } | ||
| 207 | if( $me->{nearseen} < $MAXNEARSEE ){ | ||
| 208 | for (1 .. $NEARSENDS){ | ||
| 209 | $sent ++ if $me->_start_near(); | ||
| 210 | } | ||
| 211 | } | ||
| 212 | }else{ | ||
| 213 | $sent ++ if $me->_start_near(); | ||
| 214 | } | ||
| 215 | |||
| 216 | return $sent; | ||
| 217 | } | ||
| 218 | |||
| 219 | sub _start_one { | ||
| 220 | my $me = shift; | ||
| 221 | my $far = shift; | ||
| 222 | |||
| 223 | if( $far ){ | ||
| 224 | return if $me->{farseen} >= $MAXFARSEE; | ||
| 225 | $me->_start_far(); | ||
| 226 | }else{ | ||
| 227 | return if $me->{nearseen} >= $MAXNEARSEE; | ||
| 228 | $me->_start_near(); | ||
| 229 | } | ||
| 230 | } | ||
| 231 | |||
| 232 | sub _start_peer { | ||
| 233 | my $me = shift; | ||
| 234 | my $id = shift; | ||
| 235 | my $far = shift; | ||
| 236 | |||
| 237 | my $pd = AC::Yenta::Status->peer($id); | ||
| 238 | my $addr = $pd->{ip}; # array of nat ip info | ||
| 239 | |||
| 240 | my $enc = use_encryption($pd); | ||
| 241 | my $ect = ''; | ||
| 242 | my $proto = AC::Yenta::Protocol->new( secret => conf_value('secret') ); | ||
| 243 | $ect = $enc ? $proto->encrypt(undef, ${$me->{content}}) : ${$me->{content}} if $me->{content}; | ||
| 244 | |||
| 245 | # build request | ||
| 246 | my $request = $proto->encode_request( { | ||
| 247 | type => 'yenta_distrib', | ||
| 248 | msgidno => $msgid++, | ||
| 249 | want_reply => 1, | ||
| 250 | data_encrypted => $enc, | ||
| 251 | content_encrypted => $enc, | ||
| 252 | }, { | ||
| 253 | sender => AC::Yenta::Status->my_server_id(), | ||
| 254 | hop => $me->{req}{hop} + 1, | ||
| 255 | expire => $me->{req}{expire}, | ||
| 256 | datum => $me->{req}{datum}, | ||
| 257 | }, \$ect ); | ||
| 258 | |||
| 259 | # connect + send | ||
| 260 | my $io = AC::Yenta::Kibitz::Store::Client->new($addr, undef, | ||
| 261 | $request . $ect, | ||
| 262 | info => "distrib $me->{info} to $id", | ||
| 263 | ); | ||
| 264 | |||
| 265 | if( $io ){ | ||
| 266 | $io->set_callback('load', \&_onload, $me, $id, $far); | ||
| 267 | $io->set_callback('error', \&_onerror, $me, $id, $far); | ||
| 268 | $io->start(); | ||
| 269 | }else{ | ||
| 270 | debug("start client failed"); | ||
| 271 | } | ||
| 272 | } | ||
| 273 | |||
| 274 | sub _onload { | ||
| 275 | my $io = shift; | ||
| 276 | my $evt = shift; | ||
| 277 | my $me = shift; | ||
| 278 | my $id = shift; | ||
| 279 | my $far = shift; | ||
| 280 | |||
| 281 | debug("dist finish $me->{info} with $id => $evt->{data}{haveit}"); | ||
| 282 | |||
| 283 | if( $evt->{data}{haveit} ){ | ||
| 284 | if( $far ){ | ||
| 285 | $me->{farseen} ++; | ||
| 286 | inc_stat('dist_send_far_seen'); | ||
| 287 | }else{ | ||
| 288 | $me->{nearseen} ++; | ||
| 289 | inc_stat('dist_send_near_seen'); | ||
| 290 | } | ||
| 291 | } | ||
| 292 | |||
| 293 | if( !$me->{faraway} && !$far ){ | ||
| 294 | # orderly distribution. hop away. | ||
| 295 | if( $evt->{data}{haveit} ){ | ||
| 296 | shift @{$me->{nearsend}}; | ||
| 297 | }else{ | ||
| 298 | my $n = $me->{ordershift}; | ||
| 299 | $n = @{$me->{nearsend}} / 2 if $n > @{$me->{nearsend}} / 2; | ||
| 300 | shift @{$me->{nearsend}} for (1 .. $n); | ||
| 301 | $me->{ordershift} *= 2; | ||
| 302 | } | ||
| 303 | } | ||
| 304 | |||
| 305 | $me->_start_one($far); | ||
| 306 | } | ||
| 307 | |||
| 308 | sub _onerror { | ||
| 309 | my $io = shift; | ||
| 310 | my $evt = shift; | ||
| 311 | my $me = shift; | ||
| 312 | my $id = shift; | ||
| 313 | my $far = shift; | ||
| 314 | |||
| 315 | verbose("error distributing $me->{info} to $id"); | ||
| 316 | # don't need to track anything | ||
| 317 | |||
| 318 | $me->_start_one($far); | ||
| 319 | } | ||
| 320 | |||
| 321 | sub _orderly { | ||
| 322 | my $peers = shift; | ||
| 323 | |||
| 324 | my $myself = AC::Yenta::Status->my_server_id(); | ||
| 325 | my @p = sort {$a cmp $b} @$peers; | ||
| 326 | |||
| 327 | my @left = grep { $_ lt $myself } @p; | ||
| 328 | my @right = grep { $_ gt $myself } @p; | ||
| 329 | |||
| 330 | @p = (@right, @left); | ||
| 331 | return \@p; | ||
| 332 | } | ||
| 333 | |||
| 334 | 1; |