MrGamoo/Client.pm untested

filter: Statementstmt 0.0% 0 / 162 Branchbran 0.0% 0 / 30 Conditioncond 0.0% 0 / 11 MC/DCmcdc 0.0% 0 / 7 Subroutinesub 0.0% 0 / 25 Podpod 0.0% 0 / 11 totaltotal 0.0% 0 / 246 CC 16 SCAR 24.8
CC
16
Cov
0%
run_console
30.0
submit
30.0
_pick_master_and_send
20.0
CRAP 11.9
1 # -*- perl -*-
2  
3 # Copyright (c) 2010 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2010-Jan-25 16:32 (EST)
6 # Function: m/r client
7 #
8 # $Id: Client.pm,v 1.5 2011/01/18 17:19:12 jaw Exp $
9  
10 package AC::MrGamoo::Client;
11 use AC::MrGamoo::Submit::Compile;
12 use AC::MrGamoo::Submit::Request;
13 use AC::MrGamoo::Debug;
14 use AC::Daemon;
15 use AC::Conf;
16 use AC::Misc;
17 use AC::MrGamoo::Protocol;
18 use JSON;
19 use Sys::Hostname;
20 use Socket;
21  
22 require 'AC/protobuf/mrgamoo.pl';
23 require 'AC/protobuf/mrgamoo_status.pl';
24 require 'AC/protobuf/std_reply.pl';
25 use strict;
26  
27  
28 sub new {
29 my $class = shift;
30 my $from = shift; # file | text
31 my $src = shift;
32 my $cfg = shift;
33  
34 my $host = hostname();
35 my $user = getpwuid($<);
36 my $trace = "$user/$$\@$host:" . ($from eq 'file' ? $src : 'text');
37  
38 my $me = bless {
39 traceinfo => $trace,
40 }, $class;
41 $me->{fdebug} = $cfg->{debug} ? sub{ print STDERR "@_\n" } : sub {};
42  
43 # compile job
44 my $mr = AC::MrGamoo::Submit::Compile->new( $from => $src );
45 $me->{program} = $mr;
46  
47 # merge job %config section with passed in config
48 $mr->add_config($cfg);
49  
50 return $me;
51 }
52  
53 sub get_config_param {
54 my $me = shift;
55  
56 $me->{program}->get_config_param(@_);
57 }
58  
59 sub set_config_param {
60 my $me = shift;
61  
62 $me->{program}->set_config_param(@_);
63 }
64  
65 sub open_console {
66 my $me = shift;
67  
68 my $fd;
69 socket($fd, PF_INET, SOCK_DGRAM, 0);
70 bind($fd, sockaddr_in(0, INADDR_ANY));
71 my $s = getsockname($fd);
72 my($port, $addr) = sockaddr_in($s);
73  
74 $me->{console_fd} = $fd;
75 $me->{console_port} = $port;
76 }
77  
78 sub run_console {
79 my $me = shift;
80 my $fd = $me->{console_fd};
81  
82 while(1){
83 my $buf;
84 recv $fd, $buf, 65535, 0;
85 my $proto = AC::MrGamoo::Protocol->decode_header($buf);
86 my $data = substr($buf, AC::MrGamoo::Protocol->header_size());
87 my $req = AC::MrGamoo::Protocol->decode_request($proto, $data);
88 last if $req->{type} eq 'finish';
89 print STDERR "$req->{msg}" if $req->{type} eq 'stderr';
90 print "$req->{msg}" if $req->{type} eq 'stdout';
91 $me->{fdebug}->("$req->{server_id}\t$req->{msg}") if $req->{type} eq 'debug';
92 }
93 }
94  
95 sub submit {
96 my $me = shift;
97 my $seed = shift; # [ "ipaddr:port", ... ]
98  
99 my $mr = $me->{program};
100 my $r = AC::MrGamoo::Submit::Request->new( $mr );
101 $r->{eu_print_stderr} = sub { print STDERR "@_\n" };
102 $r->{eu_print_stdout} = sub { print STDERR "@_\n" };
103  
104 # run init section
105 my $h_init = $mr->get_code( 'init' );
106 my $initres = ($h_init ? $h_init->{code}() : undef) || {};
107  
108 $me->{id} = unique();
109 my $req = AC::MrGamoo::Protocol->encode_request( {
110 type => 'mrgamoo_jobcreate',
111 msgidno => $^T,
112 want_reply => 1,
113 },{
114 jobid => $me->{id},
115 options => to_json( $r->{config} ),
116 initres => to_json( $initres, {allow_nonref => 1} ),
117 jobsrc => $mr->src(),
118 console => ($me->{console_port} ? ":$me->{console_port}" : ''),
119 traceinfo => $me->{traceinfo},
120 } );
121  
122 my $ok;
123 if( my $master = $me->get_config_param('master') ){
124 # use specified master (for debugging)
125 my($addr, $port) = split /:/, $master;
126 $me->_submit_to( $addr, $port, $req );
127 $me->{master} = { addr => $addr, port => $port };
128 $ok = 1;
129 }else{
130 # pick server
131 $ok = $me->_pick_master_and_send( $req, $seed );
132 }
133  
134 return $ok ? $me->{id} : undef;
135 }
136  
137 sub abort {
138 my $me = shift;
139  
140 return unless $me->{master};
141 my $res = $me->_submit_to( $me->{master}{addr}, $me->{master}{port}, AC::MrGamoo::Protocol->encode_request( {
142 type => 'mrgamoo_jobabort',
143 msgidno => $^T,
144 want_reply => 1,
145 }, {
146 jobid => $me->{id},
147 }));
148  
149 }
150  
151 ################################################################
152  
153 sub _pick_master_and_send {
154 my $me = shift;
155 my $req = shift;
156 my $seed = shift;
157  
158 my @serverlist;
159  
160 my $listreq = AC::MrGamoo::Protocol->encode_request( {
161 type => 'mrgamoo_status',
162 msgidno => $^T,
163 want_reply => 1,
164 }, {});
165  
166 # get the full list of servers
167 # contact each seed passed in above, until we get a reply
168 for my $s ( @$seed ){
169 my($addr, $port) = split /:/, $s;
170 $me->{fdebug}->("attempting to fetch server list from $addr:$port");
171 eval {
172 alarm(1);
173 my $reply = AC::MrGamoo::Protocol->send_request( inet_aton($addr), $port, $listreq, $me->{fdebug} );
174 my $res = AC::MrGamoo::Protocol->decode_reply($reply);
175 alarm(0);
176 my $list = $res->{status};
177 @serverlist = @$list if $list && @$list;
178 };
179 last if @serverlist;
180 }
181  
182 # sort+filter list
183 @serverlist = sort { ($a->{sort_metric} <=> $b->{sort_metric}) || int(rand(3)) - 1 }
184 grep { $_->{status} == 200 } @serverlist;
185  
186 # try all addresses
187 # RSN - sort addresslist in a Peers::pick_best_addr_for_peer() like manner?
188  
189 my @addrlist = map { @{$_->{ip}} } @serverlist;
190  
191 for my $ip (@addrlist){
192 my $addr = inet_itoa($ip->{ipv4});
193 my $res;
194 eval {
195 alarm(30);
196 $res = $me->_submit_to( $addr, $ip->{port}, $req );
197 alarm(0);
198 };
199 next unless $res && $res->{status_code} == 200;
200 $me->{master} = { addr => $addr, port => $ip->{port} };
201 return 1;
202 }
203 return ;
204 }
205  
206 sub _submit_to {
207 my $me = shift;
208 my $addr = shift;
209 my $port = shift;
210 my $req = shift;
211  
212 $me->{fdebug}->("sending job to $addr:$port");
213 my $reply = AC::MrGamoo::Protocol->send_request( inet_aton($addr), $port, $req, $me->{fdebug}, 120 );
214 my $res = AC::MrGamoo::Protocol->decode_reply($reply);
215  
216 return $res;
217 }
218  
219 ################################################################
220  
221 sub check_code {
222 my $me = shift;
223  
224 my $mr = $me->{program};
225 my $nr = @{ $mr->{content}{reduce} };
226  
227 $me->_check('map');
228 $me->_check('reduce', $_) for (0 .. $nr - 1);
229 $me->_check('final');
230  
231 return 1;
232 }
233  
234 sub _check {
235 my $me = shift;
236 my $mr = $me->{program};
237  
238 my $prog = $mr->compile(@_);
239 eval "sub $prog";
240 die $@ if $@;
241 }
242  
243  
244 1;
245