MrGamoo/Job.pm untested

filter: Statementstmt 0.0% 0 / 249 Branchbran 0.0% 0 / 62 Conditioncond 0.0% 0 / 58 MC/DCmcdc 0.0% 0 / 29 Subroutinesub 0.0% 0 / 32 Podpod 0.0% 0 / 14 totaltotal 0.0% 0 / 444 CC 31 SCAR 36.2
CC
31
Cov
0%
_try_to_do_something
156.0
new
20.0
_periodic
20.0
CRAP 37.2
1 # -*- perl -*-
2  
3 # Copyright (c) 2010 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2010-Jan-13 13:03 (EST)
6 # Function: m/r jobs we are controlling
7 #
8 # $Id: Job.pm,v 1.6 2011/01/18 18:00:23 jaw Exp $
9  
10 package AC::MrGamoo::Job;
11 use AC::MrGamoo::Debug 'job';
12 use AC::MrGamoo::Config;
13 use AC::MrGamoo::Job::Plan;
14 use AC::MrGamoo::Job::RePlan;
15 use AC::MrGamoo::Job::Util;
16 use AC::MrGamoo::Job::Done;
17 use AC::MrGamoo::Job::Info;
18 use AC::MrGamoo::Job::Action;
19 use AC::MrGamoo::FileList;
20 use AC::MrGamoo::PeerList;
21 use AC::MrGamoo::MySelf;
22 use AC::MrGamoo::EUConsole;
23 use AC::MrGamoo::Stats;
24 use AC::DC::IO;
25 use AC::Misc;
26 use JSON;
27 use Time::HiRes 'time';
28 use strict;
29  
30 # RSN - config? tune?
31 our $TASKTIMEOUT = 10;
32 our $XFERTIMEOUT = 10;
33 our $TASKSRVRMAX = 4;
34 our $XFERSRVRMAX = 4;
35 our $REQMAX = 10;
36 our $MAXLOAD = 0.5;
37  
38 our %REGISTRY;
39 our $MSGID = $$;
40 my $_trying;
41  
42 our $MAXFILE = `sh -c "ulimit -n"`;
43 $MAXFILE = 255 if $^O eq 'solaris' && $MAXFILE > 255;
44  
45 ################################################################
46  
47 # schedule periodic "cronjob"
48 AC::DC::Sched->new(
49 info => "job periodic",
50 freq => 2,
51 func => \&periodic,
52 );
53  
54 ################################################################
55  
56 sub new {
57 my $class = shift;
58 # %{ APCMRMJobCreate }
59  
60 my $me = bless {
61 request => { @_ },
62 phase_no => -1,
63 file_info => {},
64 tmp_file => [],
65 server_info => {},
66 task_running => {},
67 task_pending => {},
68 xfer_running => {},
69 xfer_pending => {},
70 request_running => {},
71 request_pending => {},
72 statistics => { job_start => time() },
73 }, $class;
74  
75 if( $REGISTRY{ $me->{request}{jobid} } ){
76 verbose("ignoring duplicate request job $me->{request}{jobid}");
77 # will cause a 200 OK, so the requestor will not retry
78 return $REGISTRY{ $me->{request}{jobid} };
79 }
80  
81 verbose("new job: $me->{request}{jobid} ($me->{request}{traceinfo})");
82  
83 my $cf = $me->{options} = decode_json( $me->{request}{options} ) if $me->{request}{options};
84  
85 # open connection to eu-console
86 $me->{euconsole} = AC::MrGamoo::EUConsole->new( $me->{request}{jobid}, $me->{request}{console} );
87  
88 # partially compile
89 eval {
90 $me->{mr} = AC::MrGamoo::Submit::Compile->new( text => $me->{request}{jobsrc} );
91 };
92 if(my $e = $@){
93 problem("cannot compile job: $e");
94 return;
95 }
96  
97 # RSN - get_file_list + Plan may take too long - do in sub-process
98  
99 # get file list
100 my $files = get_file_list( $cf );
101 #print STDERR "files: ", dumper($files), "\n";
102  
103 for my $f (@$files){
104 $me->{file_info}{ $f->{filename} } = $f;
105 }
106  
107 # get server list
108 my $servers = get_peer_list( $cf );
109 #print STDERR "servers: ", dumper($servers), "\n";
110  
111 # plan job
112 my $plan = AC::MrGamoo::Job::Plan->new( $me, $servers, $files );
113 #print STDERR "plan: ", dumper($plan), "\n";
114  
115 $me->{plan} = $plan;
116  
117 $me->{maxfail} = 5 * ( (keys %{$plan->{taskidx}}) + @{$plan->{copying}});
118  
119 $me->{server_info}{$_->{id}} = {} for @$servers;
120  
121 $me->_preload_file_copies();
122 $REGISTRY{ $me->{request}{jobid} } = $me;
123 return $me;
124 }
125  
126 sub start {
127 my $me = shift;
128  
129 debug("start job");
130 $me->{euconsole}->send_msg('debug', 'starting job');
131 $me->_try_to_do_something();
132 1;
133 }
134  
135 ################################################################
136  
137 # record status rcvd from task
138 sub task_status {
139 my $me = _find(shift, @_);
140 my %p = @_;
141 my $taskid = $p{taskid};
142  
143 return unless $me;
144 my $t = $me->{task_running}{$taskid};
145 return unless $t;
146  
147 $t->update_status( $me, $p{phase}, $p{progress} );
148  
149 1;
150 }
151  
152 # record status rcvd from file xfer
153 sub xfer_status {
154 my $me = _find(shift, @_);
155 my %p = @_;
156 my $copyid = $p{copyid};
157  
158 return unless $me;
159 my $c = $me->{xfer_running}{$copyid};
160 return unless $c;
161  
162 $c->update_status( $me, $p{status_code} );
163  
164 1;
165 }
166  
167 ################################################################
168  
169 sub periodic {
170 # debug("periodic check");
171  
172 $_trying = 0;
173 for my $job (values %REGISTRY){
174 $job->_periodic();
175 }
176 }
177  
178 sub _periodic {
179 my $me = shift;
180  
181 my @t = values %{$me->{task_running}};
182 for my $t ( @t ){
183 my $lt = $t->{status_time} || $t->{start_time};
184 next if $^T - $lt < ($me->{options}{tasktimeout} || $TASKTIMEOUT);
185  
186 $t->failed( $me, 'timeout' );
187 }
188  
189 my @c = values %{$me->{xfer_running}};
190 for my $c ( @c ){
191 my $lt = $c->{status_time} || $c->{start_time};
192 next if $^T - $lt < ($me->{options}{xfertimeout} || $XFERTIMEOUT);
193  
194 $c->failed( $me, 'timeout' );
195 }
196  
197 $me->_try_to_do_something();
198  
199 my $tr = keys %{ $me->{task_running} };
200 my $tp = keys %{ $me->{task_pending} };
201 my $xr = keys %{ $me->{xfer_running} };
202 my $xp = keys %{ $me->{xfer_pending} };
203 my $rr = keys %{ $me->{request_running} };
204 my $rp = keys %{ $me->{request_pending} };
205  
206 my $ph = $me->{plan}{phases}[ $me->{phase_no} ] || 'none';
207  
208 debug("status: phase $ph, task $tr / $tp, xfer $xr / $xp, reqs $rr / $rp")
209 if $tr || $tp || $xr || $xp || $rr || $rp;
210  
211 }
212  
213 ################################################################
214  
215 sub _start_next_phase {
216 my $me = shift;
217  
218 debug("next phase");
219 $me->{phase_no} ++;
220 my $tp = $me->{plan}{taskplan}[ $me->{phase_no} ];
221  
222 # finished all phases ?
223 unless( $tp ){
224 if( $me->{_cleanedup} ){
225 $me->_finished();
226 return 1;
227 }else{
228 $me->_cleanup_files();
229 return;
230 }
231 }
232  
233 debug("job $me->{request}{jobid} starting phase $tp->{phase}");
234 $me->{euconsole}->send_msg('debug', "starting phase $tp->{phase}");
235  
236 # move tasks to pending
237 for my $t (@{$tp->{task}}){
238 $t->pend($me);
239 }
240 }
241  
242 sub _maybe_start_task {
243 my $me = shift;
244 my $task = shift;
245  
246 #debug("maybe start task");
247  
248 my $server = $task->{server};
249 my $underway = keys %{$me->{server_info}{$server}{task_running}};
250 return if $underway >= $TASKSRVRMAX; # don't overload server
251 return if $task->{start_after} > $^T; # rate limit retries
252  
253 # RSN - check that prerequisite xfers completed
254 $task->start( $me );
255 return 1;
256 }
257  
258 sub _maybe_start_xfer {
259 my $me = shift;
260 my $copy = shift;
261  
262 #debug("maybe start xfer");
263  
264 my $server = $copy->{server};
265 my $underway = keys %{$me->{server_info}{$server}{xfer_running}};
266 return if $underway >= $XFERSRVRMAX; # don't overload server
267 return if $copy->{start_after} > $^T; # rate limit retries
268  
269 $copy->start( $me );
270 return 1;
271 }
272  
273 sub _maybe_start_request {
274 my $me = shift;
275 my $req = shift;
276  
277 $req->start( $me );
278 return 1;
279 }
280  
281 ################################################################
282  
283 sub _preload_file_copies {
284 my $me = shift;
285  
286 # start copying files
287  
288 for my $c ( @{$me->{plan}{copying}} ){
289 $c->pend($me);
290 }
291 }
292  
293  
294 ################################################################
295  
296 sub _try_to_do_something {
297 my $me = shift;
298  
299 # debug("try something");
300  
301 return if $me->{_finished};
302 return if $_trying;
303  
304 # is this phase done
305 if( !(keys %{$me->{task_running}})
306 && !(keys %{$me->{task_pending}})
307 && !(keys %{$me->{xfer_running}})
308 && !(keys %{$me->{xfer_pending}})
309 && !(keys %{$me->{request_running}})
310 && !(keys %{$me->{request_pending}})
311 ){
312  
313 # this phase is finished
314 return if $me->_start_next_phase();
315 }
316  
317 if( $me->{aborted}
318 && !(keys %{$me->{request_running}})
319 && !(keys %{$me->{request_pending}})
320 ){
321 # this phase is finished
322 return if $me->_start_next_phase();
323 }
324  
325 # check load ave, etc
326 return unless $me->_ok_to_do_more_p();
327  
328 $_trying ++;
329  
330 # are there requests that can start
331 my @rp = values %{$me->{request_pending}};
332 my $startreqs = $REQMAX - keys %{$me->{request_running}};
333 for my $r (@rp){
334 last if $startreqs <= 0;
335 next unless $me->_maybe_start_request( $r );
336 $startreqs --;
337 }
338  
339 unless( $me->{aborted} ){
340  
341 # are there tasks that can start
342 my $started = 0;
343 my @tp = sort { $a->{created} <=> $b->{created} } values %{$me->{task_pending}};
344 for my $t (@tp){
345 $started += $me->_maybe_start_task( $t );
346 last if $started >= $me->{plan}{nserver} / 4; # keep them from getting in to lockstep
347 }
348  
349 # are there copies that can start
350 my @cp = sort { $a->{created} <=> $b->{created} } values %{$me->{xfer_pending}};
351 for my $c (@cp){
352 $me->_maybe_start_xfer( $c );
353 }
354 }
355  
356 # should we speculatively copy some files
357  
358 # should we speculatively retry a task
359  
360  
361 $_trying --;
362 }
363  
364 ################################################################
365  
366 sub something_failed {
367 my $me = shift;
368  
369 return if ++$me->{total_fails} < $me->{maxfail};
370 $me->abort();
371 return 1;
372 }
373  
374 ################################################################
375  
376 sub report {
377  
378 my $txt;
379  
380 for my $j (values %REGISTRY){
381 my $ph;
382 $ph = 'start' if $j->{phase_no} < 0;
383 $ph ||= 'cleanup' if $j->{phase_no} >= @{$j->{plan}{phases}};
384 $ph ||= $j->{plan}{phases}[ $j->{phase_no} ];
385  
386 my $tr = keys %{$j->{task_running}};
387 my $tp = keys %{$j->{task_pending}};
388 my $cr = keys %{$j->{copy_running}};
389 my $cp = keys %{$j->{copy_pending}};
390 my $rr = keys %{$j->{request_running}};
391 my $rp = keys %{$j->{request_pending}};
392  
393 $txt .= sprintf("%s %8s %4d %4d %4d %4d %4d %4d\n", $j->{request}{jobid}, $ph, $tr, $tp, $cr, $cp, $rr, $rp);
394 # ...
395 }
396  
397 return $txt;
398 }
399  
400  
401 1;