MrGamoo/Task/Running.pm untested

filter: Statementstmt 0.0% 0 / 239 Branchbran 0.0% 0 / 50 Conditioncond 0.0% 0 / 25 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 0.0% 0 / 31 Podpod 0.0% 0 / 16 totaltotal 0.0% 0 / 374 CC 26 SCAR 35.5
CC
26
Cov
0%
_do_map
132.0
_start_task
42.0
_do_reduce
20.0
CRAP 35.0
1 # -*- perl -*-
2  
3 # Copyright (c) 2010 AdCopy
4 # Author: Jeff Weisberg
5 # Created: 2010-Jan-13 16:45 (EST)
6 # Function: child half of m/r task (the actual running task)
7 #
8 # $Id: Running.pm,v 1.5 2011/01/12 19:18:54 jaw Exp $
9  
10 package AC::MrGamoo::Task::Running;
11 use AC::MrGamoo::Config;
12 use AC::MrGamoo::Debug 'task_run';
13 use AC::MrGamoo::Iter::File;
14 use AC::MrGamoo::EUConsole;
15 use AC::MrGamoo::ReadInput;
16 use AC::MrGamoo::OutFile;
17 use AC::MrGamoo::MySelf;
18 use AC::Daemon;
19 use Digest::SHA1 'sha1';
20 use Digest::MD5 'md5';
21 use File::Path;
22 use Sys::Syslog;
23 use Socket;
24 use JSON;
25 use strict;
26  
27 my $STATUSTIME = 5; # seconds
28 my $MAXRUN = 3600; # override with %attr maxrun
29 my $SORTPROG = '/usr/bin/sort'; # override with %attr sortprog or config file
30 my $GZPROG = '/usr/bin/gzcat'; # override with %attr gzprog or config file
31  
32 # in child process
33 sub _start_task {
34 my $me = shift;
35  
36 debug("start child task");
37 $^T = time();
38 _setup_stdio_etal();
39 _setup_console( $me );
40 _update_status( 'STARTING', 0 );
41  
42 # send STDOUT + STDERR to end-user console session
43 $me->{R}{eu_print_stderr} = sub { eu_print_stderr( $me, @_ ) };
44 $me->{R}{eu_print_stdout} = sub { eu_print_stdout( $me, @_ ) };
45 $me->{R}->redirect_io();
46  
47 my $n = $me->{request}{outfile} ? @{$me->{request}{outfile}} : 0;
48 $me->{R}{func_output} = sub{ _output_partition($me, $n, @_) };
49 $me->{R}{func_progress} = sub{ _maybe_update_status($me, 'RUNNING', @_) };
50  
51 eval {
52 _setup_outfiles( $me );
53  
54 if( $me->{request}{phase} eq 'map' ){
55 _do_map( $me );
56 }elsif( $me->{request}{phase} eq 'final' ){
57 _do_final( $me );
58 }elsif( $me->{request}{phase} =~ /^reduce/ ){
59 _do_reduce( $me );
60 }else{
61 die "unknown map/reduce phase '$me->{request}{phase}'\n";
62 }
63 };
64 if( my $e = $@ ){
65 my $myid = my_server_id();
66 verbose( "ERROR: $myid - $e" );
67 _send_eumsg($me, 'stderr', "ERROR: $myid - $e");
68 _update_status( 'FAILED', 0 );
69 }
70  
71 _close_outfiles( $me );
72 _update_status( 'FINISHED', 0 );
73 debug("finish child task");
74 exit(0);
75 }
76  
77 sub _setup_stdio_etal {
78  
79 # move socket to parent from STDOUT -> STATUS
80 # so user code doesn't trample
81  
82 open( STATUS, ">&STDOUT" );
83 close STDOUT; open( STDOUT, ">/dev/null");
84 close STDIN; open( STDIN, "/dev/null");
85 select STATUS; $| = 1; select STDOUT;
86 $SIG{CHLD} = sub{};
87 $SIG{ALRM} = sub{ die "timeout\n" };
88 openlog('mrgamoo', 'ndelay, pid', (conf_value('syslog') || 'local4'));
89  
90 alarm( $MAXRUN );
91 }
92  
93 sub _setup_console {
94 my $me = shift;
95  
96 debug("setup console: $me->{request}{jobid}, $me->{request}{console}");
97 $me->{euconsole} = AC::MrGamoo::EUConsole->new( $me->{request}{jobid}, $me->{request}{console} );
98 }
99  
100 sub _send_eumsg {
101 my $me = shift;
102 my $type = shift;
103 my $msg = shift;
104  
105 return unless $me->{euconsole};
106 $me->{euconsole}->send_msg($type, $msg);
107 }
108  
109 sub _update_status {
110 my $phase = shift;
111 my $amt = shift;
112  
113 # send status to parent process
114 debug("sending status @ $^T / $phase/$amt");
115 print STATUS "$phase $amt\n";
116 }
117  
118 sub _maybe_update_status {
119 my $me = shift;
120  
121 $^T = time();
122  
123 return if $^T < ($me->{status_time} + $STATUSTIME);
124 $me->{status_time} = $^T;
125 _update_status( @_ );
126 }
127  
128 sub _setup_outfiles {
129 my $me = shift;
130 my @out;
131  
132 my $gz = $me->attr(undef, 'compress');
133 for my $file ( @{$me->{request}{outfile}} ){
134 my $f = conf_value('basedir') . '/' . $file;
135 my($dir) = $f =~ m|^(.+)/[^/]+$|;
136  
137 eval{ mkpath($dir, undef, 0777) };
138 push @out, AC::MrGamoo::OutFile->new( $f, $gz );
139 }
140  
141 $me->{outfd} = \@out;
142 }
143  
144 sub _close_outfiles {
145 my $me = shift;
146  
147 for my $io ( @{$me->{outfd}} ){
148 $io->close();
149 }
150 delete $me->{outfd};
151 }
152  
153 sub _output_partition {
154 my ($me, $n, $key, $data) = @_;
155  
156 # md5 is twice as fast as sha1.
157 # anything written in perl is 10 times slower
158 my $hash = unpack('N', md5( $key ));
159 my $p = $hash % $n;
160 my $io = $me->{outfd}[$p];
161 $io->output( encode_json( [ $key, $data ] ), "\n" );
162 }
163  
164  
165 # end-user's 'print' come here
166 sub eu_print_stdout {
167 my $me = shift;
168  
169 _send_eumsg($me, 'stdout', "@_");
170 }
171  
172 sub eu_print_stderr {
173 my $me = shift;
174  
175 _send_eumsg($me, 'stderr', "@_");
176 }
177  
178 ################################################################
179  
180 sub _do_map {
181 my $me = shift;
182 my $mr = $me->{mr};
183  
184 debug("doing map");
185 my $n = @{$me->{request}{outfile}};
186 my $h_filter = $mr->get_code('filefilter');
187 my $h_read = $mr->get_code('readinput') || { code => \&readinput };
188 my $h_map = $mr->get_code('map');
189 my $f_filter = $h_filter ? $h_filter->{code} : undef;
190 my $f_read = $h_read->{code};
191 my $f_map = $h_map->{code};
192  
193 my $linen = 0;
194  
195 my $maxrun = $me->attr($h_map, 'maxrun');
196 alarm( $maxrun ) if $maxrun;
197  
198 for my $file (@{$me->{request}{infile}}){
199 _maybe_update_status( $me, 'RUNNING', $linen );
200 $me->{R}{config}{current_file} = $file; # in case user wants for debugging
201  
202 # filter file list
203 if( $f_filter ){
204 next unless $f_filter->( $file );
205 }
206  
207 debug("map file: $file");
208  
209 my $f = conf_value('basedir') . '/' . $file;
210  
211 open(my $fd, $f) || die "cannot open file '$f': $!\n";
212  
213 while(1){
214 _maybe_update_status( $me, 'RUNNING', $linen++ );
215  
216 # read input
217 my($d, $eof) = $f_read->( $fd );
218 last if $eof;
219 next unless defined $d;
220  
221 # map
222 my($key, $data) = $f_map->( $d );
223 next unless defined $key;
224 _output_partition( $me, $n, $key, $data );
225 }
226 }
227  
228 $h_map->{cleanup}->() if $h_map->{cleanup};
229 $h_read->{cleanup}->() if $h_read->{cleanup};
230 $h_filter->{cleanup}->() if $h_filter && $h_filter->{cleanup};
231 }
232  
233 sub _do_reduce {
234 my $me = shift;
235 my $mr = $me->{mr};
236  
237 my $n = @{$me->{request}{outfile}};
238 my($stage) = $me->{request}{phase} =~ m|reduce/(\d+)|;
239 my $h_reduce = $mr->get_code('reduce', $stage);
240 my $f_reduce = $h_reduce->{code};
241 my $rown = 0;
242  
243 my $maxrun = $me->attr($h_reduce, 'maxrun');
244 alarm( $maxrun ) if $maxrun;
245  
246 debug( "doing reduce step $stage" );
247  
248 # sort
249 my @cmd = _sort_cmd( $me, $h_reduce );
250 open(SORT, '-|', @cmd) || die "cannot open sort pipe: $!\n";
251 _sort_underway( $me, \*SORT );
252 my $iter = AC::MrGamoo::Iter::File->new( \*SORT, sub{ _maybe_update_status($me, 'RUNNING', $rown++) } );
253  
254 # reduce
255 while( defined(my $k = $iter->key()) ){
256 _maybe_update_status( $me, 'RUNNING', $rown++ );
257 my($key, $data) = $f_reduce->( $k, $iter );
258 _output_partition( $me, $n, $key, $data ) if defined $key;
259 }
260  
261 $h_reduce->{cleanup}() if $h_reduce->{cleanup};
262 }
263  
264 sub _do_final {
265 my $me = shift;
266 my $mr = $me->{mr};
267  
268 my $h_final = $mr->get_code('final');
269 my $linen = 0;
270  
271 if( $h_final ){
272 debug("doing final");
273 my $maxrun = $me->attr($h_final, 'maxrun');
274 alarm( $maxrun ) if $maxrun;
275  
276 my $f_final = $h_final->{code};
277  
278 # sort
279 my @cmd = _sort_cmd( $me, $h_final );
280 open(SORT, '-|', @cmd) || die "cannot open sort pipe: $!\n";
281 _sort_underway( $me, \*SORT );
282  
283 while(<SORT>){
284 chomp;
285 _maybe_update_status( $me, 'RUNNING', $linen++ );
286 my $x = decode_json($_);
287 $f_final->( $x->[0], $x->[1] );
288 }
289  
290 $h_final->{cleanup}() if $h_final->{cleanup};
291 }
292 }
293  
294 ################################################################
295  
296 sub _sort_cmd {
297 my $me = shift;
298 my $hc = shift;
299  
300 my $gz = $me->attr(undef, 'compress');
301 my $sort = $me->attr($hc,'sortprog') || conf_value('sortprog') || $SORTPROG;
302 my @file = map { conf_value('basedir') . '/' . $_ } @{$me->{request}{infile}};
303  
304 if( $gz ){
305 my $zcat = $me->attr($hc,'gzprog') || conf_value('gzprog') || $GZPROG;
306 my $cmd = $zcat . ' ' . join(' ', @file) . ' | ' . $sort;
307 debug("running cmd: $cmd");
308 return $cmd;
309 }else{
310 my @cmd = ($sort, @file);
311 debug("running cmd: @cmd");
312 return @cmd;
313 }
314 }
315  
316 sub _sort_underway {
317 my $me = shift;
318 my $fd = shift;
319  
320 my $fn = fileno($fd);
321 my $rfd = "\0\0\0\0";
322  
323 # send progress updates to master while sort is sorting
324 while(1){
325 vec($rfd, $fn, 1) = 1;
326  
327 select($rfd, undef, undef, 5);
328 return if vec($rfd, $fn, 1);
329 _maybe_update_status( $me, 'RUNNING', 0);
330 }
331  
332 }
333  
334 ################################################################
335  
336  
337 1;