| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2010 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2010-Jan-13 13:02 (EST) | ||
| 6 | # Function: m/r task (parent side) | ||
| 7 | # | ||
| 8 | # $Id: Task.pm,v 1.10 2011/01/14 20:58:26 jaw Exp $ | ||
| 9 | |||
| 10 | package AC::MrGamoo::Task; | ||
| 11 | use AC::MrGamoo::Debug 'task'; | ||
| 12 | use AC::MrGamoo::Submit::Compile; | ||
| 13 | use AC::MrGamoo::Submit::Request; | ||
| 14 | use AC::MrGamoo::Task::Running; | ||
| 15 | use AC::MrGamoo::PeerList; | ||
| 16 | use AC::MrGamoo::Config; | ||
| 17 | use AC::DC::IO::Forked; | ||
| 18 | use JSON; | ||
| 19 | use strict; | ||
| 20 | |||
| 21 | my $TSTART = $^T; | ||
| 22 | my $TIMEOUT = 3600; | ||
| 23 | my $MAXREQ = 2; | ||
| 24 | my $MAXRUNNING = 7; # tune me! | ||
| 25 | my %REGISTRY; | ||
| 26 | my $msgid = $$; | ||
| 27 | |||
| 28 | |||
| 29 | ################################################################ | ||
| 30 | |||
| 31 | # schedule periodic "cronjob" | ||
| 32 | AC::DC::Sched->new( | ||
| 33 | info => "task periodic", | ||
| 34 | freq => 5, | ||
| 35 | func => \&periodic, | ||
| 36 | ); | ||
| 37 | |||
| 38 | ################################################################ | ||
| 39 | |||
| 40 | sub new { | ||
| 41 | my $class = shift; | ||
| 42 | # %ACPMRMTaskCreate | ||
| 43 | |||
| 44 | my $me = bless { | ||
| 45 | request => { @_ }, | ||
| 46 | }, $class; | ||
| 47 | debug("new task $me->{request}{taskid}"); | ||
| 48 | |||
| 49 | my $task = $me->{request}{taskid}; | ||
| 50 | return problem("cannot create task: no task id") unless $task; | ||
| 51 | if( $REGISTRY{$task} ){ | ||
| 52 | verbose("ignoring duplicate request task $task"); | ||
| 53 | # will cause a 200 OK, so the requestor will not retry | ||
| 54 | return $REGISTRY{$task}; | ||
| 55 | } | ||
| 56 | |||
| 57 | $me->{options} = decode_json( $me->{request}{options} ) if $me->{request}{options}; | ||
| 58 | $me->{initres} = from_json( $me->{request}{initres}, {allow_nonref => 1} ) if $me->{request}{initres}; | ||
| 59 | |||
| 60 | # compile | ||
| 61 | eval { | ||
| 62 | my $mr = AC::MrGamoo::Submit::Compile->new( text => $me->{request}{jobsrc} ); | ||
| 63 | # merge job config + opts. | ||
| 64 | $mr->set_config($me->{options}); | ||
| 65 | $mr->set_initres($me->{initres}); | ||
| 66 | $me->{R} = AC::MrGamoo::Submit::Request->new( $mr ); | ||
| 67 | $me->{R}{config}{jobid} = $me->{request}{jobid}; | ||
| 68 | $me->{R}{config}{taskid} = $me->{request}{taskid}; | ||
| 69 | $me->{mr} = $mr; | ||
| 70 | }; | ||
| 71 | if(my $e = $@){ | ||
| 72 | problem("cannot compile task: $e"); | ||
| 73 | return; | ||
| 74 | } | ||
| 75 | |||
| 76 | # measure | ||
| 77 | for my $file (@{$me->{request}{infile}}){ | ||
| 78 | my $s = (stat(conf_value('basedir') . '/' . $file))[7]; | ||
| 79 | $me->{_inputsize} += $s | ||
| 80 | } | ||
| 81 | |||
| 82 | debug("input size: $me->{_inputsize}"); | ||
| 83 | |||
| 84 | # print STDERR "Task: ", dumper($me), "\n"; | ||
| 85 | $REGISTRY{$task} = $me; | ||
| 86 | return $me; | ||
| 87 | } | ||
| 88 | |||
| 89 | sub start { | ||
| 90 | my $me = shift; | ||
| 91 | |||
| 92 | # if too many tasks are running, queue | ||
| 93 | my $nrun = 0; | ||
| 94 | for my $t (values %REGISTRY){ | ||
| 95 | $nrun ++ if $t->{io}; | ||
| 96 | } | ||
| 97 | |||
| 98 | if( $nrun >= $MAXRUNNING ){ | ||
| 99 | $me->{_queueprio} = $^T - $TSTART + $me->{_inputsize} / 1_000_000; | ||
| 100 | $me->{status}{phase} = 'QUEUED'; | ||
| 101 | $me->{status}{amt} = 0; | ||
| 102 | debug("queue $me->{request}{phase} task $me->{request}{jobid}/$me->{request}{taskid} prio $me->{_queueprio}"); | ||
| 103 | return 1; | ||
| 104 | } | ||
| 105 | |||
| 106 | $me->_start(); | ||
| 107 | } | ||
| 108 | |||
| 109 | sub _start { | ||
| 110 | my $me = shift; | ||
| 111 | |||
| 112 | debug("start $me->{request}{phase} task $me->{request}{jobid}/$me->{request}{taskid}"); | ||
| 113 | |||
| 114 | my $io = AC::DC::IO::Forked->new( | ||
| 115 | \&AC::MrGamoo::Task::Running::_start_task, [ $me ], | ||
| 116 | info => "task $me->{request}{jobid}/$me->{request}{taskid}", | ||
| 117 | ); | ||
| 118 | |||
| 119 | $me->{io} = $io; | ||
| 120 | $io->timeout_rel($TIMEOUT); | ||
| 121 | $io->set_callback('timeout', \&_timeout); | ||
| 122 | $io->set_callback('read', \&_read, $me); | ||
| 123 | $io->set_callback('shutdown', \&_shutdown, $me); | ||
| 124 | |||
| 125 | $io->start(); | ||
| 126 | } | ||
| 127 | |||
| 128 | sub abort { | ||
| 129 | my $me = _find(shift, @_); | ||
| 130 | |||
| 131 | return unless $me; | ||
| 132 | debug("abort task $me->{request}{taskid}"); | ||
| 133 | $me->{io}->shut() if $me->{io}; | ||
| 134 | return 1; | ||
| 135 | } | ||
| 136 | |||
| 137 | |||
| 138 | sub _find { | ||
| 139 | my $me = shift; | ||
| 140 | return $me if ref $me; | ||
| 141 | |||
| 142 | my %p = @_; | ||
| 143 | my $task = $p{taskid}; | ||
| 144 | $me = $REGISTRY{$task}; | ||
| 145 | |||
| 146 | return $me; | ||
| 147 | } | ||
| 148 | |||
| 149 | sub _timeout { | ||
| 150 | my $io = shift; | ||
| 151 | $io->shut(); | ||
| 152 | } | ||
| 153 | |||
| 154 | sub _shutdown { | ||
| 155 | my $io = shift; | ||
| 156 | my $evt = shift; | ||
| 157 | my $me = shift; | ||
| 158 | |||
| 159 | # send status to master | ||
| 160 | $me->_send_status(); | ||
| 161 | |||
| 162 | my $task = $me->{request}{taskid}; | ||
| 163 | delete $REGISTRY{$task}; | ||
| 164 | |||
| 165 | delete $me->{io}; | ||
| 166 | |||
| 167 | periodic(1); # try to start another task | ||
| 168 | } | ||
| 169 | |||
| 170 | sub _send_status_done { | ||
| 171 | my $io = shift; | ||
| 172 | my $evt = shift; | ||
| 173 | my $me = shift; | ||
| 174 | |||
| 175 | $me->{_status_underway} --; | ||
| 176 | } | ||
| 177 | |||
| 178 | sub _read { | ||
| 179 | my $io = shift; | ||
| 180 | my $evt = shift; | ||
| 181 | my $me = shift; | ||
| 182 | |||
| 183 | debug("read child $me->{request}{taskid}: $evt->{data}."); | ||
| 184 | # read status msg from child | ||
| 185 | $io->{rbuffer} .= $evt->{data}; | ||
| 186 | |||
| 187 | my @l = split /^/m, $io->{rbuffer}; | ||
| 188 | $io->{rbuffer} = ''; | ||
| 189 | for my $l (@l){ | ||
| 190 | unless( $l =~ /\n/ ){ | ||
| 191 | $io->{rbuffer} = $l; | ||
| 192 | last; | ||
| 193 | } | ||
| 194 | |||
| 195 | debug("got status $l"); | ||
| 196 | chomp($l); | ||
| 197 | my($phase, $amt) = split /\s+/, $l; | ||
| 198 | |||
| 199 | $me->{status}{phase} = $phase; | ||
| 200 | $me->{status}{amt} = $amt; | ||
| 201 | $me->{status}{fail} = 1 if $phase eq 'FAILED'; | ||
| 202 | |||
| 203 | # send status to master | ||
| 204 | $me->_send_status(); | ||
| 205 | } | ||
| 206 | } | ||
| 207 | |||
| 208 | sub _send_status { | ||
| 209 | my $me = shift; | ||
| 210 | |||
| 211 | # don't kill the master with too many requests | ||
| 212 | # return if $me->{_status_underway} >= $MAXREQ; | ||
| 213 | |||
| 214 | my($addr, $port) = get_peer_addr_from_id( $me->{request}{master} ); | ||
| 215 | return unless $addr; | ||
| 216 | |||
| 217 | debug("sending task status update $me->{request}{taskid} => $me->{status}{phase}"); | ||
| 218 | my $x = AC::MrGamoo::API::Client->new( $addr, $port, "task $me->{request}{taskid}", { | ||
| 219 | type => 'mrgamoo_taskstatus', | ||
| 220 | msgidno => $msgid++, | ||
| 221 | want_reply => 0, | ||
| 222 | }, { | ||
| 223 | jobid => $me->{request}{jobid}, | ||
| 224 | taskid => $me->{request}{taskid}, | ||
| 225 | phase => $me->{status}{phase}, | ||
| 226 | progress => $me->{status}{amt}, | ||
| 227 | } ); | ||
| 228 | |||
| 229 | return unless $x; | ||
| 230 | |||
| 231 | $me->{_status_underway} ++; | ||
| 232 | $x->set_callback('shutdown', \&_send_status_done, $me); | ||
| 233 | |||
| 234 | $x->start(); | ||
| 235 | |||
| 236 | } | ||
| 237 | |||
| 238 | sub attr { | ||
| 239 | my $me = shift; | ||
| 240 | my $bk = shift; | ||
| 241 | my $p = shift; | ||
| 242 | |||
| 243 | return $bk->{attr}{$p} if $bk && $bk->{attr}{$p}; | ||
| 244 | return $me->{options}{$p}; | ||
| 245 | } | ||
| 246 | |||
| 247 | sub report { | ||
| 248 | |||
| 249 | my $txt; | ||
| 250 | |||
| 251 | for my $t (values %REGISTRY){ | ||
| 252 | $txt .= "$t->{request}{jobid} $t->{request}{taskid} $t->{status}{phase}\n"; | ||
| 253 | } | ||
| 254 | |||
| 255 | return $txt; | ||
| 256 | } | ||
| 257 | |||
| 258 | sub periodic { | ||
| 259 | my $quick = shift; | ||
| 260 | |||
| 261 | # how many tasks are running? | ||
| 262 | my $nrun = 0; | ||
| 263 | for my $t (values %REGISTRY){ | ||
| 264 | $nrun ++ if $t->{io}; | ||
| 265 | } | ||
| 266 | |||
| 267 | return if $quick && $nrun >= $MAXRUNNING; | ||
| 268 | |||
| 269 | # queued? send status, maybe start | ||
| 270 | |||
| 271 | for my $t (sort { $a->{_queueprio} <=> $b->{_queueprio} } values %REGISTRY){ | ||
| 272 | next if $t->{io}; | ||
| 273 | |||
| 274 | $t->_send_status() unless $quick; | ||
| 275 | |||
| 276 | if( $nrun < $MAXRUNNING ){ | ||
| 277 | $t->_start(); | ||
| 278 | $nrun ++; | ||
| 279 | } | ||
| 280 | } | ||
| 281 | } | ||
| 282 | |||
| 283 | |||
| 284 | 1; | ||
| 285 |