| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2010 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2010-Jan-13 13:03 (EST) | ||
| 6 | # Function: m/r jobs we are controlling | ||
| 7 | # | ||
| 8 | # $Id: Job.pm,v 1.6 2011/01/18 18:00:23 jaw Exp $ | ||
| 9 | |||
| 10 | package AC::MrGamoo::Job; | ||
| 11 | use AC::MrGamoo::Debug 'job'; | ||
| 12 | use AC::MrGamoo::Config; | ||
| 13 | use AC::MrGamoo::Job::Plan; | ||
| 14 | use AC::MrGamoo::Job::RePlan; | ||
| 15 | use AC::MrGamoo::Job::Util; | ||
| 16 | use AC::MrGamoo::Job::Done; | ||
| 17 | use AC::MrGamoo::Job::Info; | ||
| 18 | use AC::MrGamoo::Job::Action; | ||
| 19 | use AC::MrGamoo::FileList; | ||
| 20 | use AC::MrGamoo::PeerList; | ||
| 21 | use AC::MrGamoo::MySelf; | ||
| 22 | use AC::MrGamoo::EUConsole; | ||
| 23 | use AC::MrGamoo::Stats; | ||
| 24 | use AC::DC::IO; | ||
| 25 | use AC::Misc; | ||
| 26 | use JSON; | ||
| 27 | use Time::HiRes 'time'; | ||
| 28 | use strict; | ||
| 29 | |||
| 30 | # RSN - config? tune? | ||
| 31 | our $TASKTIMEOUT = 10; | ||
| 32 | our $XFERTIMEOUT = 10; | ||
| 33 | our $TASKSRVRMAX = 4; | ||
| 34 | our $XFERSRVRMAX = 4; | ||
| 35 | our $REQMAX = 10; | ||
| 36 | our $MAXLOAD = 0.5; | ||
| 37 | |||
| 38 | our %REGISTRY; | ||
| 39 | our $MSGID = $$; | ||
| 40 | my $_trying; | ||
| 41 | |||
| 42 | our $MAXFILE = `sh -c "ulimit -n"`; | ||
| 43 | $MAXFILE = 255 if $^O eq 'solaris' && $MAXFILE > 255; | ||
| 44 | |||
| 45 | ################################################################ | ||
| 46 | |||
| 47 | # schedule periodic "cronjob" | ||
| 48 | AC::DC::Sched->new( | ||
| 49 | info => "job periodic", | ||
| 50 | freq => 2, | ||
| 51 | func => \&periodic, | ||
| 52 | ); | ||
| 53 | |||
| 54 | ################################################################ | ||
| 55 | |||
| 56 | sub new { | ||
| 57 | my $class = shift; | ||
| 58 | # %{ APCMRMJobCreate } | ||
| 59 | |||
| 60 | my $me = bless { | ||
| 61 | request => { @_ }, | ||
| 62 | phase_no => -1, | ||
| 63 | file_info => {}, | ||
| 64 | tmp_file => [], | ||
| 65 | server_info => {}, | ||
| 66 | task_running => {}, | ||
| 67 | task_pending => {}, | ||
| 68 | xfer_running => {}, | ||
| 69 | xfer_pending => {}, | ||
| 70 | request_running => {}, | ||
| 71 | request_pending => {}, | ||
| 72 | statistics => { job_start => time() }, | ||
| 73 | }, $class; | ||
| 74 | |||
| 75 | if( $REGISTRY{ $me->{request}{jobid} } ){ | ||
| 76 | verbose("ignoring duplicate request job $me->{request}{jobid}"); | ||
| 77 | # will cause a 200 OK, so the requestor will not retry | ||
| 78 | return $REGISTRY{ $me->{request}{jobid} }; | ||
| 79 | } | ||
| 80 | |||
| 81 | verbose("new job: $me->{request}{jobid} ($me->{request}{traceinfo})"); | ||
| 82 | |||
| 83 | my $cf = $me->{options} = decode_json( $me->{request}{options} ) if $me->{request}{options}; | ||
| 84 | |||
| 85 | # open connection to eu-console | ||
| 86 | $me->{euconsole} = AC::MrGamoo::EUConsole->new( $me->{request}{jobid}, $me->{request}{console} ); | ||
| 87 | |||
| 88 | # partially compile | ||
| 89 | eval { | ||
| 90 | $me->{mr} = AC::MrGamoo::Submit::Compile->new( text => $me->{request}{jobsrc} ); | ||
| 91 | }; | ||
| 92 | if(my $e = $@){ | ||
| 93 | problem("cannot compile job: $e"); | ||
| 94 | return; | ||
| 95 | } | ||
| 96 | |||
| 97 | # RSN - get_file_list + Plan may take too long - do in sub-process | ||
| 98 | |||
| 99 | # get file list | ||
| 100 | my $files = get_file_list( $cf ); | ||
| 101 | #print STDERR "files: ", dumper($files), "\n"; | ||
| 102 | |||
| 103 | for my $f (@$files){ | ||
| 104 | $me->{file_info}{ $f->{filename} } = $f; | ||
| 105 | } | ||
| 106 | |||
| 107 | # get server list | ||
| 108 | my $servers = get_peer_list( $cf ); | ||
| 109 | #print STDERR "servers: ", dumper($servers), "\n"; | ||
| 110 | |||
| 111 | # plan job | ||
| 112 | my $plan = AC::MrGamoo::Job::Plan->new( $me, $servers, $files ); | ||
| 113 | #print STDERR "plan: ", dumper($plan), "\n"; | ||
| 114 | |||
| 115 | $me->{plan} = $plan; | ||
| 116 | |||
| 117 | $me->{maxfail} = 5 * ( (keys %{$plan->{taskidx}}) + @{$plan->{copying}}); | ||
| 118 | |||
| 119 | $me->{server_info}{$_->{id}} = {} for @$servers; | ||
| 120 | |||
| 121 | $me->_preload_file_copies(); | ||
| 122 | $REGISTRY{ $me->{request}{jobid} } = $me; | ||
| 123 | return $me; | ||
| 124 | } | ||
| 125 | |||
| 126 | sub start { | ||
| 127 | my $me = shift; | ||
| 128 | |||
| 129 | debug("start job"); | ||
| 130 | $me->{euconsole}->send_msg('debug', 'starting job'); | ||
| 131 | $me->_try_to_do_something(); | ||
| 132 | 1; | ||
| 133 | } | ||
| 134 | |||
| 135 | ################################################################ | ||
| 136 | |||
| 137 | # record status rcvd from task | ||
| 138 | sub task_status { | ||
| 139 | my $me = _find(shift, @_); | ||
| 140 | my %p = @_; | ||
| 141 | my $taskid = $p{taskid}; | ||
| 142 | |||
| 143 | return unless $me; | ||
| 144 | my $t = $me->{task_running}{$taskid}; | ||
| 145 | return unless $t; | ||
| 146 | |||
| 147 | $t->update_status( $me, $p{phase}, $p{progress} ); | ||
| 148 | |||
| 149 | 1; | ||
| 150 | } | ||
| 151 | |||
| 152 | # record status rcvd from file xfer | ||
| 153 | sub xfer_status { | ||
| 154 | my $me = _find(shift, @_); | ||
| 155 | my %p = @_; | ||
| 156 | my $copyid = $p{copyid}; | ||
| 157 | |||
| 158 | return unless $me; | ||
| 159 | my $c = $me->{xfer_running}{$copyid}; | ||
| 160 | return unless $c; | ||
| 161 | |||
| 162 | $c->update_status( $me, $p{status_code} ); | ||
| 163 | |||
| 164 | 1; | ||
| 165 | } | ||
| 166 | |||
| 167 | ################################################################ | ||
| 168 | |||
| 169 | sub periodic { | ||
| 170 | # debug("periodic check"); | ||
| 171 | |||
| 172 | $_trying = 0; | ||
| 173 | for my $job (values %REGISTRY){ | ||
| 174 | $job->_periodic(); | ||
| 175 | } | ||
| 176 | } | ||
| 177 | |||
| 178 | sub _periodic { | ||
| 179 | my $me = shift; | ||
| 180 | |||
| 181 | my @t = values %{$me->{task_running}}; | ||
| 182 | for my $t ( @t ){ | ||
| 183 | my $lt = $t->{status_time} || $t->{start_time}; | ||
| 184 | next if $^T - $lt < ($me->{options}{tasktimeout} || $TASKTIMEOUT); | ||
| 185 | |||
| 186 | $t->failed( $me, 'timeout' ); | ||
| 187 | } | ||
| 188 | |||
| 189 | my @c = values %{$me->{xfer_running}}; | ||
| 190 | for my $c ( @c ){ | ||
| 191 | my $lt = $c->{status_time} || $c->{start_time}; | ||
| 192 | next if $^T - $lt < ($me->{options}{xfertimeout} || $XFERTIMEOUT); | ||
| 193 | |||
| 194 | $c->failed( $me, 'timeout' ); | ||
| 195 | } | ||
| 196 | |||
| 197 | $me->_try_to_do_something(); | ||
| 198 | |||
| 199 | my $tr = keys %{ $me->{task_running} }; | ||
| 200 | my $tp = keys %{ $me->{task_pending} }; | ||
| 201 | my $xr = keys %{ $me->{xfer_running} }; | ||
| 202 | my $xp = keys %{ $me->{xfer_pending} }; | ||
| 203 | my $rr = keys %{ $me->{request_running} }; | ||
| 204 | my $rp = keys %{ $me->{request_pending} }; | ||
| 205 | |||
| 206 | my $ph = $me->{plan}{phases}[ $me->{phase_no} ] || 'none'; | ||
| 207 | |||
| 208 | debug("status: phase $ph, task $tr / $tp, xfer $xr / $xp, reqs $rr / $rp") | ||
| 209 | if $tr || $tp || $xr || $xp || $rr || $rp; | ||
| 210 | |||
| 211 | } | ||
| 212 | |||
| 213 | ################################################################ | ||
| 214 | |||
| 215 | sub _start_next_phase { | ||
| 216 | my $me = shift; | ||
| 217 | |||
| 218 | debug("next phase"); | ||
| 219 | $me->{phase_no} ++; | ||
| 220 | my $tp = $me->{plan}{taskplan}[ $me->{phase_no} ]; | ||
| 221 | |||
| 222 | # finished all phases ? | ||
| 223 | unless( $tp ){ | ||
| 224 | if( $me->{_cleanedup} ){ | ||
| 225 | $me->_finished(); | ||
| 226 | return 1; | ||
| 227 | }else{ | ||
| 228 | $me->_cleanup_files(); | ||
| 229 | return; | ||
| 230 | } | ||
| 231 | } | ||
| 232 | |||
| 233 | debug("job $me->{request}{jobid} starting phase $tp->{phase}"); | ||
| 234 | $me->{euconsole}->send_msg('debug', "starting phase $tp->{phase}"); | ||
| 235 | |||
| 236 | # move tasks to pending | ||
| 237 | for my $t (@{$tp->{task}}){ | ||
| 238 | $t->pend($me); | ||
| 239 | } | ||
| 240 | } | ||
| 241 | |||
| 242 | sub _maybe_start_task { | ||
| 243 | my $me = shift; | ||
| 244 | my $task = shift; | ||
| 245 | |||
| 246 | #debug("maybe start task"); | ||
| 247 | |||
| 248 | my $server = $task->{server}; | ||
| 249 | my $underway = keys %{$me->{server_info}{$server}{task_running}}; | ||
| 250 | return if $underway >= $TASKSRVRMAX; # don't overload server | ||
| 251 | return if $task->{start_after} > $^T; # rate limit retries | ||
| 252 | |||
| 253 | # RSN - check that prerequisite xfers completed | ||
| 254 | $task->start( $me ); | ||
| 255 | return 1; | ||
| 256 | } | ||
| 257 | |||
| 258 | sub _maybe_start_xfer { | ||
| 259 | my $me = shift; | ||
| 260 | my $copy = shift; | ||
| 261 | |||
| 262 | #debug("maybe start xfer"); | ||
| 263 | |||
| 264 | my $server = $copy->{server}; | ||
| 265 | my $underway = keys %{$me->{server_info}{$server}{xfer_running}}; | ||
| 266 | return if $underway >= $XFERSRVRMAX; # don't overload server | ||
| 267 | return if $copy->{start_after} > $^T; # rate limit retries | ||
| 268 | |||
| 269 | $copy->start( $me ); | ||
| 270 | return 1; | ||
| 271 | } | ||
| 272 | |||
| 273 | sub _maybe_start_request { | ||
| 274 | my $me = shift; | ||
| 275 | my $req = shift; | ||
| 276 | |||
| 277 | $req->start( $me ); | ||
| 278 | return 1; | ||
| 279 | } | ||
| 280 | |||
| 281 | ################################################################ | ||
| 282 | |||
| 283 | sub _preload_file_copies { | ||
| 284 | my $me = shift; | ||
| 285 | |||
| 286 | # start copying files | ||
| 287 | |||
| 288 | for my $c ( @{$me->{plan}{copying}} ){ | ||
| 289 | $c->pend($me); | ||
| 290 | } | ||
| 291 | } | ||
| 292 | |||
| 293 | |||
| 294 | ################################################################ | ||
| 295 | |||
| 296 | sub _try_to_do_something { | ||
| 297 | my $me = shift; | ||
| 298 | |||
| 299 | # debug("try something"); | ||
| 300 | |||
| 301 | return if $me->{_finished}; | ||
| 302 | return if $_trying; | ||
| 303 | |||
| 304 | # is this phase done | ||
| 305 | if( !(keys %{$me->{task_running}}) | ||
| 306 | && !(keys %{$me->{task_pending}}) | ||
| 307 | && !(keys %{$me->{xfer_running}}) | ||
| 308 | && !(keys %{$me->{xfer_pending}}) | ||
| 309 | && !(keys %{$me->{request_running}}) | ||
| 310 | && !(keys %{$me->{request_pending}}) | ||
| 311 | ){ | ||
| 312 | |||
| 313 | # this phase is finished | ||
| 314 | return if $me->_start_next_phase(); | ||
| 315 | } | ||
| 316 | |||
| 317 | if( $me->{aborted} | ||
| 318 | && !(keys %{$me->{request_running}}) | ||
| 319 | && !(keys %{$me->{request_pending}}) | ||
| 320 | ){ | ||
| 321 | # this phase is finished | ||
| 322 | return if $me->_start_next_phase(); | ||
| 323 | } | ||
| 324 | |||
| 325 | # check load ave, etc | ||
| 326 | return unless $me->_ok_to_do_more_p(); | ||
| 327 | |||
| 328 | $_trying ++; | ||
| 329 | |||
| 330 | # are there requests that can start | ||
| 331 | my @rp = values %{$me->{request_pending}}; | ||
| 332 | my $startreqs = $REQMAX - keys %{$me->{request_running}}; | ||
| 333 | for my $r (@rp){ | ||
| 334 | last if $startreqs <= 0; | ||
| 335 | next unless $me->_maybe_start_request( $r ); | ||
| 336 | $startreqs --; | ||
| 337 | } | ||
| 338 | |||
| 339 | unless( $me->{aborted} ){ | ||
| 340 | |||
| 341 | # are there tasks that can start | ||
| 342 | my $started = 0; | ||
| 343 | my @tp = sort { $a->{created} <=> $b->{created} } values %{$me->{task_pending}}; | ||
| 344 | for my $t (@tp){ | ||
| 345 | $started += $me->_maybe_start_task( $t ); | ||
| 346 | last if $started >= $me->{plan}{nserver} / 4; # keep them from getting in to lockstep | ||
| 347 | } | ||
| 348 | |||
| 349 | # are there copies that can start | ||
| 350 | my @cp = sort { $a->{created} <=> $b->{created} } values %{$me->{xfer_pending}}; | ||
| 351 | for my $c (@cp){ | ||
| 352 | $me->_maybe_start_xfer( $c ); | ||
| 353 | } | ||
| 354 | } | ||
| 355 | |||
| 356 | # should we speculatively copy some files | ||
| 357 | |||
| 358 | # should we speculatively retry a task | ||
| 359 | |||
| 360 | |||
| 361 | $_trying --; | ||
| 362 | } | ||
| 363 | |||
| 364 | ################################################################ | ||
| 365 | |||
| 366 | sub something_failed { | ||
| 367 | my $me = shift; | ||
| 368 | |||
| 369 | return if ++$me->{total_fails} < $me->{maxfail}; | ||
| 370 | $me->abort(); | ||
| 371 | return 1; | ||
| 372 | } | ||
| 373 | |||
| 374 | ################################################################ | ||
| 375 | |||
| 376 | sub report { | ||
| 377 | |||
| 378 | my $txt; | ||
| 379 | |||
| 380 | for my $j (values %REGISTRY){ | ||
| 381 | my $ph; | ||
| 382 | $ph = 'start' if $j->{phase_no} < 0; | ||
| 383 | $ph ||= 'cleanup' if $j->{phase_no} >= @{$j->{plan}{phases}}; | ||
| 384 | $ph ||= $j->{plan}{phases}[ $j->{phase_no} ]; | ||
| 385 | |||
| 386 | my $tr = keys %{$j->{task_running}}; | ||
| 387 | my $tp = keys %{$j->{task_pending}}; | ||
| 388 | my $cr = keys %{$j->{copy_running}}; | ||
| 389 | my $cp = keys %{$j->{copy_pending}}; | ||
| 390 | my $rr = keys %{$j->{request_running}}; | ||
| 391 | my $rp = keys %{$j->{request_pending}}; | ||
| 392 | |||
| 393 | $txt .= sprintf("%s %8s %4d %4d %4d %4d %4d %4d\n", $j->{request}{jobid}, $ph, $tr, $tp, $cr, $cp, $rr, $rp); | ||
| 394 | # ... | ||
| 395 | } | ||
| 396 | |||
| 397 | return $txt; | ||
| 398 | } | ||
| 399 | |||
| 400 | |||
| 401 | 1; |