| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2010 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2010-Apr-22 10:50 (EDT) | ||
| 6 | # Function: info about tasks | ||
| 7 | # | ||
| 8 | # $Id: TaskInfo.pm,v 1.1 2010/11/01 18:41:57 jaw Exp $ | ||
| 9 | |||
| 10 | package AC::MrGamoo::Job::TaskInfo; | ||
| 11 | use AC::MrGamoo::Debug 'job_taskinfo'; | ||
| 12 | use AC::MrGamoo::PeerList; | ||
| 13 | use AC::Misc; | ||
| 14 | use strict; | ||
| 15 | |||
| 16 | our @ISA = 'AC::MrGamoo::Job::Info'; | ||
| 17 | |||
| 18 | my $MAXRETRY = 2; | ||
| 19 | |||
| 20 | sub new { | ||
| 21 | my $class = shift; | ||
| 22 | my $job = shift; | ||
| 23 | |||
| 24 | return bless { @_ }, $class; | ||
| 25 | } | ||
| 26 | |||
| 27 | sub pend { | ||
| 28 | my $me = shift; | ||
| 29 | my $job = shift; | ||
| 30 | |||
| 31 | return if $me->{replaced}; | ||
| 32 | return if $me->{finished}; | ||
| 33 | |||
| 34 | # create instance, put on pending queue | ||
| 35 | my $t = AC::MrGamoo::Job::Task->new($job, $me, $me->{server}); | ||
| 36 | return unless $t; | ||
| 37 | $me->{instance}{ $t->{id} } = $t; | ||
| 38 | |||
| 39 | return; | ||
| 40 | } | ||
| 41 | |||
| 42 | sub finished { | ||
| 43 | my $me = shift; | ||
| 44 | my $t = shift; | ||
| 45 | my $job = shift; | ||
| 46 | |||
| 47 | delete $me->{instance}{ $t->{id} }; | ||
| 48 | |||
| 49 | $me->{finished} = 1; | ||
| 50 | my $outfiles = $me->{outfile}; | ||
| 51 | my $server = $t->{server}; | ||
| 52 | |||
| 53 | debug("task finished $me->{id} on $server"); | ||
| 54 | # copy files | ||
| 55 | for my $fi (@$outfiles){ | ||
| 56 | # add to file_info - file is now on one server | ||
| 57 | debug(" outfile $fi->{filename}"); | ||
| 58 | $job->{file_info}{ $fi->{filename} } = { | ||
| 59 | filename => $fi->{filename}, | ||
| 60 | location => [ $server ], | ||
| 61 | }; | ||
| 62 | $job->{server_info}{$server}{has_files}{$fi->{filename}} = 1; | ||
| 63 | # QQQ - optionally leave final files? | ||
| 64 | push @{$job->{tmp_file}}, { filename => $fi->{filename}, server => $server }; | ||
| 65 | |||
| 66 | # add to copy_pending | ||
| 67 | foreach my $s ( @{$fi->{dst}} ){ | ||
| 68 | next if $job->{server_info}{$s}{has_files}{$fi->{filename}}; | ||
| 69 | my $c = AC::MrGamoo::Job::XferInfo->new( $job, | ||
| 70 | id => unique(), | ||
| 71 | filename => $fi->{filename}, | ||
| 72 | dst => $s, | ||
| 73 | ); | ||
| 74 | next unless $c; | ||
| 75 | $c->pend($job); | ||
| 76 | debug(" => pending copy for $s"); | ||
| 77 | } | ||
| 78 | } | ||
| 79 | } | ||
| 80 | |||
| 81 | sub failed { | ||
| 82 | my $me = shift; | ||
| 83 | my $t = shift; | ||
| 84 | my $job = shift; | ||
| 85 | |||
| 86 | delete $me->{instance}{ $t->{id} }; | ||
| 87 | |||
| 88 | my $server = $me->{server}; | ||
| 89 | my $status = get_peer_status_from_id($server); | ||
| 90 | if( $status != 200 ){ | ||
| 91 | # replan tasks | ||
| 92 | $job->_replan_server($server, 'task', $me); | ||
| 93 | return; | ||
| 94 | } | ||
| 95 | |||
| 96 | if( $me->{retries} ++ > $MAXRETRY ){ | ||
| 97 | # replan tasks | ||
| 98 | $me->replan($job); | ||
| 99 | return; | ||
| 100 | } | ||
| 101 | |||
| 102 | # retry | ||
| 103 | debug("retry task"); | ||
| 104 | $me->pend($job); | ||
| 105 | } | ||
| 106 | |||
| 107 | ################################################################ | ||
| 108 | |||
| 109 | sub replan { | ||
| 110 | my $me = shift; | ||
| 111 | my $job = shift; | ||
| 112 | |||
| 113 | return if $me->{replaced}; | ||
| 114 | |||
| 115 | return $job->abort( reason => "too many failed tasks. out of replan options.") | ||
| 116 | if $me->{replaces}; | ||
| 117 | |||
| 118 | return $me->_replan_altserver($job) if $me->{altserver}; | ||
| 119 | |||
| 120 | if( $me->{phase} eq 'reduce' ){ | ||
| 121 | verbose("cannot replan task. no altserver"); | ||
| 122 | $job->abort(reason => "cannot replan task. no alternate server available"); | ||
| 123 | return; | ||
| 124 | } | ||
| 125 | |||
| 126 | $me->_replan_map($job); | ||
| 127 | } | ||
| 128 | |||
| 129 | sub _replan_altserver { | ||
| 130 | my $me = shift; | ||
| 131 | my $job = shift; | ||
| 132 | |||
| 133 | $me->{server} = $me->{altserver}; | ||
| 134 | delete $me->{retries}; | ||
| 135 | delete $me->{altserver}; | ||
| 136 | |||
| 137 | debug("replanning task to new server"); | ||
| 138 | $me->pend($job); | ||
| 139 | } | ||
| 140 | |||
| 141 | sub _replan_map { | ||
| 142 | my $me = shift; | ||
| 143 | my $job = shift; | ||
| 144 | |||
| 145 | # remove task | ||
| 146 | # divy files among servers | ||
| 147 | # create new tasks | ||
| 148 | # rediddle next phase | ||
| 149 | |||
| 150 | my %newplan; # server => @files | ||
| 151 | |||
| 152 | $me->{replaced} = 1; | ||
| 153 | |||
| 154 | unless( $me->{altplan} ){ | ||
| 155 | verbose("no alt task available - aborting"); | ||
| 156 | $job->abort(reason => "cannot replan task. no alternate available"); | ||
| 157 | return; | ||
| 158 | } | ||
| 159 | |||
| 160 | # divy files | ||
| 161 | for my $f (@{$me->{infile}}){ | ||
| 162 | # alt loc for this file? | ||
| 163 | my $loc = $me->{altplan}{$f}; | ||
| 164 | |||
| 165 | unless($loc){ | ||
| 166 | verbose("file unavailable - aborting ($f)"); | ||
| 167 | $job->abort(reason => "file unavailable: $f"); | ||
| 168 | return; | ||
| 169 | } | ||
| 170 | push @{$newplan{$loc}}, $f; | ||
| 171 | } | ||
| 172 | |||
| 173 | my @new; | ||
| 174 | for my $as (keys %newplan){ | ||
| 175 | my $newid = unique(); | ||
| 176 | my $oldid = $me->{id}; | ||
| 177 | |||
| 178 | my $new = AC::MrGamoo::Job::TaskInfo->new($job, | ||
| 179 | id => $newid, | ||
| 180 | phase => $me->{phase}, | ||
| 181 | infile => $newplan{$as}, | ||
| 182 | replaces => $oldid, | ||
| 183 | outfile => [ map { | ||
| 184 | (my $f = $_->{filename}) =~ s/$oldid/$newid/; | ||
| 185 | { dst => $_->{dst}, filename => $f, } | ||
| 186 | } @{$me->{outfile}} ], | ||
| 187 | server => $as, | ||
| 188 | ); | ||
| 189 | debug("replan map $oldid => $newid on $as"); | ||
| 190 | |||
| 191 | # keep plan up to date | ||
| 192 | $job->{plan}{taskidx}{$newid} = $new; | ||
| 193 | push @{$job->{plan}{taskplan}[0]{task}}, $new; | ||
| 194 | |||
| 195 | # move to pending queue | ||
| 196 | $new->pend($job) if $job->{phase_no} == 0; | ||
| 197 | |||
| 198 | push @new, $new; | ||
| 199 | } | ||
| 200 | |||
| 201 | $me->_replan_replace_files( $job, @new ); | ||
| 202 | |||
| 203 | } | ||
| 204 | |||
| 205 | sub _replan_replace_files { | ||
| 206 | my $me = shift; | ||
| 207 | my $job = shift; | ||
| 208 | my @new = shift; | ||
| 209 | |||
| 210 | my $oldid = $me->{id}; | ||
| 211 | my $curphase = 0; # map | ||
| 212 | my $nxtphase = 1; # reduce/0 | ||
| 213 | |||
| 214 | # remove old task's files, add new tasks' files | ||
| 215 | for my $ti ( @{$job->{plan}{taskplan}[$nxtphase]{task}} ){ | ||
| 216 | my @infile; | ||
| 217 | for my $file (@{$ti->{infile}}){ | ||
| 218 | if( $file =~ /$oldid/ ){ | ||
| 219 | for my $new (@new){ | ||
| 220 | my $newid = $new->{id}; | ||
| 221 | (my $n = $file) =~ s/$oldid/$newid/; | ||
| 222 | push @infile, $n; | ||
| 223 | } | ||
| 224 | }else{ | ||
| 225 | push @infile, $file; | ||
| 226 | } | ||
| 227 | } | ||
| 228 | $ti->{infile} = \@infile; | ||
| 229 | } | ||
| 230 | } | ||
| 231 | |||
| 232 | 1; |