| 1 | package MyCPAN::Indexer::Worker; | ||
| 2 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | |||
| 5 | 1 | ▶ | use parent qw(MyCPAN::Indexer::Component); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 6 | 1 | ▶ | use vars qw($VERSION $logger); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 7 | $VERSION = '1.282'; | ||
| 8 | |||
| 9 | 1 | ▶ | use Cwd; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 1 | ▶ | use File::Basename; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 11 | 1 | ▶ | use File::Spec::Functions qw(catfile); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | 1 | ▶ | use Log::Log4perl; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 13 | 0 | ▶ | use MyCPAN::Indexer; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 14 | use Proc::ProcessTable; | ||
| 15 | use YAML; | ||
| 16 | |||
| 17 | =encoding utf8 | ||
| 18 | |||
| 19 | =head1 NAME | ||
| 20 | |||
| 21 | MyCPAN::Indexer::Worker - Do the indexing | ||
| 22 | |||
| 23 | =head1 SYNOPSIS | ||
| 24 | |||
| 25 | Use this in backpan_indexer.pl by specifying it as the queue class: | ||
| 26 | |||
| 27 | # in backpan_indexer.config | ||
| 28 | worker_class MyCPAN::Indexer::Worker | ||
| 29 | |||
| 30 | =head1 DESCRIPTION | ||
| 31 | |||
| 32 | This class takes a distribution and analyses it. This is what the dispatcher | ||
| 33 | hands a disribution to for the actual indexing. | ||
| 34 | |||
| 35 | =head2 Methods | ||
| 36 | |||
| 37 | =over 4 | ||
| 38 | |||
| 39 | =item get_task | ||
| 40 | |||
| 41 | C<get_task> sets the C<child_task> key in the notes. The | ||
| 42 | value is a code reference that takes a distribution path as its only | ||
| 43 | argument and indexes that distribution. | ||
| 44 | |||
| 45 | See L<MyCPAN::Indexer::Tutorial> for details about what C<get_task> expects | ||
| 46 | and should do. | ||
| 47 | |||
| 48 | =cut | ||
| 49 | |||
| 50 | BEGIN { | ||
| 51 | $logger = Log::Log4perl->get_logger( 'Worker' ); | ||
| 52 | } | ||
| 53 | |||
| 54 | =item component_type | ||
| 55 | |||
| 56 | This is a worker component. | ||
| 57 | |||
| 58 | =cut | ||
| 59 | |||
| 60 | sub component_type { $_[0]->worker_type } | ||
| 61 | |||
| 62 | =item get_task | ||
| 63 | |||
| 64 | =cut | ||
| 65 | |||
| 66 | sub get_task | ||
| 67 | { | ||
| 68 | my( $self ) = @_; | ||
| 69 | |||
| 70 | my $config = $self->get_config; | ||
| 71 | |||
| 72 | my $coordinator = $self->get_coordinator; | ||
| 73 | |||
| 74 | my $indexer = $coordinator->get_component( 'indexer' ); | ||
| 75 | |||
| 76 | $logger->debug( "Worker class is " . __PACKAGE__ ); | ||
| 77 | $logger->debug( "Indexer class is " . $indexer->class ); | ||
| 78 | |||
| 79 | my $child_task = sub { | ||
| 80 | my $dist = shift; | ||
| 81 | |||
| 82 | my $dist_basename = basename( $dist ); | ||
| 83 | |||
| 84 | my $basename = $coordinator->get_reporter->check_for_previous_successful_result( $dist ); | ||
| 85 | $logger->debug( "Found successful report for $dist_basename" ) unless $basename; | ||
| 86 | return bless { | ||
| 87 | dist_info => { | ||
| 88 | dist_path => $dist, | ||
| 89 | dist_basename => $dist_basename | ||
| 90 | }, | ||
| 91 | skipped => 1, | ||
| 92 | }, $indexer->class unless $basename; | ||
| 93 | |||
| 94 | my $previous_error_basename = $coordinator->get_reporter->check_for_previous_error_result( $dist ) || ''; | ||
| 95 | $logger->debug( "Error report returned [$previous_error_basename]" ); | ||
| 96 | $logger->debug( "Found error report for $dist_basename" ) if $previous_error_basename; | ||
| 97 | |||
| 98 | # we used to handle this by just deleting all the old error | ||
| 99 | # reports in setup_dirs over in MyCPAN::App::BackPAN::Indexer | ||
| 100 | # deleting all the reports before we got started made it | ||
| 101 | # impossible to get a list of error reports to retry | ||
| 102 | if( $previous_error_basename and ! $config->retry_errors ) | ||
| 103 | { | ||
| 104 | $logger->debug( "By config, skipping $dist because I'm not retrying errors" ); | ||
| 105 | return bless { | ||
| 106 | dist_info => { | ||
| 107 | dist_path => $dist, | ||
| 108 | dist_basename => $dist_basename, | ||
| 109 | }, | ||
| 110 | skip_error => 1, | ||
| 111 | }, $self->get_config->indexer_class; | ||
| 112 | } | ||
| 113 | elsif( $previous_error_basename and $config->retry_errors ) | ||
| 114 | { | ||
| 115 | # if we are re-trying errors and there is already a report | ||
| 116 | # unlink the previous report | ||
| 117 | my $report_full_path = $coordinator->get_reporter->get_error_report_path( $dist ); | ||
| 118 | |||
| 119 | $logger->debug( "Trying to unlink $report_full_path" ); | ||
| 120 | my $rc = unlink $report_full_path; | ||
| 121 | $logger->debug( ($rc ? 'unlinked ' : 'failed to unlink ') . $report_full_path ); | ||
| 122 | } | ||
| 123 | |||
| 124 | $logger->info( "Starting Worker for $dist_basename\n" ); | ||
| 125 | |||
| 126 | my $starting_dir = cwd(); | ||
| 127 | |||
| 128 | unless( chdir $config->temp_dir ) | ||
| 129 | { | ||
| 130 | $logger->error( "Could not change to " . $config->temp_dir . " : $!\n" ); | ||
| 131 | exit 255; | ||
| 132 | } | ||
| 133 | |||
| 134 | $logger->debug( sprintf "Setting alarm for %d seconds", $config->alarm ); | ||
| 135 | local $SIG{ALRM} = sub { | ||
| 136 | $logger->info( "Alarm rang for $dist_basename in process $$!\n" ); | ||
| 137 | $self->_cleanup_children; | ||
| 138 | $logger->info( "Cleaned up" ); | ||
| 139 | die; | ||
| 140 | }; | ||
| 141 | |||
| 142 | local $SIG{CHLD} = 'IGNORE'; | ||
| 143 | alarm( $config->alarm || 15 ); | ||
| 144 | $logger->debug( "Examining $dist_basename" ); | ||
| 145 | |||
| 146 | my $info = do { | ||
| 147 | unless( -e $dist ) | ||
| 148 | { | ||
| 149 | $logger->warn( "Dist $dist does not exist" ); | ||
| 150 | undef; | ||
| 151 | } | ||
| 152 | elsif( ! -s $dist ) | ||
| 153 | { | ||
| 154 | $logger->warn( "Dist $dist has zero size" ); | ||
| 155 | my $info = bless {}, $self->get_config->indexer_class; | ||
| 156 | $info->setup_dist_info( $dist ); | ||
| 157 | $info->set_dist_info( 'unindexable', 'zero size' ); | ||
| 158 | $info->setup_run_info; | ||
| 159 | $info->set_run_info( qw(completed 1) ); | ||
| 160 | $info; | ||
| 161 | } | ||
| 162 | else | ||
| 163 | { | ||
| 164 | $logger->warn( "Indexing $dist" ); | ||
| 165 | eval { $indexer->run( $dist ) }; | ||
| 166 | } | ||
| 167 | |||
| 168 | }; | ||
| 169 | |||
| 170 | $logger->debug( "Done examining $dist_basename" ); | ||
| 171 | my $at = $@; chomp $at; | ||
| 172 | alarm 0; | ||
| 173 | |||
| 174 | chdir $starting_dir; | ||
| 175 | |||
| 176 | unless( defined $info ) | ||
| 177 | { | ||
| 178 | $logger->error( "Indexing failed for $dist_basename: $at" ); | ||
| 179 | $info = bless {}, $self->get_config->indexer_class; # XXX TODO make this a real class | ||
| 180 | $info->setup_dist_info( $dist ); | ||
| 181 | $info->setup_run_info; | ||
| 182 | $info->set_run_info( qw(completed 0) ); | ||
| 183 | $info->set_run_info( error => $at ); | ||
| 184 | } | ||
| 185 | elsif( ! eval { $info->run_info( 'completed' ) } ) | ||
| 186 | { | ||
| 187 | $logger->error( "Indexing for $dist_basename did not complete" ); | ||
| 188 | $self->_copy_bad_dist( $info ) if $config->copy_bad_dists; | ||
| 189 | } | ||
| 190 | |||
| 191 | $self->_add_run_info( $info ); | ||
| 192 | |||
| 193 | $coordinator->get_note('reporter')->( $info ); | ||
| 194 | |||
| 195 | $logger->debug( "Worker for $dist_basename done" ); | ||
| 196 | |||
| 197 | # some things hang anyway, so just to be careful we'll cleanup | ||
| 198 | # everything here. | ||
| 199 | $self->_cleanup_children; | ||
| 200 | |||
| 201 | $logger->debug( "Cleaned up, returning..." ); | ||
| 202 | $info; | ||
| 203 | }; | ||
| 204 | |||
| 205 | $coordinator->set_note( 'child_task', $child_task ); | ||
| 206 | |||
| 207 | 1; | ||
| 208 | } | ||
| 209 | |||
| 210 | sub _cleanup_children | ||
| 211 | { | ||
| 212 | $logger->warn( "Cleaning up after $$" ); | ||
| 213 | |||
| 214 | my %children = | ||
| 215 | map { $_->{pid}, 1 } | ||
| 216 | grep { $_->{'ppid'} == $$ } | ||
| 217 | @{ Proc::ProcessTable->new->table }; | ||
| 218 | $logger->debug( "Child processes are @{[keys %children]}" ); | ||
| 219 | |||
| 220 | my @grandchildren = | ||
| 221 | map { $_->{pid} } | ||
| 222 | grep { exists $children{ $_->{'ppid'} } } | ||
| 223 | @{ Proc::ProcessTable->new->table }; | ||
| 224 | $logger->debug( "Grandchild processes are @grandchildren" ); | ||
| 225 | |||
| 226 | my @processes = ( keys %children, @grandchildren ); | ||
| 227 | $logger->debug( "There are " . @processes . " processes to clean up" ); | ||
| 228 | return unless @processes; | ||
| 229 | |||
| 230 | $logger->debug( "Preparing to kill" ); | ||
| 231 | |||
| 232 | kill 9, @processes; | ||
| 233 | |||
| 234 | return; | ||
| 235 | } | ||
| 236 | |||
| 237 | sub _copy_bad_dist | ||
| 238 | { | ||
| 239 | my( $self, $info ) = @_; | ||
| 240 | |||
| 241 | my $config = $self->get_config; | ||
| 242 | my $bad_dist_dir = $config->copy_bad_dists; | ||
| 243 | return unless $bad_dist_dir; | ||
| 244 | |||
| 245 | unless( -d $bad_dist_dir and mkdir $bad_dist_dir ) { | ||
| 246 | $logger->error( "Could not make dist dir [$bad_dist_dir]: $!" ); | ||
| 247 | return; | ||
| 248 | } | ||
| 249 | |||
| 250 | my $dist_file = $info->dist_info( 'dist_file' ); | ||
| 251 | my $basename = $info->dist_info( 'dist_basename' ); | ||
| 252 | my $new_name = catfile( $bad_dist_dir, $basename ); | ||
| 253 | |||
| 254 | unless( -e $new_name ) | ||
| 255 | { | ||
| 256 | $logger->debug( "Copying bad dist" ); | ||
| 257 | |||
| 258 | my( $in, $out ); | ||
| 259 | |||
| 260 | unless( open $out, ">", $new_name ) | ||
| 261 | { | ||
| 262 | $logger->fatal( "Could not copy bad dist to $new_name: $!" ); | ||
| 263 | return; | ||
| 264 | } | ||
| 265 | |||
| 266 | unless( open $in, "<", $dist_file ) | ||
| 267 | { | ||
| 268 | $logger->fatal( "Could not open bad dist to $dist_file: $!" ); | ||
| 269 | return; | ||
| 270 | } | ||
| 271 | |||
| 272 | while( <$in> ) { print { $out } $_ } | ||
| 273 | close $in; | ||
| 274 | close $out; | ||
| 275 | } | ||
| 276 | } | ||
| 277 | |||
| 278 | sub _add_run_info | ||
| 279 | { | ||
| 280 | my( $self, $info ) = @_; | ||
| 281 | |||
| 282 | my $config = $self->get_config; | ||
| 283 | |||
| 284 | return unless eval { $info->can( 'set_run_info' ) }; | ||
| 285 | |||
| 286 | $info->set_run_info( $_, $config->get( $_ ) ) | ||
| 287 | foreach ( $config->directives ); | ||
| 288 | |||
| 289 | $info->set_run_info( 'uuid', $self->get_note( 'UUID' ) ); | ||
| 290 | |||
| 291 | $info->set_run_info( 'child_pid', $$ ); | ||
| 292 | $info->set_run_info( 'parent_pid', eval { $config->indexer_class->getppid } ); | ||
| 293 | |||
| 294 | $info->set_run_info( 'ENV', \%ENV ); | ||
| 295 | |||
| 296 | return 1; | ||
| 297 | } | ||
| 298 | |||
| 299 | =back | ||
| 300 | |||
| 301 | =head1 SEE ALSO | ||
| 302 | |||
| 303 | MyCPAN::Indexer, MyCPAN::Indexer::Tutorial | ||
| 304 | |||
| 305 | =head1 SOURCE AVAILABILITY | ||
| 306 | |||
| 307 | This code is in Github: | ||
| 308 | |||
| 309 | git://github.com/briandfoy/mycpan-indexer.git | ||
| 310 | |||
| 311 | =head1 AUTHOR | ||
| 312 | |||
| 313 | brian d foy, C<< <bdfoy@cpan.org> >> | ||
| 314 | |||
| 315 | =head1 COPYRIGHT AND LICENSE | ||
| 316 | |||
| 317 | Copyright © 2008-2018, brian d foy <bdfoy@cpan.org>. All rights reserved. | ||
| 318 | |||
| 319 | You may redistribute this under the terms of the Artistic License 2.0. | ||
| 320 | |||
| 321 | =cut | ||
| 322 | |||
| 323 | 1; | ||