Indexer/Worker.pm

filter: Statementstmt 92.5% 25 / 27 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 9 / 9 Podpod n/a totaltotal 94.4% 34 / 36 CC 1 SCAR 0.3
CC
1
Cov
93%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« Indexer/CPANMiniInject.pm ↑ Summary BackPANstats.pm »
1 package MyCPAN::Indexer::Worker;
2 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 1 ▶use parent qw(MyCPAN::Indexer::Component);
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use vars qw($VERSION $logger);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 $VERSION = '1.282';
8  
9 1 ▶use Cwd;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use File::Basename;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use File::Spec::Functions qw(catfile);
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use Log::Log4perl;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 0 ▶use MyCPAN::Indexer;
Subroutine BEGIN: called CC=1 SCAR=2.6
14 use Proc::ProcessTable;
15 use YAML;
16  
17 =encoding utf8
18  
19 =head1 NAME
20  
21 MyCPAN::Indexer::Worker - Do the indexing
22  
23 =head1 SYNOPSIS
24  
25 Use this in backpan_indexer.pl by specifying it as the queue class:
26  
27 # in backpan_indexer.config
28 worker_class MyCPAN::Indexer::Worker
29  
30 =head1 DESCRIPTION
31  
32 This class takes a distribution and analyses it. This is what the dispatcher
33 hands a disribution to for the actual indexing.
34  
35 =head2 Methods
36  
37 =over 4
38  
39 =item get_task
40  
41 C<get_task> sets the C<child_task> key in the notes. The
42 value is a code reference that takes a distribution path as its only
43 argument and indexes that distribution.
44  
45 See L<MyCPAN::Indexer::Tutorial> for details about what C<get_task> expects
46 and should do.
47  
48 =cut
49  
50 BEGIN {
51 $logger = Log::Log4perl->get_logger( 'Worker' );
52 }
53  
54 =item component_type
55  
56 This is a worker component.
57  
58 =cut
59  
60 sub component_type { $_[0]->worker_type }
61  
62 =item get_task
63  
64 =cut
65  
66 sub get_task
67 {
68 my( $self ) = @_;
69  
70 my $config = $self->get_config;
71  
72 my $coordinator = $self->get_coordinator;
73  
74 my $indexer = $coordinator->get_component( 'indexer' );
75  
76 $logger->debug( "Worker class is " . __PACKAGE__ );
77 $logger->debug( "Indexer class is " . $indexer->class );
78  
79 my $child_task = sub {
80 my $dist = shift;
81  
82 my $dist_basename = basename( $dist );
83  
84 my $basename = $coordinator->get_reporter->check_for_previous_successful_result( $dist );
85 $logger->debug( "Found successful report for $dist_basename" ) unless $basename;
86 return bless {
87 dist_info => {
88 dist_path => $dist,
89 dist_basename => $dist_basename
90 },
91 skipped => 1,
92 }, $indexer->class unless $basename;
93  
94 my $previous_error_basename = $coordinator->get_reporter->check_for_previous_error_result( $dist ) || '';
95 $logger->debug( "Error report returned [$previous_error_basename]" );
96 $logger->debug( "Found error report for $dist_basename" ) if $previous_error_basename;
97  
98 # we used to handle this by just deleting all the old error
99 # reports in setup_dirs over in MyCPAN::App::BackPAN::Indexer
100 # deleting all the reports before we got started made it
101 # impossible to get a list of error reports to retry
102 if( $previous_error_basename and ! $config->retry_errors )
103 {
104 $logger->debug( "By config, skipping $dist because I'm not retrying errors" );
105 return bless {
106 dist_info => {
107 dist_path => $dist,
108 dist_basename => $dist_basename,
109 },
110 skip_error => 1,
111 }, $self->get_config->indexer_class;
112 }
113 elsif( $previous_error_basename and $config->retry_errors )
114 {
115 # if we are re-trying errors and there is already a report
116 # unlink the previous report
117 my $report_full_path = $coordinator->get_reporter->get_error_report_path( $dist );
118  
119 $logger->debug( "Trying to unlink $report_full_path" );
120 my $rc = unlink $report_full_path;
121 $logger->debug( ($rc ? 'unlinked ' : 'failed to unlink ') . $report_full_path );
122 }
123  
124 $logger->info( "Starting Worker for $dist_basename\n" );
125  
126 my $starting_dir = cwd();
127  
128 unless( chdir $config->temp_dir )
129 {
130 $logger->error( "Could not change to " . $config->temp_dir . " : $!\n" );
131 exit 255;
132 }
133  
134 $logger->debug( sprintf "Setting alarm for %d seconds", $config->alarm );
135 local $SIG{ALRM} = sub {
136 $logger->info( "Alarm rang for $dist_basename in process $$!\n" );
137 $self->_cleanup_children;
138 $logger->info( "Cleaned up" );
139 die;
140 };
141  
142 local $SIG{CHLD} = 'IGNORE';
143 alarm( $config->alarm || 15 );
144 $logger->debug( "Examining $dist_basename" );
145  
146 my $info = do {
147 unless( -e $dist )
148 {
149 $logger->warn( "Dist $dist does not exist" );
150 undef;
151 }
152 elsif( ! -s $dist )
153 {
154 $logger->warn( "Dist $dist has zero size" );
155 my $info = bless {}, $self->get_config->indexer_class;
156 $info->setup_dist_info( $dist );
157 $info->set_dist_info( 'unindexable', 'zero size' );
158 $info->setup_run_info;
159 $info->set_run_info( qw(completed 1) );
160 $info;
161 }
162 else
163 {
164 $logger->warn( "Indexing $dist" );
165 eval { $indexer->run( $dist ) };
166 }
167  
168 };
169  
170 $logger->debug( "Done examining $dist_basename" );
171 my $at = $@; chomp $at;
172 alarm 0;
173  
174 chdir $starting_dir;
175  
176 unless( defined $info )
177 {
178 $logger->error( "Indexing failed for $dist_basename: $at" );
179 $info = bless {}, $self->get_config->indexer_class; # XXX TODO make this a real class
180 $info->setup_dist_info( $dist );
181 $info->setup_run_info;
182 $info->set_run_info( qw(completed 0) );
183 $info->set_run_info( error => $at );
184 }
185 elsif( ! eval { $info->run_info( 'completed' ) } )
186 {
187 $logger->error( "Indexing for $dist_basename did not complete" );
188 $self->_copy_bad_dist( $info ) if $config->copy_bad_dists;
189 }
190  
191 $self->_add_run_info( $info );
192  
193 $coordinator->get_note('reporter')->( $info );
194  
195 $logger->debug( "Worker for $dist_basename done" );
196  
197 # some things hang anyway, so just to be careful we'll cleanup
198 # everything here.
199 $self->_cleanup_children;
200  
201 $logger->debug( "Cleaned up, returning..." );
202 $info;
203 };
204  
205 $coordinator->set_note( 'child_task', $child_task );
206  
207 1;
208 }
209  
210 sub _cleanup_children
211 {
212 $logger->warn( "Cleaning up after $$" );
213  
214 my %children =
215 map { $_->{pid}, 1 }
216 grep { $_->{'ppid'} == $$ }
217 @{ Proc::ProcessTable->new->table };
218 $logger->debug( "Child processes are @{[keys %children]}" );
219  
220 my @grandchildren =
221 map { $_->{pid} }
222 grep { exists $children{ $_->{'ppid'} } }
223 @{ Proc::ProcessTable->new->table };
224 $logger->debug( "Grandchild processes are @grandchildren" );
225  
226 my @processes = ( keys %children, @grandchildren );
227 $logger->debug( "There are " . @processes . " processes to clean up" );
228 return unless @processes;
229  
230 $logger->debug( "Preparing to kill" );
231  
232 kill 9, @processes;
233  
234 return;
235 }
236  
237 sub _copy_bad_dist
238 {
239 my( $self, $info ) = @_;
240  
241 my $config = $self->get_config;
242 my $bad_dist_dir = $config->copy_bad_dists;
243 return unless $bad_dist_dir;
244  
245 unless( -d $bad_dist_dir and mkdir $bad_dist_dir ) {
246 $logger->error( "Could not make dist dir [$bad_dist_dir]: $!" );
247 return;
248 }
249  
250 my $dist_file = $info->dist_info( 'dist_file' );
251 my $basename = $info->dist_info( 'dist_basename' );
252 my $new_name = catfile( $bad_dist_dir, $basename );
253  
254 unless( -e $new_name )
255 {
256 $logger->debug( "Copying bad dist" );
257  
258 my( $in, $out );
259  
260 unless( open $out, ">", $new_name )
261 {
262 $logger->fatal( "Could not copy bad dist to $new_name: $!" );
263 return;
264 }
265  
266 unless( open $in, "<", $dist_file )
267 {
268 $logger->fatal( "Could not open bad dist to $dist_file: $!" );
269 return;
270 }
271  
272 while( <$in> ) { print { $out } $_ }
273 close $in;
274 close $out;
275 }
276 }
277  
278 sub _add_run_info
279 {
280 my( $self, $info ) = @_;
281  
282 my $config = $self->get_config;
283  
284 return unless eval { $info->can( 'set_run_info' ) };
285  
286 $info->set_run_info( $_, $config->get( $_ ) )
287 foreach ( $config->directives );
288  
289 $info->set_run_info( 'uuid', $self->get_note( 'UUID' ) );
290  
291 $info->set_run_info( 'child_pid', $$ );
292 $info->set_run_info( 'parent_pid', eval { $config->indexer_class->getppid } );
293  
294 $info->set_run_info( 'ENV', \%ENV );
295  
296 return 1;
297 }
298  
299 =back
300  
301 =head1 SEE ALSO
302  
303 MyCPAN::Indexer, MyCPAN::Indexer::Tutorial
304  
305 =head1 SOURCE AVAILABILITY
306  
307 This code is in Github:
308  
309 git://github.com/briandfoy/mycpan-indexer.git
310  
311 =head1 AUTHOR
312  
313 brian d foy, C<< <bdfoy@cpan.org> >>
314  
315 =head1 COPYRIGHT AND LICENSE
316  
317 Copyright © 2008-2018, brian d foy <bdfoy@cpan.org>. All rights reserved.
318  
319 You may redistribute this under the terms of the Artistic License 2.0.
320  
321 =cut
322  
323 1;
« Indexer/CPANMiniInject.pm ↑ Summary BackPANstats.pm »