Indexer/Dispatcher/Parallel.pm

filter: Statementstmt 27.6% 29 / 105 Branchbran 0.0% 0 / 8 Conditioncond 0.0% 0 / 7 MC/DCmcdc 0.0% 0 / 4 Subroutinesub 52.6% 10 / 19 Podpod 100% 3 / 3 totaltotal 28.7% 42 / 146 CC 12 SCAR 19.3
CC
12
Cov
24%
finish
20.0
_remove_stuck_processes
12.0
__ANON__
9.8
CRAP 6.9
« Indexer/Queue/FromList.pm ↑ Summary Indexer/Reporter/Base.pm »
1 package MyCPAN::Indexer::Dispatcher::Parallel;
2 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 1 ▶use parent qw(MyCPAN::Indexer::Component);
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 ▶use vars qw($VERSION $logger);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 $VERSION = '1.282';
8  
9 1 ▶use Log::Log4perl;
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 $logger = Log::Log4perl->get_logger( 'Dispatcher' );
13 }
14  
15 BEGIN {
16 # override since Tk overrides exit and this needs the real exit
17 1 ▶ no warnings 'redefine';
Subroutine BEGIN: called CC=1 SCAR=0.0
18 1 ▶ use Parallel::ForkManager;
Subroutine BEGIN: called CC=1 SCAR=0.0
19  
20 0 ▶ sub Parallel::ForkManager::finish {
Subroutine finish: not called CC=4 SCAR=30.0
Pod documented
21 0 $logger->debug( "In Parallel::ForkManager::finish" );
22 0 my ($s, $x) = @_;
23 0 ▶ CORE::exit ($x || 0) if $s->{in_child};
Branch: 1 branch
BranchTrueFalseTotal
if $s->{"in_child"} 0 0 0
Condition0%0/2
$x || 0
l!l
00
MC/DC0%0/1
$x || 0
$x
Truth table0%0/2
A
A= $x
Aresult
11
00
24 0 ▶ if( $s->{max_proc} == 0 ) { # max_proc == 0
Branch: 1 branch
BranchTrueFalseTotal
if ($s->{"max_proc"} == 0) 0 0 0
25 0 $s->on_finish($$, $x ,$s->{processes}->{$$}, 0, 0);
26 0 delete $s->{processes}->{$$};
27 }
28 0 return 0;
29 }
30 }
31  
32  
33 =encoding utf8
34  
35 =head1 NAME
36  
37 MyCPAN::Indexer::Dispatcher::Parallel - Pass out work to sub-processes
38  
39 =head1 SYNOPSIS
40  
41 Use this in C<backpan_indexer.pl> by specifying it as the queue class:
42  
43 # in backpan_indexer.config
44 dispatch_class MyCPAN::Indexer::Dispatcher::Parallel
45  
46 =head1 DESCRIPTION
47  
48 This class takes the list of distributions to process and passes them
49 out to the code that will do the work.
50  
51 =head2 Methods
52  
53 =over 4
54  
55 =item component_type
56  
57 This is a dispatcher type
58  
59 =cut
60  
61 0 ▶sub component_type { $_[0]->dispatcher_type }
Subroutine component_type: not called CC=1 SCAR=6.9
Pod documented
62  
63 =item get_dispatcher
64  
65 Takes the C<$Notes> hash and adds the C<dispatcher> key with a code
66 reference. This module uses C<Parallel::ForkManager> to run
67 jobs in parallel, and looks at the
68  
69 It also sets up keys for PID, whose value is an anonymous array
70 of process IDs. That array matches up with the one in the key
71 C<recent> which keeps track of the distributions it's processing.
72 It adds:
73  
74 dispatcher => sub { ... },
75 PID => [],
76 recent => [],
77  
78 =cut
79  
80  
81 sub get_dispatcher
82 0 ▶ {
Subroutine get_dispatcher: not called CC=1 SCAR=6.9
Pod documented
83 0 my( $self ) = @_;
84  
85 0 $self->set_note( 'Threads', $self->get_config->parallel_jobs );
86 0 $self->set_note( 'dispatcher', $self->_make_forker );
87 0 $self->set_note( 'interface_callback', $self->_make_interface_callback );
88 }
89  
90 sub _make_forker
91 0 ▶ {
Subroutine _make_forker: not called CC=2 SCAR=17.9
92 0 my( $self ) = @_;
93  
94 0 ▶ Parallel::ForkManager->new(
Condition0%0/2
$self->get_config->parallel_jobs || 1
l!l
00
MC/DC0%0/1
$self->get_config->parallel_jobs || 1
$self->get_config->parallel_jobs
Truth table0%0/2
A
A= $self->get_config->parallel_jobs
Aresult
11
00
95 $self->get_config->parallel_jobs || 1
96 );
97 }
98  
99 sub _make_interface_callback
100 0 ▶ {
Subroutine _make_interface_callback: not called CC=2 SCAR=16.6
101 0 my( $self ) = @_;
102  
103 0 foreach my $key ( qw(PID recent errors ) )
104 {
105 0 $self->set_note( $key, [] );
106 }
107  
108 0 $self->set_note( 'Total', scalar @{ $self->get_note( 'queue' ) } );
109 0 $self->set_note( 'Left', $self->get_note('Total') );
110 0 $self->set_note( 'Errors', 0 );
111 0 $self->set_note( 'Done', 0 );
112 0 $self->set_note( 'Started', scalar localtime );
113 0 $self->set_note( 'Finished', 0 );
114  
115 0 $self->set_note( 'queue_cursor', 0 );
116  
117 0 ▶ my $interface_callback = sub {
Subroutine __ANON__: not called CC=3 SCAR=22.8
118 0 $self->_remove_old_processes;
119  
120 $logger->debug( sprintf
121 "Finished: %s Left: %s",
122 0 map { $self->get_note( $_ ) } qw(Finished Left)
123 );
124  
125 0 ▶ unless( $self->get_note( 'Left' ) )
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->get_note("Left")) 0 0 0
126 {
127 0 $logger->debug( "Waiting on all children [" . time . "]" );
128 0 $self->get_note( 'dispatcher' )->wait_all_children;
129 0 $self->set_note( 'Finished', 1 );
130 0 return;
131 };
132  
133 0 $self->set_note_unless_defined( '_started', time );
134  
135 0 $self->set_note(
136 '_elapsed',
137 time - $self->get_note( '_started' )
138 );
139  
140 0 $self->set_note(
141 'Elapsed',
142 _elapsed( $self->get_note( '_elapsed' ) )
143 );
144  
145 0 my $item = $self->get_note_list_element(
146 'queue',
147 $self->increment_note( 'queue_cursor' )
148 );
149  
150 0 my $info;
151  
152 0 ▶ if( my $pid = $self->get_note( 'dispatcher' )->start )
Branch: 1 branch
BranchTrueFalseTotal
if (my $pid = $self->get_note('dispatcher')->start) { } 0 0 0
153 { #parent
154 0 $self->unshift_onto_note( 'PID', $pid );
155 0 $self->unshift_onto_note( 'recent', $item );
156  
157 0 $self->increment_note( 'Done' );
158  
159 0 $self->set_note(
160 'Left',
161 $self->get_note( 'Total' ) - $self->get_note( 'Done' )
162 );
163  
164 $logger->debug(
165 sprintf "Total: %s Done: %s Left: %s Finished: %s",
166 0 map { $self->get_note( $_ ) } qw( Total Done Left Finished )
167 );
168  
169 1 ▶ no warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
170 $self->set_note(
171 'Rate',
172 0 eval { $self->get_note( 'Done' ) / $self->get_note( '_elapsed' ) }
173 );
174  
175 }
176 else
177 { # child
178 0 $info = $self->get_note( 'child_task' )->( $item );
179 0 $self->get_note( 'dispatcher' )->finish;
180 0 $logger->error( "The child is still running!" );
181 }
182  
183 0 $info;
184 0 };
185  
186 0 $self->set_note( 'interface_callback', $interface_callback );
187 }
188  
189 sub _remove_old_processes
190 0 ▶ {
Subroutine _remove_old_processes: not called CC=2 SCAR=17.9
191 0 my( $self ) = @_;
192  
193 0 my $pid = $self->get_note( 'PID' );
194  
195 my @delete_indices = grep
196 0 { ! kill 0, $pid->[$_] }
197 0 .. $#$pid;
198  
199 0 my $recent = $self->get_note( 'recent' );
200  
201 0 foreach my $index ( reverse @delete_indices )
202 {
203 0 splice @$recent, $index, 1;
204 0 splice @$pid, $index, 1;
205 }
206 }
207  
208 sub _remove_stuck_processes
209 0 ▶ {
Subroutine _remove_stuck_processes: not called CC=3 SCAR=24.8
210 0 my( $self ) = @_;
211  
212 0 my $pids = $self->get_note( 'PID' );
213  
214 0 foreach my $pid ( @$pids )
215 {
216 0 $logger->debug( "Trying to remove stuck process $pid" );
217 my %children =
218 0 map { $_->{pid}, 1 }
219 0 ▶ grep { $_->{'ppid'} == $$ and $_->{'size'} > 1234 } #XXX
Condition0%0/3
$_->{'ppid'} == $$ && $_->{'size'} > 1234
!ll&&!rl&&r
000
MC/DC0%0/2
$_->{'ppid'} == $$ && $_->{'size'} > 1234
$_->{'ppid'} == $$ $_->{'size'} > 1234
Truth table0%0/3
A && B
A= $_->{'ppid'} == $$B= $_->{'size'} > 1234
ABresult
0X0
100
111
220 0 @{ Proc::ProcessTable->new->table };
221 }
222 }
223  
224 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
225 1 my %hash = ( days => 864000, hours => 3600, minutes => 60 );
226  
227 sub _elapsed
228 0 ▶ {
Subroutine _elapsed: not called CC=2 SCAR=17.9
229 0 my $seconds = shift;
230  
231 0 my @v;
232 0 foreach my $key ( qw(days hours minutes) )
233 {
234 0 push @v, int( $seconds / $hash{$key} );
235 0 $seconds -= $v[-1] * $hash{$key}
236 }
237  
238 0 push @v, $seconds;
239  
240 0 sprintf "%dd %02dh %02dm %02ds", @v;
241 }
242 }
243 1 1;
244  
245 1 1;
246  
247 =back
248  
249  
250 =head1 SEE ALSO
251  
252 MyCPAN::Indexer, MyCPAN::Indexer::Tutorial
253  
254 =head1 SOURCE AVAILABILITY
255  
256 This code is in Github:
257  
258 git://github.com/briandfoy/mycpan-indexer.git
259  
260 =head1 AUTHOR
261  
262 brian d foy, C<< <bdfoy@cpan.org> >>
263  
264 =head1 COPYRIGHT AND LICENSE
265  
266 Copyright © 2008-2018, brian d foy <bdfoy@cpan.org>. All rights reserved.
267  
268 You may redistribute this under the terms of the Artistic License 2.0.
269  
270 =cut
« Indexer/Queue/FromList.pm ↑ Summary Indexer/Reporter/Base.pm »