ForkQueue.pm

filter: Statementstmt 91.6% 22 / 24 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 9 / 9 Podpod n/a totaltotal 93.9% 31 / 33 CC 1 SCAR 0.3
CC
1
Cov
92%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« ForkAsync.pm ↑ Summary
1 package Sys::ForkQueue;
2 {
3 $Sys::ForkQueue::VERSION = '0.14';
4 }
5 1 ▶BEGIN {
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 $Sys::ForkQueue::AUTHORITY = 'cpan:TEX';
7 }
8 # ABSTRACT: Run any number of jobs in a controlled manner in parallel.
9  
10 1 ▶use 5.010_000;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use mro 'c3';
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use feature ':5.10';
Subroutine BEGIN: called CC=1 SCAR=0.0
13  
14 1 ▶use Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 1 ▶use namespace::autoclean;
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 # use IO::Handle;
18 # use autodie;
19 # use MooseX::Params::Validate;
20  
21 # for fork()
22 1 ▶use Errno qw(EAGAIN);
Subroutine BEGIN: called CC=1 SCAR=0.0
23 1 ▶use POSIX qw(WNOHANG SIGTERM);
Subroutine BEGIN: called CC=1 SCAR=0.0
24 0 ▶use Sys::CPU;
Subroutine BEGIN: called CC=1 SCAR=2.6
25  
26 # DGR: we'll it's ugly but that's the way fork() works in perl ...
27 ## no critic (ProhibitPackageVars)
28 # for fork control
29 our $zombies = 0;
30 our %Kid_Status;
31 our %childs_running = ();
32 ## use critic
33 ## no critic (RequireLocalizedPunctuationVars)
34 $SIG{CHLD} = sub { $zombies++ };
35 $SIG{INT} = \&_sigterm;
36 $SIG{TERM} = \&_sigterm;
37 ## use critic
38  
39 has 'chdir' => (
40 'is' => 'rw',
41 'isa' => 'Str',
42 'default' => 0,
43 );
44  
45 has 'umask' => (
46 'is' => 'rw',
47 'isa' => 'Str',
48 'default' => 0,
49 );
50  
51 has 'jobs' => (
52 'is' => 'ro',
53 'isa' => 'ArrayRef[Str]',
54 'required' => 1,
55 );
56  
57 has '_job_status' => (
58 'is' => 'ro',
59 'isa' => 'HashRef[Int]',
60 'default' => sub { {} },
61 );
62  
63 has 'code' => (
64 'is' => 'ro',
65 'isa' => 'CodeRef',
66 'required' => 1,
67 );
68  
69 has 'args' => (
70 'is' => 'rw',
71 'isa' => 'HashRef',
72 'default' => sub { {} },
73 );
74  
75 has 'concurrency' => (
76 'is' => 'rw',
77 'isa' => 'Int',
78 'lazy' => 1,
79 'builder' => '_num_cores',
80 );
81  
82 has 'redirect_output' => (
83 'is' => 'rw',
84 'isa' => 'Str',
85 );
86  
87 has 'chdir' => (
88 'is' => 'rw',
89 'isa' => 'Str',
90 'default' => 0,
91 );
92  
93 has 'setsid' => (
94 'is' => 'rw',
95 'isa' => 'Bool',
96 'default' => 0,
97 );
98  
99 has 'delayedfork' => (
100 'is' => 'rw',
101 'isa' => 'Bool',
102 'default' => 1,
103 );
104  
105 with qw(Log::Tree::RequiredLogger);
106  
107 sub _num_cores {
108 my $self = shift;
109  
110 return Sys::CPU::cpu_count() || 1;
111 }
112  
113 sub run {
114 my $self = shift;
115  
116 # Loop control
117 my $concurrency = $self->concurrency(); # 0 means inifite num. of forks
118 my $forks_running = 0;
119 my $childs_returned = 0;
120 my $ok = 1;
121  
122 JOB: foreach my $job ( @{ $self->jobs() } ) {
123  
124 while ( $concurrency && $forks_running >= $concurrency ) {
125  
126 # wait until there is a free slot to run
127 ## no critic (ProhibitSleepViaSelect)
128 select undef, undef, undef, 0.2;
129 ## use critic
130 if ($zombies) {
131 my $reaped = $self->_reaper();
132 $childs_returned += $reaped if $reaped;
133 $forks_running = $forks_running - $reaped if $reaped;
134 }
135 }
136 if ( !$concurrency || $forks_running < $concurrency ) {
137 $self->logger->log( message => "Creating fork for Job: $job", level => 'debug', );
138  
139 # fork() - see Programming Perl p. 737
140 FORK:
141 {
142 if ( my $pid = fork ) {
143  
144 # This is the parent process, child pid is in $pid
145 $forks_running++;
146 $childs_running{$pid} = 1;
147 ## no critic (ProhibitSleepViaSelect)
148 select undef, undef, undef, 0.1 if $self->delayedfork();
149 ## use critic
150 }
151 elsif ( defined $pid ) {
152  
153 # prevent the possibility to acquire a controlling terminal
154 $SIG{'HUP'} = 'IGNORE';
155  
156 # bring Logger in a suitable state
157 # this will at least clear the internal logging buffer, other tasks may be performed as well depending on
158 # the implementation of the Logger
159 $self->logger()->forked();
160  
161 if ( $self->setsid() ) {
162 $self->logger()->log( message => 'Calling setsid', level => 'debug', );
163 POSIX::setsid() # create own process group
164 }
165 if ( $self->chdir() && -d $self->chdir() ) {
166 $self->logger()->log( message => 'Changing work dir to ' . $self->chdir(), level => 'debug', );
167 chdir( $self->chdir() );
168 }
169 elsif ( $self->chdir() ) {
170 $self->logger()->log( message => 'Changing work dir to /.', level => 'debug', );
171 chdir(q{/});
172 }
173  
174 # clear the file creation mask
175 umask $self->umask();
176 ## no critic (RequireCheckedClose)
177 close(STDIN);
178 if ( $self->redirect_output() ) {
179 $self->logger()->log( message => 'Redirecting output to ' . $self->redirect_output(), level => 'debug', );
180 close(STDOUT);
181 close(STDERR);
182 }
183 ## use critic
184 ## no critic (RequireCheckedOpen)
185 open( STDIN, '<', '/dev/null' );
186 if ( $self->redirect_output() ) {
187 open( STDOUT, '>>', $self->redirect_output() . q{.} . $job );
188 open( STDERR, '>>', $self->redirect_output() . q{.} . $job );
189 }
190 ## use critic
191  
192 # $pid is null, if defined
193 # This is the child process
194 # get the pid of the parent via getppid
195 my $pid = $$;
196 my $ppid = getppid();
197 $self->logger()->prefix('[CHILD '.$job.q{ }.$pid.q{/}.$ppid.']');
198  
199 $self->logger->log( message => 'Fork for Job '.$job.' running ...', level => 'debug', );
200  
201 my $t0 = time(); # starttime
202 my $status = &{ $self->code() }( $job, $self->args() );
203 my $d0 = time() - $t0; # duration
204 if ($status) {
205 $self->logger->log( message => 'Fork finished with SUCCESS after running for ' . $d0 . 's.', level => 'debug', );
206 exit 0;
207 }
208 else {
209 $self->logger->log( message => 'Fork finished with FAILURE after running ' . $d0 . 's.', level => 'warning', );
210 exit 1;
211 }
212  
213 # end of fork(). The child _must_ exit here!
214 }
215 elsif ( $! == EAGAIN ) {
216  
217 # EAGAIN, probably temporary fork error
218 sleep 5;
219 redo FORK;
220 }
221 else {
222  
223 # Strange fork error
224 warn 'Can not exec fork: '.$!."\n";
225 }
226 } # FORK
227 } # if-forks-running-lt-concurrency
228 else {
229 $self->logger->log( message => 'Too many childs to spawn a new one (Running: '.$forks_running.' / Max: '.$concurrency.')', level => 'debug', );
230 sleep 1;
231 redo JOB;
232 }
233 } # end of foreach jobs
234 $self->logger()->log( message => 'Dispatched all childs. Waiting for them to finish ...', level => 'debug', );
235 my $child;
236 while ( ( $child = waitpid( -1, 0 ) ) > 0 ) {
237 $self->_job_status()->{$child} = $? >> 8;
238 delete( $childs_running{$child} );
239 $childs_returned++;
240 if ( $self->_job_status()->{$child} != 0 ) {
241 $ok = 0;
242 }
243 }
244 $self->logger()->log( message => '[PARENT] Collected all child stati.', level => 'debug', );
245 $self->logger()->prefix(q{});
246 if ($ok) {
247 $self->logger()->log( message => 'All childs returned w/o error', level => 'debug', );
248 return 1;
249 }
250 else {
251 $self->logger()->log( message => 'Some childs returned an error', level => 'error', );
252 return;
253 }
254 }
255  
256 ############################################
257 # Usage : none, called by $SIG{CHLD}
258 # Purpose : Collect zombies
259 # Returns : Number of zombies collected
260 # Parameters : none
261 # Throws : no exceptions
262 # Comments : none
263 # See Also : Programming Perl, p. 432
264 sub _reaper {
265 my $self = shift;
266  
267 $zombies = 0;
268 my $childs_finished = 0;
269 my $child;
270 while ( ( $child = waitpid( -1, WNOHANG ) ) > 0 ) {
271 $self->_job_status()->{$child} = $? >> 8;
272 delete( $childs_running{$child} );
273 $childs_finished++;
274 }
275 return $childs_finished;
276 }
277  
278 sub _sigterm {
279 #print "Received SIGTERM. Aborting running forks ...\n";
280  
281 # kill childs - kill(TERM, -$$):
282 my $cnt = kill( SIGTERM, q{-} . $$ );
283 say 'Signaled '.$cnt.' processes in current processgroup';
284 foreach my $child_pid ( keys %childs_running ) {
285 next unless $child_pid;
286 kill( SIGTERM, $child_pid );
287 say 'Signaled '.$child_pid;
288 }
289  
290 # die
291 exit;
292 }
293  
294 no Moose;
295 __PACKAGE__->meta->make_immutable;
296  
297 1;
298  
299 __END__
« ForkAsync.pm ↑ Summary