CAB/Fork/Pool.pm untested

filter: Statementstmt 0.0% 0 / 108 Branchbran 0.0% 0 / 38 Conditioncond 0.0% 0 / 8 MC/DCmcdc 0.0% 0 / 5 Subroutinesub 0.0% 0 / 25 Podpod 0.0% 0 / 21 totaltotal 0.0% 0 / 205 CC 20 SCAR 26.0
CC
20
Cov
0%
spawn
42.0
serverMain
30.0
qclient
12.0
CRAP 13.4
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Fork::Pool.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: generic thread pool for DTA::CAB
6  
7 package DTA::CAB::Fork::Pool;
8 use DTA::CAB::Queue::Server;
9 use POSIX qw(:sys_wait_h);
10 use Carp;
11 use strict;
12  
13 ##==============================================================================
14 ## Globals
15 ##==============================================================================
16  
17 our @ISA = qw(DTA::CAB::Queue::Server);
18  
19 ##==============================================================================
20 ## Constructors etc.
21 ##==============================================================================
22  
23 ## $fp = CLASS_OR_OBJ->new(%args)
24 ## + %$fp, %args:
25 ## {
26 ## ##-- high-level subprocess pool options
27 ## njobs => $n_threads, ##-- number of subprocesses to fork() off (default=0)
28 ## init => \&init, ##-- called as init($fp) in each subprocess after creation (default: $fp->can('init'))
29 ## work => \&work, ##-- called as work($fp,$item) in a subprocess to process a queue item (default: $fp->can('work'))
30 ## reap => \&reap, ##-- called as reap($fp,$pid,$?) after subprocess exit (default: $fp->can('reap'))
31 ## free => \&free, ##-- called as free($fp) in each subprocess before termination (default: $fp->can('free'))
32 ##
33 ## logSpawn => $level, ##-- log-level for spawning subprocesses [default: 'info']
34 ## logReap => $level, ##-- log-level for own pids reaped with default reap() [default: 'info']
35 ## logReapOther => $level, ##-- log-level for other pids reaped with default reap() [default: none]
36 ## propagateErrors => $bool, ##-- if true (default), default 'reap' method will exit() the whole process
37 ## installReaper => $bool, ##-- if true (default), spawn() sets $SIG{CHLD}=$fp->reaper()
38 ##
39 ## ##-- Low-level data
40 ## pids => \@pids, ##-- PIDs of spawned subprocesses
41 ## ppid => $ppid, ##-- parent pid (default=$$)
42 ## qc => $qclient, ##-- queue client (subprocesses only; see $fp->qclient(), below)
43 ##
44 ## ##-- INHERITED from DTA::CAB::Queue::Server
45 ## queue => \@queue, ##-- actual queue data
46 ## status => $str, ##-- queue status (default: 'active')
47 ## ntok => $ntok, ##-- total number of tokens processed by clients
48 ## nchr => $nchr, ##-- total number of characters processed by clients
49 ## blocks => \%blocks, ##-- output tracker: %blocks = ($outfile => $po={cur=>$max_input_byte_written, pending=>\@pending}, ...)
50 ## ## + @pending is a list of pending blocks ($pb={off=>$ioffset, len=>$ilength, data=>\$data, ...}, ...)
51 ## logBlock => $level, ##-- log-level for block merge operations (default=undef (none))
52 ##
53 ## ##-- INHERITED from DTA::CAB::Socket::UNIX
54 ## local => $path, ##-- path to local UNIX socket (server only; set to empty string to use a tempfile): OVERRIDE default=''
55 ## #peer => $path, ##-- path to peer socket (client only)
56 ## listen => $n, ##-- queue size for listen (default=SOMAXCONN)
57 ## unlink => $bool, ##-- if true, server socket will be unlink()ed on DESTROY() (default=true)
58 ## perms => $perms, ##-- file create permissions for server socket (default=0600)
59 ## pid => $pid, ##-- pid of creating process (for auto-unlink)
60 ##
61 ## ##-- INHERITED from DTA::CAB::Socket
62 ## fh => $sockfh, ##-- an IO::Socket::UNIX object for the socket
63 ## timeout => $secs, ##-- default timeout for select() (default=undef: none); OVERRIDE default=1
64 ## nonblocking => $bool, ##-- set O_NONBLOCK on open? (override default=true); OVERRIDE default=1
65 ## logSocket => $level, ##-- log level for full socket trace (default=undef (none))
66 ## logRequest => $level, ##-- log level for client requests (server only; default=undef (none))
67 ## }
68 sub new {
69 my $that = shift;
70 my $fp = $that->SUPER::new
71 (
72 njobs => 0,
73 init => $that->can('init'),
74 work => $that->can('work'),
75 reap => $that->can('reap'),
76 free => $that->can('free'),
77 propagateErrors => 1,
78 installReaper => 1,
79 pids => [],
80 ppid => $$,
81  
82 ##-- logging
83 logSpawn => 'info',
84 logReap => 'info',
85 logReapOther => 'debug',
86 #logSocket => 'debug',
87 #logRequest => 'trace',
88 #logBlock => 'debug',
89  
90 ##-- overrides
91 local => '',
92 timeout => 1,
93 nonblocking => 1,
94  
95 @_,
96 );
97 return $fp;
98 }
99  
100 ##==============================================================================
101 ## Methods: Pool Maintainence
102  
103 ## $fp = $fp->spawn()
104 ## + ensures that at least $fp->{njobs} PIDs are defined in $fp->{pids}
105 ## + you must completely populate the queue BEFORE calling this method!
106 sub spawn {
107 my $fp = shift;
108 $fp->{pids} = [] if (!$fp->{pids});
109 $SIG{CHLD} = $fp->reaper() if ($fp->{installReaper});
110 while (@{$fp->{pids}} < $fp->{njobs}) {
111 my $pid = fork();
112 if ($pid) {
113 ##-- parent
114 $fp->vlog($fp->{logSpawn},"spawned subprocess $pid");
115 push(@{$fp->{pids}},$pid);
116 } else {
117 ##-- child
118 $fp->logconfess("couldn't fork()") if (!defined($pid));
119 $fp->{fh}->close() if ($fp->{fh}); ##-- close server queue socket in child
120 exit $fp->childMain();
121 }
122 }
123 return $fp;
124 }
125  
126 ## $fp = $fp->is_child()
127 ## + returns true if current process is a child; wraps ($fp->{ppid} && $$!=$fp->{ppid})
128 sub is_child {
129 return $_[0]{ppid} && $$!=$_[0]{ppid};
130 }
131  
132 ## $fp = $fp->abort()
133 ## $fp = $fp->abort($SIGNAL)
134 ## + kills subprocesses
135 sub abort {
136 my ($fp,$sig) = @_;
137 $sig = 'TERM' if (!defined($sig));
138 my ($pid);
139 while (defined($pid=shift(@{$fp->{pids}}))) {
140 kill($sig,$pid);
141 }
142 return $fp;
143 }
144  
145 ## $fp = $fp->reset()
146 ## + aborts all running subprocesses and empties the queue
147 sub reset {
148 my $fp = shift;
149 $fp->abort();
150 $fp->clear();
151 return $fp;
152 }
153  
154 ## @pids = $fp->pids
155 ## + list of all child processes
156 sub pids {
157 return @{$_[0]{pids}};
158 }
159  
160 ## $fp = $fp->waitall()
161 ## + waits on all child pids
162 sub waitall {
163 my $fp = shift;
164 my ($child);
165 while (($child = waitpid(-1,0))>0) {
166 $fp->{reap}->($fp,$child,$?) if ($fp->{reap});
167 @{$fp->{pids}} = grep {$_ != $child} @{$fp->{pids}};
168 }
169 return $fp;
170 }
171  
172 ## \&reaper = $fp->reaper()
173 ## + zombie-harvesting code; installed to local %SIG if $fp->{installReaper} is true (default)
174 sub reaper {
175 my $fp = shift;
176 return sub {
177 my ($child);
178 while (($child = waitpid(-1,WNOHANG)) > 0) {
179 if ($fp->{reap}) {
180 $fp->{reap}->($fp,$child,$?) ;
181 } else {
182 $fp->vlog($fp->{logReap},"reaper got subprocess pid=$child, status=$?");
183 }
184 @{$fp->{pids}} = grep {$_ != $child} @{$fp->{pids}};
185 }
186 #$SIG{CHLD}=$fp->reaper() if ($fp->{installReaper}); ##-- re-install reaper for SysV
187 };
188 }
189  
190 ##==============================================================================
191 ## Methods: Queue Maintainence
192 ## + see DTA::CAB::Queue::Server
193  
194 ## $client = $fp->qclient()
195 ## + returns a (new) pseudo-client for the $fp queue
196 ## + just returns $fp in main process
197 ## + uses cached $fp->{qc} if available, otherwise caches $fp->{qc} to a new DTA::CAB::Queue::Client
198 sub qclient {
199 return $_[0] if ($$ == $_[0]{ppid}); ##-- main process: use $fp methods directly
200 return $_[0]{qc} if (defined($_[0]{qc})); ##-- child with cached client object
201 return $_[0]{qc} = DTA::CAB::Queue::Client->new(peer=>$_[0]{local}, logSocket=>$_[0]{logSocket});
202 }
203  
204 ## undef = $fp->qenq($item)
205 ## + enqueue a single item; server or client
206 sub qenq {
207 $_[0]->qclient->enq(@_[1..$#_]);
208 }
209  
210 ## $item = $fp->qdeq()
211 ## + dequeue a single item; server or client
212 sub qdeq {
213 $_[0]->qclient->deq(@_[1..$#_]);
214 }
215  
216 ## $size = $fp->qsize()
217 sub qsize {
218 $_[0]->qclient->size;
219 }
220  
221 ## undef = $fp->qaddcounts($ntok,$nchr)
222 sub qaddcounts {
223 $_[0]->qclient->addcounts(@_[1..$#_]);
224 }
225  
226 ## undef = $fp->qaddblock(\%blk)
227 sub qaddblock {
228 $_[0]->qclient->addblock(@_[1..$#_]);
229 }
230  
231 ##==============================================================================
232 ## Methods: Main Server Loop
233  
234 ## undef = $fp->serverMain()
235 ## + main server loop
236 ## + spawns sub-process(es) if $fp->{njobs}>=1
237 ## + serves the queue until all subprocesses have existed
238 sub serverMain {
239 my $fp = shift;
240  
241 ##-- do we need to fork at all?
242 if ($fp->{njobs} < 1) {
243 $fp->info("requested njobs=$fp->{njobs}; not forking");
244 return $fp->processMain();
245 }
246  
247 ##-- sanity check(s)
248 $fp->open() if (!defined($fp->{fh})); ##-- ensure server socket is open
249 $fp->{njobs} = $fp->size if ($fp->{njobs} > $fp->size); ##-- don't fork() more than we need to
250 $fp->spawn();
251 $fp->info("spawned $fp->{njobs} worker subprocess(es)");
252  
253 ##-- serve the queue
254 my ($cli);
255 while (@{$fp->{pids}}) {
256 #$fp->trace("polling for client(s)..."); ##-- DEBUG
257 next if (!defined($cli=$fp->accept));
258 $fp->handleClient($cli);
259 $cli->close();
260 }
261  
262 return;
263 }
264  
265 ##==============================================================================
266 ## Methods: Job Processing
267  
268 ## undef = PACKAGE::childMain($fp)
269 ## + queue worker sub which wraps init() and free() calls around $fp->processMain()
270 sub childMain {
271 my $fp = shift;
272 $fp->{init}->($fp) if ($fp->{init});
273 $fp->processMain();
274 $fp->{free}->($fp) if ($fp->{free});
275 exit 0;
276 }
277  
278 ## undef = PACKAGE::processMain($fp)
279 ## + main queue processing loop (can be called either from main or child thread)
280 sub processMain {
281 my $fp = shift;
282 my ($item);
283 while (defined($item=$fp->qdeq)) {
284 $fp->{work}->($fp,$item) if ($fp->{work});
285 }
286 }
287  
288 ##==============================================================================
289 ## Methods: Default Child Callbacks
290  
291 ## undef = init($fp)
292 ## + default thread initializer (no-op)
293 sub init { ; }
294  
295 ## undef = work($fp,$item)
296 ## + default thread queue-processor (no-op)
297 sub work { ; }
298  
299 ## undef = free($fp)
300 ## + default thread cleanup function (no-op)
301 sub free { ; }
302  
303 ## undef = reap($fp,$pid,$?)
304 ## + called from main thread on subprocess exit
305 sub reap {
306 my ($fp,$pid,$status) = @_;
307 if (grep {$_==$pid} @{$fp->{pids}//[]}) {
308 $fp->vlog($fp->{logReap},"reaped subprocess $pid with exit status $status");
309 if ($fp->{propagateErrors} && $status != 0) {
310 $fp->abort();
311 $fp->logdie("subprocess $pid exited with abnormal status $status");
312 }
313 } else {
314 $fp->vlog($fp->{logReapOther},"reaped external subprocess $pid with exit status $status");
315 }
316 }
317  
318  
319  
320 1; ##-- be happy
321  
322 __END__