Stepford/Runner.pm

filter: Statementstmt 98.8% 174 / 176 Branchbran 92.1% 35 / 38 Conditioncond 66.6% 2 / 3 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 97.2% 35 / 36 Podpod 50.0% 1 / 2 totaltotal 96.1% 247 / 257 CC 23 SCAR 9.3
CC
23
Cov
97%
__ANON__
6.0
__ANON__
5.0
_build_step_classes
4.0
CRAP 2.5
« Stepford/Graph.pm ↑ Summary Stepford/Role/Step/FileGenerator/Atomic.pm »
1 package Stepford::Runner;
2  
3 39 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 39 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 39 ▶use namespace::autoclean;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 39 our $VERSION = '0.006001';
8  
9 39 ▶use List::AllUtils qw( first );
Subroutine BEGIN: called CC=1 SCAR=0.0
10 39 ▶use Module::Pluggable::Object;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 39 ▶use MooseX::Params::Validate qw( validated_list );
Subroutine BEGIN: called CC=1 SCAR=0.0
12 39 ▶use Parallel::ForkManager;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 39 ▶use Stepford::Error;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 39 ▶use Stepford::GraphBuilder;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 39 ▶use Stepford::Types qw(
Subroutine BEGIN: called CC=1 SCAR=0.0
16 ArrayOfClassPrefixes ArrayOfSteps Bool ClassName
17 HashRef Logger Maybe PositiveInt Step
18 39 );
19 39 ▶use Try::Tiny;
Subroutine BEGIN: called CC=1 SCAR=0.0
20  
21 39 ▶use Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
22 39 ▶use MooseX::StrictConstructor;
Subroutine BEGIN: called CC=1 SCAR=0.0
23  
24 39 has _step_namespaces => (
25 traits => ['Array'],
26 is => 'ro',
27 isa => ArrayOfClassPrefixes,
28 coerce => 1,
29 required => 1,
30 init_arg => 'step_namespaces',
31 handles => {
32 step_namespaces => 'elements',
33 },
34 );
35  
36 39 has logger => (
37 is => 'ro',
38 isa => Logger,
39 lazy => 1,
40 builder => '_build_logger',
41 );
42  
43 39 has jobs => (
44 is => 'ro',
45 isa => PositiveInt,
46 default => 1,
47 );
48  
49 has _memory_stats => (
50 is => 'ro',
51 isa => Maybe ['Memory::Stats'],
52 145 ▶ default => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
53 145 ▶ return try {
Subroutine __ANON__: called CC=1 SCAR=0.0
54 145 require Memory::Stats;
55 145 my $s = Memory::Stats->new;
56 145 $s->start;
57 145 $s;
58 }
59 0 ▶ catch {
Subroutine __ANON__: not called CC=1 SCAR=6.9
60 0 undef;
61 145 };
62 },
63 39 );
64  
65 39 has _step_classes => (
66 is => 'ro',
67 isa => ArrayOfSteps,
68 init_arg => undef,
69 lazy => 1,
70 builder => '_build_step_classes',
71 );
72  
73 # We want to preload all the step classes so that the final_steps passed to
74 # run are recognized as valid classes.
75 145 ▶sub BUILD {
Subroutine BUILD: called CC=1 SCAR=0.0
Pod undocumented
76 145 my $self = shift;
77  
78 145 $self->_step_classes;
79  
80 144 return;
81 }
82  
83 146 ▶sub run {
Subroutine run: called CC=3 SCAR=11.0
Pod documented
84 146 my $self = shift;
85 146 my ( $final_steps, $config, $force_step_execution, $dry_run )
86 = validated_list(
87 \@_,
88 final_steps => {
89 isa => ArrayOfSteps,
90 coerce => 1,
91 },
92 config => {
93 isa => HashRef,
94 default => {},
95 },
96 force_step_execution => {
97 isa => Bool,
98 default => 0,
99 },
100 dry_run => {
101 isa => Bool,
102 default => 'none',
103 },
104 );
105  
106 146 my $root_graph
107 = $self->_make_root_graph_builder( $final_steps, $config )->graph;
108  
109 143 ▶ if ( $dry_run ne 'none' ) {
Branch: 2 branches
BranchTrueFalseTotal
if ($dry_run ne 'none') { } 1 142 143
elsif ($self->jobs > 1) { } 77 65 142
110 ## no critic (InputOutput::RequireCheckedSyscalls)
111 1 print $root_graph->as_string($dry_run);
112 }
113 elsif ( $self->jobs > 1 ) {
114 77 $self->_run_parallel( $root_graph, $force_step_execution );
115 }
116 else {
117 65 $self->_run_sequential( $root_graph, $force_step_execution );
118 }
119  
120 108 return;
121 }
122  
123 65 ▶sub _run_sequential {
Subroutine _run_sequential: called CC=1 SCAR=0.0
124 65 my $self = shift;
125 65 my $root_graph = shift;
126 65 my $force_step_execution = shift;
127  
128 314 ▶ $root_graph->traverse( sub {
Subroutine __ANON__: called CC=2 SCAR=6.9
129 314 my $graph = shift;
130  
131 return
132 314 ▶ unless $self->_should_run_step(
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_should_run_step($graph, $force_step_execution) 133 181 314
133 $graph,
134 $force_step_execution,
135 );
136  
137 181 $self->_run_step_in_process($graph);
138 181 return;
139 65 } );
140 }
141  
142 77 ▶sub _run_parallel {
Subroutine _run_parallel: called CC=3 SCAR=11.0
143 77 my $self = shift;
144 77 my $root_graph = shift;
145 77 my $force_step_execution = shift;
146  
147 77 my $pm = Parallel::ForkManager->new( $self->jobs );
148  
149 77 my %graphs;
150 77 my $steps_finished_since_last_iteration = 0;
151  
152 167 ▶ $pm->run_on_finish( sub {
Subroutine __ANON__: called CC=6 SCAR=17.9
153 167 my ( $pid, $exit_code, $step_name, $signal, $message )
154 = @_[ 0 .. 3, 5 ];
155  
156 167 ▶ if ($exit_code) {
Branch: 3 branches
BranchTrueFalseTotal
if ($exit_code) { } 1 166 167
elsif (not $message) { } 3 163 166
elsif ($message->{'error'}) { } 4 159 163
157 1 _kill_all_children($pm);
158 1 die "Child process $pid failed while running step $step_name"
159 . " (exited with code $exit_code)";
160 }
161 elsif ( !$message ) {
162 3 _kill_all_children($pm);
163 3 my $error = "Child process $pid did not send back any data";
164 3 $error .= " while running step $step_name";
165 3 ▶ $error .= " (exited because of signal $signal)" if $signal;
Branch: 1 branch
BranchTrueFalseTotal
if $signal 3 0 3
166 3 die $error;
167 }
168 elsif ( $message->{error} ) {
169 4 _kill_all_children($pm);
170 4 die "Child process $pid died"
171 . " while running step $step_name"
172 . " with error:\n$message->{error}";
173 }
174 else {
175 159 my $graph = $graphs{$pid};
176 159 ▶ die "Could not find step graph for $pid"
Branch: 1 branch
BranchTrueFalseTotal
unless defined $graph 0 159 159
177 unless defined $graph;
178  
179 159 $graph->set_last_run_time( $message->{last_run_time} );
180 $graph->set_step_productions_as_hashref(
181 159 $message->{productions} );
182 159 $graph->set_is_being_processed(0);
183 159 $graph->set_has_been_processed(1);
184 159 $steps_finished_since_last_iteration++;
185 }
186 77 } );
187  
188 77 ▶ while ( !$root_graph->has_been_processed ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($root_graph->has_been_processed) 195 42 237
189 901 ▶ $root_graph->traverse( sub {
Subroutine __ANON__: called CC=5 SCAR=16.1
190 901 my $graph = shift;
191  
192 901 my $class = $graph->step_class;
193  
194 return
195 901 ▶ unless $self->_should_run_step(
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_should_run_step($graph, $force_step_execution) 700 201 901
196 $graph,
197 $force_step_execution,
198 );
199  
200 201 ▶ unless ( $graph->is_serializable ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($graph->is_serializable) 4 197 201
201 4 $self->_run_step_in_process($graph);
202 4 $steps_finished_since_last_iteration++;
203 4 return;
204 }
205  
206 197 ▶ if ( my $pid = $pm->start($class) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $pid = $pm->start($class)) 170 27 197
207 170 $graph->set_is_being_processed(1);
208  
209 170 $graphs{$pid} = $graph;
210  
211 170 $self->logger->debug("Forked child to run $class - pid $pid");
212 170 return;
213 }
214  
215 # child
216 27 my $error;
217 27 ▶ try {
Subroutine __ANON__: called CC=1 SCAR=0.0
218 27 $self->_run_step_in_process($graph);
219 }
220 1 ▶ catch {
Subroutine __ANON__: called CC=1 SCAR=0.0
221 1 $error = $_;
222 27 };
223  
224 26 ▶ my %message
Branch: 1 branch
BranchTrueFalseTotal
defined $error ? : 1 25 26
225 = defined $error
226 ? ( error => $error . q{} )
227 : (
228 last_run_time => scalar $graph->last_run_time,
229 productions => $graph->step_productions_as_hashref,
230 );
231  
232 26 $pm->finish( 0, \%message );
233 195 } );
234  
235 168 $pm->reap_finished_children;
236  
237 # If none of the child processes have finished, there is no point in
238 # wasting a bunch of CPU checking for steps that can be worked on. We
239 # wait until at least one step has finished.
240 168 ▶ $pm->wait_one_child unless $steps_finished_since_last_iteration;
Branch: 1 branch
BranchTrueFalseTotal
unless $steps_finished_since_last_iteration 141 27 168
241  
242 160 $steps_finished_since_last_iteration = 0;
243 }
244  
245 42 $self->logger->debug('Waiting for children');
246 42 $pm->wait_all_children;
247 }
248  
249 8 ▶sub _kill_all_children {
Subroutine _kill_all_children: called CC=2 SCAR=7.6
250 8 my $pm = shift;
251  
252 8 for my $pid ( $pm->running_procs ) {
253  
254 # This is a best-effort attempt to kill direct children.
255 ## no critic (RequireCheckedSyscalls)
256 0 kill 'TERM', $pid;
257 }
258 }
259  
260 1215 ▶sub _should_run_step {
Subroutine _should_run_step: called CC=3 SCAR=11.0
261 1215 my $self = shift;
262 1215 my $graph = shift;
263 1215 my $force_step_execution = shift;
264  
265 1215 ▶ return 0 unless $graph->can_run_step;
Branch: 1 branch
BranchTrueFalseTotal
unless $graph->can_run_step 596 619 1215
266  
267 619 ▶ if ( $graph->step_is_up_to_date($force_step_execution) ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($graph->step_is_up_to_date($force_step_execution)) 237 382 619
268 237 $self->logger->info( 'Skipping ' . $graph->step_class );
269 237 $graph->set_has_been_processed(1);
270 237 return 0;
271 }
272  
273 382 return 1;
274 }
275  
276 212 ▶sub _run_step_in_process {
Subroutine _run_step_in_process: called CC=1 SCAR=0.0
277 212 my $self = shift;
278 212 my $graph = shift;
279  
280 212 $self->_log_memory_usage( 'Before running ' . $graph->step_class );
281 212 $graph->run_step;
282 210 $self->_log_memory_usage( 'After running ' . $graph->step_class );
283  
284 210 return;
285 }
286  
287 151 ▶sub _make_root_graph_builder {
Subroutine _make_root_graph_builder: called CC=1 SCAR=0.0
288 151 my $self = shift;
289 151 my $final_steps = shift;
290 151 my $config = shift;
291  
292 151 return Stepford::GraphBuilder->new(
293 config => $config,
294 step_classes => $self->_step_classes,
295 final_steps => $final_steps,
296 logger => $self->logger,
297 );
298 }
299  
300 145 ▶sub _build_step_classes {
Subroutine _build_step_classes: called CC=4 SCAR=13.9
301 145 my $self = shift;
302  
303 # Module::Pluggable does not document whether it returns class names in
304 # any specific order.
305 145 my $sorter = $self->_step_class_sorter;
306  
307 145 my @classes;
308  
309 145 for my $class (
310 653 sort { $sorter->() } Module::Pluggable::Object->new(
311 search_path => [ $self->step_namespaces ],
312 require => 1,
313 )->plugins
314 ) {
315  
316 # We need to skip roles
317 602 ▶ next unless $class->isa('Moose::Object');
Branch: 1 branch
BranchTrueFalseTotal
unless $class->isa("Moose::Object") 1 601 602
318  
319 601 ▶ unless ( $class->does('Stepford::Role::Step') ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($class->does("Stepford::Role::Step")) 1 600 601
320 1 Stepford::Error->throw( message =>
321 qq{Found a class which doesn't do the Stepford::Role::Step role: $class}
322 );
323 }
324  
325 600 $self->logger->debug("Found step class $class");
326 600 push @classes, $class;
327 }
328  
329 144 return \@classes;
330 }
331  
332 145 ▶sub _step_class_sorter {
Subroutine _step_class_sorter: called CC=1 SCAR=0.0
333 145 my $self = shift;
334  
335 145 my $x = 0;
336 145 my @namespaces = $self->step_namespaces;
337 145 my %order = map { $_ => $x++ } @namespaces;
338  
339 653 ▶ return sub {
Subroutine __ANON__: called CC=2 SCAR=7.0
340 653 ▶ my $a_prefix = first { $a =~ /^\Q$_/ } @namespaces;
Subroutine __ANON__: called CC=1 SCAR=0.0
341 653 ▶ my $b_prefix = first { $b =~ /^\Q$_/ } @namespaces;
Subroutine __ANON__: called CC=1 SCAR=0.0
342  
343 653 ▶ return ( $order{$a_prefix} <=> $order{$b_prefix} or $a cmp $b );
Condition66%2/3
$order{$a_prefix} <=> $order{$b_prefix} || $a cmp $b
l!l&&r!l&&!r
26510
MC/DC0%0/2
$order{$a_prefix} <=> $order{$b_prefix} || $a cmp $b
$order{$a_prefix} <=> $order{$b_prefix} $a cmp $b
Truth table66%2/3
A || B
A= $order{$a_prefix} <=> $order{$b_prefix}B= $a cmp $b
ABresult
1X1
011
000
344 145 };
345 }
346  
347 130 ▶sub _build_logger {
Subroutine _build_logger: called CC=1 SCAR=0.0
348 130 my $self = shift;
349  
350 130 require Log::Dispatch;
351 130 require Log::Dispatch::Null;
352 130 return Log::Dispatch->new(
353 outputs => [ [ Null => min_level => 'emerg' ] ] );
354 }
355  
356 422 ▶sub _log_memory_usage {
Subroutine _log_memory_usage: called CC=2 SCAR=7.0
357 422 my $self = shift;
358 422 my $checkpoint = shift;
359  
360 422 ▶ return unless $self->_memory_stats;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->_memory_stats 0 422 422
361  
362 422 $self->_memory_stats->checkpoint($checkpoint);
363  
364 # There's no way to get the total use so far without calling stop(), which
365 # is quite annoying. See
366 # https://github.com/celogeek/perl-memory-stats/issues/3.
367  
368 ## no critic (Subroutines::ProtectPrivateSubs)
369 422 $self->logger->info( sprintf(
370 'Total memory use since Stepford started is %d',
371 $self->_memory_stats->_memory_usage->[-1][1]
372 - $self->_memory_stats->_memory_usage->[0][1],
373 ) );
374 422 $self->logger->info( sprintf(
375 'Memory since last checked is %d',
376 $self->_memory_stats->delta_usage,
377 ) );
378 }
379  
380 39 __PACKAGE__->meta->make_immutable;
381  
382 39 1;
383  
384 # ABSTRACT: Takes a set of steps and figures out what order to run them in
385  
386 __END__
« Stepford/Graph.pm ↑ Summary Stepford/Role/Step/FileGenerator/Atomic.pm »