Analizo/Batch/Runner/Parallel.pm

filter: Statementstmt 77.7% 7 / 9 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 3 / 3 Podpod n/a totaltotal 83.3% 10 / 12 CC 1 SCAR 0.9
CC
1
Cov
78%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« Test/Analizo/BDD/Cucumber/Extension.pm ↑ Summary Analizo/Batch/Job/Directories.pm »
1 package Analizo::Batch::Runner::Parallel;
2 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 0 ▶use ZMQ::FFI qw(ZMQ_PUSH ZMQ_PULL ZMQ_REQ ZMQ_REP);
Subroutine BEGIN: called CC=1 SCAR=2.6
5 use YAML::XS;
6  
7 use parent qw( Analizo::Batch::Runner );
8  
9 $YAML::XS::LoadBlessed = 1;
10 $YAML::XS::LoadCode = 1;
11 $YAML::XS::DumpCode = 1;
12  
13 sub new {
14 my ($class, $parallelism) = @_;
15 $parallelism ||= 2;
16 $class->SUPER::new(parallelism => $parallelism);
17 }
18  
19 sub parallelism {
20 my ($self) = @_;
21 return $self->{parallelism};
22 }
23  
24 sub actually_run {
25 my ($self, $batch, $output) = @_;
26 $self->start_workers();
27 $self->coordinate_workers($batch, $output);
28 $self->wait_for_workers();
29 }
30  
31 sub _socket_spec {
32 my ($name, $ppid) = @_;
33 return "ipc:///tmp/.analizo-$name-$ppid";
34 }
35  
36 sub start_workers {
37 my ($self) = @_;
38 $self->{workers} = [];
39 my $n = $self->parallelism();
40 my $ppid = $$;
41 for my $i (1..$n) {
42 my $pid = fork();
43 if ($pid) {
44 # on parent
45 push(@{$self->{workers}}, $pid);
46 } else {
47 # on child
48 $0 = '[analizo worker]';
49 worker($ppid);
50 exit();
51 }
52 }
53 my $distributor_pid = fork();
54 if ($distributor_pid) {
55 push(@{$self->{workers}}, $distributor_pid);
56 } else {
57 $0 = '[analizo queue]';
58 distributor($ppid, $n);
59 exit();
60 }
61 }
62  
63 sub wait_for_workers {
64 my ($self) = @_;
65 for my $pid (@{$self->{workers}}) {
66 waitpid($pid, 0);
67 }
68 }
69  
70 sub coordinate_workers {
71 my ($self, $batch, $output) = @_;
72  
73 my $context = ZMQ::FFI->new();
74  
75 my $queue = $context->socket(ZMQ_PUSH);
76 $queue->bind(_socket_spec('queue', $$));
77  
78 my $results = $context->socket(ZMQ_PULL);
79 $results->bind(_socket_spec('results', $$));
80  
81 # push jobs to queue
82 my $results_expected = 0;
83 while (my $job = $batch->next()) {
84 $queue->send(Dump($job));
85 $results_expected++;
86 }
87 $queue->send(Dump({}));
88  
89 # collect results
90 my $results_received = 0;
91 while ($results_received < $results_expected) {
92 my $msg = $results->recv();
93 my $job = Load($msg);
94 $output->push($job);
95 $results_received++;
96 $self->report_progress($job, $results_received, $results_expected);
97 }
98 }
99  
100 sub distributor {
101 my ($parent_pid, $number_of_workers) = @_;
102 my $context = ZMQ::FFI->new();
103  
104 my $queue = $context->socket(ZMQ_PULL);
105 $queue->connect(_socket_spec('queue', $parent_pid));
106  
107 my $job_source = $context->socket(ZMQ_REP);
108 $job_source->bind(_socket_spec('job_source', $parent_pid));
109  
110 my @queue;
111 my $job;
112 while(1) {
113 my $msg = $queue->recv();
114 $job = Load($msg);
115 last if !exists($job->{id});
116 push(@queue, $job);
117 }
118  
119 my $workers_finished = 0;
120 while ($workers_finished < $number_of_workers) {
121 $job_source->recv();
122 if(scalar(@queue) > 0) {
123 $job = shift(@queue);
124 $job_source->send(Dump($job));
125 } else {
126 $job_source->send(Dump({}));
127 $workers_finished++;
128 }
129 }
130 }
131  
132 sub worker {
133 my ($parent_pid) = @_;
134 my $context = ZMQ::FFI->new();
135 my $source = $context->socket(ZMQ_REQ);
136 $source->connect(_socket_spec('job_source', $parent_pid));
137 my $results = $context->socket(ZMQ_PUSH);
138 $results->connect(_socket_spec('results', $parent_pid));
139 my $run = 1;
140 my $last_job = undef;
141 while ($run) {
142 $source->send('');
143 my $msg = $source->recv();
144 my $job = Load($msg);
145 if (exists($job->{id})) {
146 $last_job = $job;
147 $job->parallel_prepare();
148 $job->execute();
149 $results->send(Dump($job));
150 } else {
151 # a job without an id means that there are no more jobs to process, we
152 # should exit.
153 $run = 0;
154 }
155 }
156 if ($last_job) {
157 $last_job->parallel_cleanup();
158 }
159 }
160  
161 1;
« Test/Analizo/BDD/Cucumber/Extension.pm ↑ Summary Analizo/Batch/Job/Directories.pm »