| 1 | package Analizo::Batch::Runner::Parallel; | ||
| 2 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 0 | ▶ | use ZMQ::FFI qw(ZMQ_PUSH ZMQ_PULL ZMQ_REQ ZMQ_REP); |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 5 | use YAML::XS; | ||
| 6 | |||
| 7 | use parent qw( Analizo::Batch::Runner ); | ||
| 8 | |||
| 9 | $YAML::XS::LoadBlessed = 1; | ||
| 10 | $YAML::XS::LoadCode = 1; | ||
| 11 | $YAML::XS::DumpCode = 1; | ||
| 12 | |||
| 13 | sub new { | ||
| 14 | my ($class, $parallelism) = @_; | ||
| 15 | $parallelism ||= 2; | ||
| 16 | $class->SUPER::new(parallelism => $parallelism); | ||
| 17 | } | ||
| 18 | |||
| 19 | sub parallelism { | ||
| 20 | my ($self) = @_; | ||
| 21 | return $self->{parallelism}; | ||
| 22 | } | ||
| 23 | |||
| 24 | sub actually_run { | ||
| 25 | my ($self, $batch, $output) = @_; | ||
| 26 | $self->start_workers(); | ||
| 27 | $self->coordinate_workers($batch, $output); | ||
| 28 | $self->wait_for_workers(); | ||
| 29 | } | ||
| 30 | |||
| 31 | sub _socket_spec { | ||
| 32 | my ($name, $ppid) = @_; | ||
| 33 | return "ipc:///tmp/.analizo-$name-$ppid"; | ||
| 34 | } | ||
| 35 | |||
| 36 | sub start_workers { | ||
| 37 | my ($self) = @_; | ||
| 38 | $self->{workers} = []; | ||
| 39 | my $n = $self->parallelism(); | ||
| 40 | my $ppid = $$; | ||
| 41 | for my $i (1..$n) { | ||
| 42 | my $pid = fork(); | ||
| 43 | if ($pid) { | ||
| 44 | # on parent | ||
| 45 | push(@{$self->{workers}}, $pid); | ||
| 46 | } else { | ||
| 47 | # on child | ||
| 48 | $0 = '[analizo worker]'; | ||
| 49 | worker($ppid); | ||
| 50 | exit(); | ||
| 51 | } | ||
| 52 | } | ||
| 53 | my $distributor_pid = fork(); | ||
| 54 | if ($distributor_pid) { | ||
| 55 | push(@{$self->{workers}}, $distributor_pid); | ||
| 56 | } else { | ||
| 57 | $0 = '[analizo queue]'; | ||
| 58 | distributor($ppid, $n); | ||
| 59 | exit(); | ||
| 60 | } | ||
| 61 | } | ||
| 62 | |||
| 63 | sub wait_for_workers { | ||
| 64 | my ($self) = @_; | ||
| 65 | for my $pid (@{$self->{workers}}) { | ||
| 66 | waitpid($pid, 0); | ||
| 67 | } | ||
| 68 | } | ||
| 69 | |||
| 70 | sub coordinate_workers { | ||
| 71 | my ($self, $batch, $output) = @_; | ||
| 72 | |||
| 73 | my $context = ZMQ::FFI->new(); | ||
| 74 | |||
| 75 | my $queue = $context->socket(ZMQ_PUSH); | ||
| 76 | $queue->bind(_socket_spec('queue', $$)); | ||
| 77 | |||
| 78 | my $results = $context->socket(ZMQ_PULL); | ||
| 79 | $results->bind(_socket_spec('results', $$)); | ||
| 80 | |||
| 81 | # push jobs to queue | ||
| 82 | my $results_expected = 0; | ||
| 83 | while (my $job = $batch->next()) { | ||
| 84 | $queue->send(Dump($job)); | ||
| 85 | $results_expected++; | ||
| 86 | } | ||
| 87 | $queue->send(Dump({})); | ||
| 88 | |||
| 89 | # collect results | ||
| 90 | my $results_received = 0; | ||
| 91 | while ($results_received < $results_expected) { | ||
| 92 | my $msg = $results->recv(); | ||
| 93 | my $job = Load($msg); | ||
| 94 | $output->push($job); | ||
| 95 | $results_received++; | ||
| 96 | $self->report_progress($job, $results_received, $results_expected); | ||
| 97 | } | ||
| 98 | } | ||
| 99 | |||
| 100 | sub distributor { | ||
| 101 | my ($parent_pid, $number_of_workers) = @_; | ||
| 102 | my $context = ZMQ::FFI->new(); | ||
| 103 | |||
| 104 | my $queue = $context->socket(ZMQ_PULL); | ||
| 105 | $queue->connect(_socket_spec('queue', $parent_pid)); | ||
| 106 | |||
| 107 | my $job_source = $context->socket(ZMQ_REP); | ||
| 108 | $job_source->bind(_socket_spec('job_source', $parent_pid)); | ||
| 109 | |||
| 110 | my @queue; | ||
| 111 | my $job; | ||
| 112 | while(1) { | ||
| 113 | my $msg = $queue->recv(); | ||
| 114 | $job = Load($msg); | ||
| 115 | last if !exists($job->{id}); | ||
| 116 | push(@queue, $job); | ||
| 117 | } | ||
| 118 | |||
| 119 | my $workers_finished = 0; | ||
| 120 | while ($workers_finished < $number_of_workers) { | ||
| 121 | $job_source->recv(); | ||
| 122 | if(scalar(@queue) > 0) { | ||
| 123 | $job = shift(@queue); | ||
| 124 | $job_source->send(Dump($job)); | ||
| 125 | } else { | ||
| 126 | $job_source->send(Dump({})); | ||
| 127 | $workers_finished++; | ||
| 128 | } | ||
| 129 | } | ||
| 130 | } | ||
| 131 | |||
| 132 | sub worker { | ||
| 133 | my ($parent_pid) = @_; | ||
| 134 | my $context = ZMQ::FFI->new(); | ||
| 135 | my $source = $context->socket(ZMQ_REQ); | ||
| 136 | $source->connect(_socket_spec('job_source', $parent_pid)); | ||
| 137 | my $results = $context->socket(ZMQ_PUSH); | ||
| 138 | $results->connect(_socket_spec('results', $parent_pid)); | ||
| 139 | my $run = 1; | ||
| 140 | my $last_job = undef; | ||
| 141 | while ($run) { | ||
| 142 | $source->send(''); | ||
| 143 | my $msg = $source->recv(); | ||
| 144 | my $job = Load($msg); | ||
| 145 | if (exists($job->{id})) { | ||
| 146 | $last_job = $job; | ||
| 147 | $job->parallel_prepare(); | ||
| 148 | $job->execute(); | ||
| 149 | $results->send(Dump($job)); | ||
| 150 | } else { | ||
| 151 | # a job without an id means that there are no more jobs to process, we | ||
| 152 | # should exit. | ||
| 153 | $run = 0; | ||
| 154 | } | ||
| 155 | } | ||
| 156 | if ($last_job) { | ||
| 157 | $last_job->parallel_cleanup(); | ||
| 158 | } | ||
| 159 | } | ||
| 160 | |||
| 161 | 1; | ||