| 1 | package Peta::NN::Parallel; | ||
| 2 | # ABSTRACT: independent pieces of work on several cores | ||
| 3 | |||
| 4 | # Independent pieces of work on several cores: each task runs in a process | ||
| 5 | # of its own (Parallel::ForkManager) and hands its result back as plain data. | ||
| 6 | # | ||
| 7 | # This is coarse on purpose. Training runs of different models, of different | ||
| 8 | # widths of one model, or of one width on different seeds have nothing to do | ||
| 9 | # with each other, and each takes seconds to minutes; a process per run costs | ||
| 10 | # nothing beside that. A child starts as a copy of its parent, so what the | ||
| 11 | # parent has loaded (word lists, training pairs) is there without being sent. | ||
| 12 | |||
| 13 | use v5.36; | ||
| 14 | |||
| 15 | use Exporter qw(import); | ||
| 16 | |||
| 17 | our $VERSION = '0.2610090'; | ||
| 18 | our @EXPORT_OK = qw(in_parallel workers); | ||
| 19 | |||
| 20 | # How many processes to use when the caller leaves it open: PETA_NN_WORKERS, | ||
| 21 | # or 1, which is no parallelism at all. | ||
| 22 | sub workers () { | ||
| 23 | my $asked = $ENV{PETA_NN_WORKERS} // 1; | ||
| 24 | die "PETA_NN_WORKERS is a whole number of 1 or more, not '$asked'\n" if $asked !~ /\A[1-9][0-9]*\z/; | ||
| 25 | return $asked; | ||
| 26 | } | ||
| 27 | |||
| 28 | # Run the tasks, at most $workers at a time, and return what they returned, | ||
| 29 | # in the order of the tasks. A task is a sub; what it returns must be plain | ||
| 30 | # data (no objects, no code), since it crosses from one process to another. | ||
| 31 | # With one worker, or one task, everything runs here, in order. | ||
| 32 | # A task that dies takes the whole call with it, after the others are done. | ||
| 33 | sub in_parallel ($workers, @tasks) { | ||
| 34 | return map { scalar $_->() } @tasks if $workers <= 1 || @tasks <= 1; | ||
| 35 | |||
| 36 | require Parallel::ForkManager; | ||
| 37 | my $manager = Parallel::ForkManager->new($workers < @tasks ? $workers : scalar @tasks); | ||
| 38 | my (@result, @failed); | ||
| 39 | $manager->run_on_finish(sub ($pid, $code, $n, $signal, $core, $data) { | ||
| 40 | if ($signal || $code || !$data) { push @failed, "task $n ended with " . ($signal ? "signal $signal" : "status $code") } | ||
| 41 | elsif (defined $data->{error}) { push @failed, "task $n: $data->{error}" } | ||
| 42 | else { $result[$n] = $data->{value} } | ||
| 43 | }); | ||
| 44 | for my $n (0 .. $#tasks) { | ||
| 45 | $manager->start($n) and next; | ||
| 46 | my $value = eval { scalar $tasks[$n]->() }; | ||
| 47 | $manager->finish(0, defined $value || !$@ ? { value => $value } : { error => $@ =~ s/\s+\z//r }); | ||
| 48 | } | ||
| 49 | $manager->wait_all_children; | ||
| 50 | die join("\n", @failed) . "\n" if @failed; | ||
| 51 | return @result; | ||
| 52 | } | ||
| 53 | |||
| 54 | 1; | ||
| 55 | |||
| 56 | __END__ |