blib/lib/Peta/NN/Job.pm untested

filter: Statementstmt 0.0% 0 / 291 Branchbran 0.0% 0 / 128 Conditioncond 0.0% 0 / 120 MC/DCmcdc 0.0% 0 / 61 Subroutinesub 0.0% 0 / 25 Podpod 0.0% 0 / 20 totaltotal 0.0% 0 / 645 CC 65 SCAR 48.0
CC
65
Cov
0%
_stage
342.0
report
210.0
run
110.0
CRAP 121.8
1 package Peta::NN::Job;
2 # ABSTRACT: train a model until it meets given fidelity thresholds, at the smallest size that can
3  
4 # An optimisation job: say what a model must achieve and what it may cost,
5 # and get the smallest model that does.
6 #
7 # A job trains ONE model until it has got there. It does not run from a model
8 # that falls short to another one:
9 #
10 # train until every threshold is met, for as long as the model keeps
11 # getting closer to them. What is watched is the distance to the
12 # thresholds, not the loss: a model that still has three of its
13 # exceptions to learn is not done because its loss has gone flat.
14 # Training on is gentler than training: a lower learning rate,
15 # so that the model keeps what it has while it learns the rest.
16 # weigh a model that has stopped getting closer while a subset it must
17 # retain is still short is trained on with more weight on that
18 # subset
19 # widen a model that has stopped getting closer all the same has
20 # learned what its size allows. It is given wider hidden layers,
21 # which changes none of its answers (Peta::NN::Model's widen),
22 # and trains on with what it knows
23 #
24 # The job starts at the smallest width it is allowed and widens by `grow`, so
25 # the model that gets there is the smallest that does, to within that factor.
26 # A second seed then has to get there as well, from that width: one lucky
27 # initialisation is not an answer.
28 #
29 # What no training can reach is taken out of what must be retained, and
30 # reported: pairs the model reads exactly as it reads another pair with a
31 # different answer, because they differ only outside its window. Chasing
32 # those would widen the model for ever.
33 #
34 # The data is split three ways. Thresholds are checked on the validation part
35 # while training, with the subsets that must be retained on the training
36 # part; the test part is measured once, on the model returned.
37  
38 use v5.36;
39 use utf8;
40  
41 use Time::HiRes qw(time);
42  
43 use Peta::NN::Model;
44 use Peta::NN::RNG;
45  
46 our $VERSION = '0.2610090';
47  
48 my %DEFAULT_SEARCH = (scale => [ 8, 256 ], depth => [1], grow => 2);
49 # epochs is the most one stage trains before the job looks at it again;
50 # patience is how many epochs a model may go without getting closer to its
51 # thresholds before it counts as stuck.
52 my %DEFAULT_TRAIN = (epochs => 60, batch => 32, lr => 0.01, lr_decay => 0.95, patience => 20);
53 my %DEFAULT_BUDGET = (runs => 40);
54 my @DEFAULT_SPLIT = (0.70, 0.15, 0.15);
55  
56 my $BOOST = 30; # the weight of the pairs of a subset that has to be retained and is short
57 my $FINE = 0.3; # a model that is trained on is trained at this share of the learning rate:
58 # at the full rate it first unlearns, and takes a dozen epochs to be where it was
59 my $TIMED_ITEMS = 256;
60 my $TIMED_REPEATS = 3;
61  
62 # The default network for a scale: an embedding that grows slowly with it,
63 # and `depth` hidden layers of `scale` units. A job takes the embedding of
64 # its starting width and keeps it; widening changes the hidden layers.
65 sub default_shape ($scale, $depth) {
66 my $embed = int($scale / 4);
67 $embed = 4 if $embed < 4;
68 $embed = 16 if $embed > 16;
69 return [ [ embed => $embed ], map { ([ dense => $scale ], 'relu') } 1 .. $depth ];
70 }
71  
72 sub new ($class, %arg) {
73 my $self = bless {
74 model => $arg{model} // die("model => { kind => ..., window => ... } is required\n"),
75 pairs => $arg{pairs} // die("pairs => [[input, output], ...] is required\n"),
76 fidelity => $arg{fidelity} // die("fidelity => { all => 0.97, ... } is required\n"),
77 subsets => $arg{subsets} // {},
78 group => $arg{group} // sub ($pair) { $pair->[0] },
79 always_train => $arg{always_train},
80 split => $arg{split} // [@DEFAULT_SPLIT],
81 shape => $arg{shape} // \&default_shape,
82 search => { %DEFAULT_SEARCH, %{ $arg{search} // {} } },
83 train => { %DEFAULT_TRAIN, %{ $arg{train} // {} } },
84 budget => { %DEFAULT_BUDGET, %{ $arg{budget} // {} } },
85 seed => $arg{seed} // 1,
86 backend => $arg{backend} // $ENV{PETA_NN_BACKEND} // 'auto',
87 attempts => [],
88 }, $class;
89 for my $name (keys %{ $self->{fidelity} }) {
90 die "fidelity names '$name', which is neither 'all' nor one of the subsets\n"
91 if $name ne 'all' && !$self->{subsets}{$name};
92 }
93 my ($min, $max) = @{ $self->{search}{scale} };
94 $self->{search}{start} //= $min;
95 die "the search starts at a width of $self->{search}{start}, outside its scale of $min to $max\n"
96 if $self->{search}{start} < $min || $self->{search}{start} > $max;
97 die "grow is a factor above 1, not '$self->{search}{grow}'\n" if $self->{search}{grow} <= 1;
98 $self->_drop_contradictions;
99 $self->_split;
100 return $self;
101 }
102  
103 # Two pairs that give one input (with the same parameters) different outputs cannot both be
104 # answered right, by any model. Such an input keeps its first pair; the
105 # others are set aside and listed by contradictions(), so that a threshold is
106 # not missed for a reason no training can remove.
107 sub _drop_contradictions ($self) {
108 my (%first, @kept, @dropped);
109 for my $pair (@{ $self->{pairs} }) {
110 my $key = join "\x{1f}", @$pair[ 0, 2 .. $#$pair ];
111 if (!exists $first{$key}) { $first{$key} = $pair->[1]; push @kept, $pair }
112 elsif ($first{$key} eq $pair->[1]) { push @kept, $pair }
113 else { push @dropped, [ $pair, $first{$key} ] }
114 }
115 @$self{qw(pairs contradictions)} = (\@kept, \@dropped);
116 return;
117 }
118  
119 # [the pair, the output that was kept for its input] for every pair set aside.
120 sub contradictions ($self) { return @{ $self->{contradictions} } }
121  
122 # Whole groups go to one part: all forms of one word stay together, so the
123 # held-out parts hold words the model has never seen in any form.
124 sub _split ($self) {
125 my (%group, @order);
126 for my $pair (@{ $self->{pairs} }) {
127 my $key = $self->{group}->($pair);
128 push @order, $key if !$group{$key};
129 push @{ $group{$key} }, $pair;
130 }
131 my $forced = $self->{always_train};
132 my @fixed = grep { $forced && grep { $forced->($_) } @{ $group{$_} } } @order;
133 my @free = grep { !($forced && grep { $forced->($_) } @{ $group{$_} }) } @order;
134 Peta::NN::RNG->new($self->{seed})->shuffle(\@free);
135  
136 my ($train_share, $validation_share) = @{ $self->{split} };
137 my $n_train = int(@free * $train_share);
138 my $n_validation = int(@free * $validation_share);
139 my %part = (
140 train => [ @fixed, @free[ 0 .. $n_train - 1 ] ],
141 validation => [ @free[ $n_train .. $n_train + $n_validation - 1 ] ],
142 test => [ @free[ $n_train + $n_validation .. $#free ] ],
143 );
144 $self->{part}{$_} = [ map { @{ $group{$_} } } @{ $part{$_} } ] for keys %part;
145 die "the split leaves the validation part empty\n" if !@{ $self->{part}{validation} };
146 return;
147 }
148  
149 sub part ($self, $name) { return @{ $self->{part}{$name} } }
150  
151 # The pairs a threshold is measured on: 'all' is the whole held-out part; a
152 # subset is the pairs its `where` accepts, of that part or, with of =>
153 # 'train', of the training part (what was shown and must be retained).
154 sub _measured_on ($self, $name, $held_out) {
155 return $self->{part}{$held_out} if $name eq 'all';
156 my $subset = $self->{subsets}{$name};
157 return [ grep { $subset->{where}->($_) } @{ $self->{part}{$held_out} } ] if ($subset->{of} // '') ne 'train';
158 # What must be retained, but for what the model cannot tell apart.
159 return $self->{retained}{$name} //= [ grep { $subset->{where}->($_) && !$self->{is_indistinct}{$_} } @{ $self->{part}{train} } ];
160 }
161  
162 # The pairs of the training part that the model reads as it reads another
163 # pair with a different answer; see Peta::NN::Model's indistinct. They are
164 # found once, with the first model that has been fitted: what a model reads
165 # does not depend on its width.
166 sub _find_indistinct ($self, $model) {
167 return if $self->{indistinct};
168 $self->{indistinct} = [ $model->indistinct($self->{part}{train}) ];
169 $self->{is_indistinct} = { map { $_->[0] => 1 } @{ $self->{indistinct} } };
170 delete $self->{retained};
171 return;
172 }
173  
174 # [pair, another answer, a pair that has it] for every training pair the
175 # model cannot tell from one with a different answer.
176 sub indistinct ($self) { return @{ $self->{indistinct} // [] } }
177  
178 # { name => share of pairs answered exactly } for every threshold.
179 # With $quick the model is measured where it is trained: on the graphics card,
180 # if that is where, which is what watching every epoch can afford. What a
181 # stage records, and what decides, is measured by the inference leg.
182 sub _fidelity ($self, $model, $held_out, $quick = 0) {
183 my %value;
184 for my $name (sort keys %{ $self->{fidelity} }) {
185 my $pairs = $self->_measured_on($name, $held_out);
186 $value{$name} = @$pairs ? $model->accuracy($pairs, where_trained => $quick) : 1;
187 }
188 return \%value;
189 }
190  
191 # Milliseconds per item on the inference leg, which is what a user of the
192 # model runs: the best of a few timed passes over a batch of held-out inputs.
193 sub _ms_per_item ($self, $model) {
194 my @pairs = @{ $self->{part}{validation} };
195 # One pass has one set of parameters: those of the first pair.
196 my @values = @{ $pairs[0] }[ 2 .. $#{ $pairs[0] } ];
197 @pairs = grep { "@$_[ 2 .. $#$_ ]" eq "@values" } @pairs;
198 $#pairs = $TIMED_ITEMS - 1 if @pairs > $TIMED_ITEMS;
199 my @inputs = map { $_->[0] } @pairs;
200 my $best;
201 for (1 .. $TIMED_REPEATS) {
202 my $start = time;
203 my @answers = @{ $model->inference->answers(\@inputs, @values) };
204 my $seconds = time - $start;
205 $best = $seconds if !defined $best || $seconds < $best;
206 }
207 return 1000 * $best / @inputs;
208 }
209  
210 # How much a pair counts in training: the pairs of the subsets in %$boost
211 # count that many times.
212 sub _weight ($self, $boost) {
213 return if !%$boost;
214 my $subsets = $self->{subsets};
215 return sub (@pair) {
216 my $weight = 1;
217 for my $name (keys %$boost) {
218 $weight *= $boost->{$name} if $subsets->{$name}{where}->(\@pair);
219 }
220 return $weight;
221 };
222 }
223  
224 sub _out_of_budget ($self) {
225 my $budget = $self->{budget};
226 return 'run limit reached' if @{ $self->{attempts} } >= $budget->{runs};
227 return 'time limit reached' if defined $budget->{seconds} && time - $self->{started} >= $budget->{seconds};
228 return;
229 }
230  
231 # How far a model is from the thresholds: the shortfalls added up, 0 when
232 # every one is met.
233 sub _distance ($self, $fidelity) {
234 my $distance = 0;
235 for my $name (keys %$fidelity) {
236 my $gap = $self->{fidelity}{$name} - $fidelity->{$name};
237 $distance += $gap if $gap > 0;
238 }
239 return $distance;
240 }
241  
242 # One stage of training: the model is trained, from nothing if it is new and
243 # on from where it is if not, until every threshold is met, or it has gone
244 # `patience` epochs without getting closer to them, or the stage's epochs or
245 # the job's time are used up. The weights it is left with are those of the
246 # epoch that was closest, or the ones it came with if no epoch got closer
247 # than it already was. The record says what the stage came to; `ended` is
248 # 'met', 'stuck', 'epochs' or 'time'.
249 sub _stage ($self, $model, $stage, $boost, %opt) {
250 my $start = time;
251 my $train = $self->{train};
252 my $patience = $train->{patience};
253 my $limit = $self->{budget}{seconds};
254 my ($closest, $stale, $ended) = (undef, 0, 'epochs');
255  
256 # A model that has been trained comes with a distance of its own to beat.
257 my $came_with = $model->net ? { distance => $self->_distance($self->_fidelity($model, 'validation')), weights => $model->net->weights } : undef;
258 $closest = $came_with->{distance} if $came_with;
259  
260 # What training watches, epoch by epoch: the distance to the thresholds.
261 my $watch = sub ($loss, $held) {
262 my $distance = $self->_distance($self->_fidelity($model, 'validation', 1));
263 if (!defined $closest || $distance < $closest) { ($closest, $stale) = ($distance, 0) }
264 else { $stale++ }
265 return $distance;
266 };
267 my $go_on = sub ($epoch, $loss) {
268 if ($closest == 0) { $ended = 'met' }
269 elsif ($stale >= $patience) { $ended = 'stuck' }
270 elsif (defined $limit && time - $self->{started} >= $limit) { $ended = 'time' }
271 return $ended eq 'epochs';
272 };
273 my $weight = $self->_weight($boost);
274 my @how = (validate => $self->{part}{validation}, ($weight ? (weight => $weight) : ()),
275 %$train, ($came_with ? (lr => $train->{lr} * $FINE) : ()),
276 patience => $train->{epochs} + 1, watch => $watch, on_epoch => $go_on);
277 if ($model->net) { $model->tune($self->{part}{train}, @how) }
278 else {
279 # A new model is fitted for one epoch first, so that what it cannot
280 # tell apart is known before anything is asked of it.
281 $model->fit($self->{part}{train}, @how, epochs => 1, watch => undef, on_epoch => undef, patience => undef);
282 $self->_find_indistinct($model);
283 $model->tune($self->{part}{train}, @how, epochs => $train->{epochs} - 1);
284 }
285 my $epochs = ($came_with ? 0 : 1) + (() = $model->net->history);
286 if ($came_with && $self->_distance($self->_fidelity($model, 'validation')) > $came_with->{distance}) {
287 $model->net->set_weights($came_with->{weights});
288 $ended = 'stuck' if $ended eq 'epochs';
289 }
290  
291 my $fidelity = $self->_fidelity($model, 'validation');
292 my ($width) = map { $_->[1] } grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} };
293 my $record = {
294 stage => $stage,
295 scale => $width,
296 depth => scalar(grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} }),
297 seed => $model->{seed},
298 params => $model->net->n_params,
299 epochs => $epochs,
300 fidelity => $fidelity,
301 missed => [ grep { $fidelity->{$_} < $self->{fidelity}{$_} } sort keys %$fidelity ],
302 ended => $ended,
303 ms => $self->_ms_per_item($model),
304 seconds => time - $start,
305 confirmation => $opt{confirmation} ? 1 : 0,
306 backend => $model->net->backend,
307 };
308 my $budget = $self->{budget};
309 $record->{over} = join ', ',
310 (defined $budget->{params} && $record->{params} > $budget->{params} ? 'parameters' : ()),
311 (defined $budget->{ms} && $record->{ms} > $budget->{ms} ? 'milliseconds' : ());
312 $record->{passed} = !@{ $record->{missed} } && !$record->{over} ? 1 : 0;
313 push @{ $self->{attempts} }, $record;
314 $self->{progress}->($record) if $self->{progress};
315 return $record;
316 }
317  
318 # Train one model until it has got there, giving it weight on what is short
319 # and, with $widen, more width when it is stuck. Returns the model and the
320 # record of its last stage.
321 sub _grow ($self, $width, $depth, $seed, %opt) {
322 my ($min, $max) = @{ $self->{search}{scale} };
323 my $model = Peta::NN::Model->new(%{ $self->{model} }, layers => $self->{shape}->($width, $depth), seed => $seed,
324 backend => $self->{backend});
325 my (%boost, $record);
326 my $stage = 'train';
327 while (!$self->_out_of_budget) {
328 $record = $self->_stage($model, $stage, \%boost, confirmation => $opt{confirmation});
329 last if $record->{passed} || $record->{over} || $record->{ended} eq 'time';
330 if ($record->{ended} eq 'epochs') { $stage = 'train on'; next } # still getting closer: it only needs more
331  
332 # Stuck. First more weight on what must be retained and is not, once
333 # per subset; then more room.
334 my @short = grep { $_ ne 'all' && ($self->{subsets}{$_}{of} // '') eq 'train' && !$boost{$_} } @{ $record->{missed} };
335 if (@short) {
336 $boost{$_} = $BOOST for @short;
337 $stage = "more weight on: @short";
338 next;
339 }
340 last if !$opt{widen} || $width >= $max;
341 my $wider = int($width * $self->{search}{grow} + 0.5);
342 $width = $wider > $max ? $max : $wider;
343 $model->widen($width);
344 $stage = "widened to $width";
345 }
346 return ($model, $record);
347 }
348  
349 # Find the smallest model that meets the thresholds, and return it. If none
350 # does within the budget, the closest is returned; result() says which.
351 sub run ($self, %arg) {
352 $self->{progress} = $arg{progress};
353 $self->{started} = time;
354  
355 # One model per depth, grown from the starting width until it gets there.
356 my ($chosen, $model);
357 for my $depth (@{ $self->{search}{depth} }) {
358 last if $self->_out_of_budget;
359 my ($grown, $record) = $self->_grow($self->{search}{start}, $depth, $self->{seed}, widen => 1);
360 next if !$record;
361 my $better = !$chosen
362 || $record->{passed} > $chosen->{passed}
363 || $record->{passed} == $chosen->{passed}
364 && ($record->{passed} ? $record->{params} < $chosen->{params}
365 : $self->_distance($record->{fidelity}) < $self->_distance($chosen->{fidelity}));
366 ($chosen, $model) = ($record, $grown) if $better;
367 last if $record->{passed};
368 }
369 die "the budget allows no training at all\n" if !$chosen;
370  
371 # Confirm it: a second seed, from nothing and from the size the first one
372 # reached, has to get there by the same rules. The model returned is the
373 # first seed's.
374 my $confirmed = 0;
375 if ($chosen->{passed} && !$self->_out_of_budget) {
376 my (undef, $again) = $self->_grow(@$chosen{qw(scale depth)}, $self->{seed} + 1, widen => 1, confirmation => 1);
377 $confirmed = 1 if $again && $again->{passed};
378 }
379  
380 $self->{result} = {
381 met => $chosen->{passed} ? 1 : 0,
382 confirmed => $confirmed,
383 %$chosen{qw(scale depth params ms fidelity missed over stage)},
384 test => $self->_fidelity($model, 'test'),
385 runs => scalar @{ $self->{attempts} },
386 epochs => eval { my $n = 0; $n += $_->{epochs} for grep { !$_->{confirmation} } @{ $self->{attempts} }; $n },
387 seconds => time - $self->{started},
388 stopped => $self->_out_of_budget // 'done',
389 };
390 return $model;
391 }
392  
393 sub attempts ($self) { return @{ $self->{attempts} } }
394 sub result ($self) { return $self->{result} }
395  
396 # Every stage in the order it was run, and the outcome.
397 sub report ($self) {
398 my @names = sort keys %{ $self->{fidelity} };
399 my @clash = $self->contradictions;
400 my $text = !@clash ? '' : sprintf "%d pairs set aside: their input already had another answer (%s%s)\n", scalar @clash,
401 join(', ', map { "$_->[0][0] → $_->[0][1], kept $_->[1]" } @clash[ 0 .. ($#clash < 2 ? $#clash : 2) ]), @clash > 3 ? ', ...' : '';
402 my @same = $self->indistinct;
403 $text .= sprintf "%d training pairs the model cannot tell from a pair with a different answer; a wider window would (%s%s)\n", scalar @same,
404 join(', ', map { "$_->[0][0] → $_->[0][1] as $_->[2][0] → $_->[2][1]" } @same[ 0 .. ($#same < 2 ? $#same : 2) ]), @same > 3 ? ', ...' : '' if @same;
405 $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s %s\n", '#', 'stage', 'width', 'params', 'epochs', 'on', 'ms',
406 join(' ', map { sprintf '%-12s', substr($_, 0, 12) } @names), 'outcome';
407 $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s\n", '', 'required', '', '', '', '', $self->{budget}{ms} // '',
408 join(' ', map { sprintf '%-12s', sprintf '>= %.1f%%', 100 * $self->{fidelity}{$_} } @names);
409 my %ENDED = (stuck => 'no closer', epochs => 'still getting closer', time => 'out of time');
410 my $n = 0;
411 for my $a (@{ $self->{attempts} }) {
412 my $outcome = $a->{passed} ? ($a->{confirmation} ? 'confirmed' : 'met')
413 : $a->{over} ? "over budget: $a->{over}"
414 : "$ENDED{ $a->{ended} }; short: @{ $a->{missed} }";
415 $text .= sprintf "%-3d %-24s %5d %7d %6d %-5s %7.3f %s %s\n", ++$n,
416 substr(($a->{confirmation} ? 'second seed: ' : '') . $a->{stage}, 0, 24), @$a{qw(scale params epochs backend ms)},
417 join(' ', map { sprintf '%-12s', sprintf '%.2f%%', 100 * $a->{fidelity}{$_} } @names), $outcome;
418 }
419 my $r = $self->{result} or return $text;
420 $text .= sprintf "\n%s: depth %d, width %d, %d parameters, %.3f ms per item. %s%d stages, %d epochs, %.0f s (%s).\n",
421 ($r->{met} ? 'the model meets the thresholds' : 'the model does NOT meet the thresholds; this is the closest it got'),
422 @$r{qw(depth scale params ms)},
423 ($r->{met} ? ($r->{confirmed} ? 'Confirmed on a second seed. ' : 'NOT confirmed on a second seed. ') : ''),
424 @$r{qw(runs epochs seconds stopped)};
425 $text .= sprintf "on the test part, measured once: %s\n",
426 join ', ', map { sprintf '%s %.1f%%', $_, 100 * $r->{test}{$_} } @names;
427 return $text;
428 }
429  
430 1;
431  
432 __END__