blib/lib/Peta/NN.pm untested

filter: Statementstmt 0.0% 0 / 369 Branchbran 0.0% 0 / 92 Conditioncond 0.0% 0 / 91 MC/DCmcdc 0.0% 0 / 47 Subroutinesub 0.0% 0 / 45 Podpod 0.0% 0 / 35 totaltotal 0.0% 0 / 679 CC 47 SCAR 39.0
CC
47
Cov
0%
train
210.0
_seconds_per_step
90.0
_settle
42.0
CRAP 49.4
1 package Peta::NN;
2 # ABSTRACT: a small neural network, defined, trained and saved in Perl
3  
4 # A small feed-forward neural network: a list of layers, a loss, and a
5 # training loop. A mini-batch goes through as one tensor; what a tensor is,
6 # and how the arithmetic on it is done, is the backend's business.
7  
8 use v5.36;
9  
10 use Storable ();
11 use Time::HiRes qw(time);
12  
13 use Peta::NN::RNG;
14 use Peta::NN::Backend;
15 use Peta::NN::Inference;
16 use Peta::NN::Optimizer;
17 use Peta::NN::Layer::Dense;
18 use Peta::NN::Layer::Embed;
19 use Peta::NN::Layer::Activation;
20  
21 our $VERSION = '0.2610090';
22  
23 my %IS_ACTIVATION = map { $_ => 1 } Peta::NN::Layer::Activation::kinds();
24 my %IS_LOSS = map { $_ => 1 } qw(softmax mse);
25  
26 my $EVAL_BATCH = 256; # samples per forward pass when only predicting
27  
28 # Whether train() leaves the samples with a backend that can keep them. On
29 # unless a test wants the two ways compared.
30 our $EPOCH_ON_BACKEND = 1;
31  
32 # A network that was left the choice of its backend ('auto') settles it when
33 # it is trained: only then is it known how much work there is. If the run
34 # would take the backend it stands on $WORTH seconds or more, a few steps
35 # are timed on every backend this perl and this machine have, and the run
36 # goes to the fastest. Nothing is assumed about which that is: plain Perl
37 # under a JIT, PDL and a graphics card each win somewhere, and each may be
38 # missing. $TRIAL_WARM steps are run before the $TRIAL_STEPS that are timed.
39 our $WORTH = 2;
40 our $TRIAL_STEPS = 6; # steps in a timed round
41 our $TRIAL_WARM = 2;
42 my @TRIAL_TIME = (0.3, 1); # seconds of timed rounds, at least and at most; the fastest round counts. The
43 # first steps of anything are slow, and a card that sat idle takes a few tenths
44 # of a second to come up to speed
45 my $SETTLED = 0.05; # two rounds in a row that do not beat the best by this share: it is found
46 my $PROBE = 16; # samples in the one small step that is tried first
47 my $HOPELESS = 3; # a backend this many times slower than the best so far is not timed to the end
48  
49 my $STATE_FORMAT = 'Peta::NN network state';
50  
51 # A layer is written [dense => N], [embed => DIM], or an activation's name.
52 sub _build_layer ($spec) {
53 if (!ref $spec) {
54 die "unknown layer '$spec'\n" if !$IS_ACTIVATION{$spec};
55 return Peta::NN::Layer::Activation->new(kind => $spec);
56 }
57 my ($kind, $size) = @$spec;
58 return Peta::NN::Layer::Dense->new(out => $size) if $kind eq 'dense';
59 return Peta::NN::Layer::Embed->new(dim => $size) if $kind eq 'embed';
60 die "unknown layer '$kind'\n";
61 }
62  
63 sub new ($class, %arg) {
64 my $self = bless {
65 input => $arg{input} // die("input => size, or { tokens =>, vocab => }, is required\n"),
66 loss => $arg{loss} // 'softmax',
67 seed => $arg{seed} // 1,
68 }, $class;
69 die "unknown loss '$self->{loss}' (have: @{[ sort keys %IS_LOSS ]})\n" if !$IS_LOSS{ $self->{loss} };
70 my $backend = $arg{backend} // $ENV{PETA_NN_BACKEND} // 'plain';
71 $self->{auto} = $backend eq 'auto';
72 $self->{backend} = Peta::NN::Backend::create($backend);
73 $self->{rng} = Peta::NN::RNG->new($self->{seed});
74 $self->{generation} = 0;
75 $self->{layers} = [ map { _build_layer($_) } @{ $arg{layers} // die "layers => [...] is required\n" } ];
76  
77 my $size = $self->{input};
78 $size = $_->init($size, $self->{rng}, $self->{backend}) for @{ $self->{layers} };
79 die "the network needs at least one layer\n" if ref $size;
80 $self->{n_out} = $size;
81  
82 # Nothing consumes the first layer's input gradient.
83 my $first = $self->{layers}[0];
84 $first->{need_dx} = 0 if $first->type eq 'dense';
85 return $self;
86 }
87  
88 sub n_out ($self) { return $self->{n_out} }
89  
90 # A number that changes whenever the weights do, for whoever caches something
91 # derived from them.
92 sub generation ($self) { return $self->{generation} }
93 sub layers ($self) { return @{ $self->{layers} } }
94 sub backend ($self) { return $self->{backend}->name }
95  
96 # How the backend was settled by the last train(), for a network that was
97 # left the choice: { backend, steps, seconds_per_step => { backend => seconds } },
98 # the timings being those that were taken. Nothing for any other network, or
99 # for a run of too few steps to time.
100 sub settled ($self) { return $self->{settled} }
101  
102 # Every parameter as [layer, name], in layer order: the tensor is
103 # $layer->{name}, its gradient after backprop() is $layer->{"g$name"}.
104 sub params ($self) {
105 return map { my $layer = $_; map { [ $layer, $_ ] } $layer->param_names } @{ $self->{layers} };
106 }
107  
108 # The parameters, and their gradients, as flat Perl lists in params() order.
109 sub weights ($self) {
110 return [ map { $self->{backend}->flat($_->[0]{ $_->[1] }) } $self->params ];
111 }
112  
113 sub gradients ($self) {
114 return [ map { $self->{backend}->flat($_->[0]{"g$_->[1]"}) } $self->params ];
115 }
116  
117 sub set_weights ($self, $lists) {
118 my @params = $self->params;
119 die "these weights do not fit this network\n" if @params != @$lists;
120 for my $p (0 .. $#params) {
121 my ($layer, $name) = @{ $params[$p] };
122 die "these weights do not fit this network\n"
123 if @{ $lists->[$p] } != @{ $self->{backend}->flat($layer->{$name}) };
124 $layer->{$name} = $self->{backend}->tensor($lists->[$p], $layer->param_cols($name));
125 }
126 $self->{generation}++;
127 return $self;
128 }
129  
130 sub n_params ($self) {
131 my $n = 0;
132 $n += @$_ for @{ $self->weights };
133 return $n;
134 }
135  
136 # A list of samples as the backend's batch: rows of numbers, or of tokens.
137 sub _batch ($self, $inputs) {
138 my $input = $self->{input};
139 return ref $input
140 ? $self->{backend}->tokens([ map { @$_ } @$inputs ], $input->{tokens})
141 : $self->{backend}->tensor([ map { @$_ } @$inputs ], $input);
142 }
143  
144 sub _forward ($self, $inputs) {
145 my $x = $self->_batch($inputs);
146 $x = $_->forward($x) for @{ $self->{layers} };
147 return $x;
148 }
149  
150 # The loss of a batch of [input, target] or [input, target, weight] pairs,
151 # given the network's output for them. A weight multiplies that sample's loss
152 # and gradient; if no pair has one, none is passed on.
153 sub _loss ($self, $out, $pairs) {
154 my @targets = map { $_->[1] } @$pairs;
155 my $weights = (grep { defined $_->[2] } @$pairs) ? [ map { $_->[2] // 1 } @$pairs ] : undef;
156 return $self->{loss} eq 'softmax'
157 ? $self->{backend}->softmax_ce($out, \@targets, $weights)
158 : $self->{backend}->mse($out, [ map { @$_ } @targets ], $weights);
159 }
160  
161 # The raw output for one sample: logits under the softmax loss.
162 sub forward ($self, $x) {
163 return $self->{backend}->flat($self->_forward([$x]));
164 }
165  
166 # What a caller wants to see: class probabilities under the softmax loss,
167 # the output itself otherwise.
168 sub predict ($self, $x) {
169 my $out = $self->forward($x);
170 return $out if $self->{loss} ne 'softmax';
171 my $max = $out->[0];
172 for my $v (@$out) { $max = $v if $v > $max }
173 my @p = map { exp($_ - $max) } @$out;
174 my $sum = 0;
175 $sum += $_ for @p;
176 return [ map { $_ / $sum } @p ];
177 }
178  
179 # The most probable class of each of many samples.
180 sub classify_all ($self, $inputs) {
181 my @classes;
182 for (my $start = 0; $start < @$inputs; $start += $EVAL_BATCH) {
183 my $end = $start + $EVAL_BATCH - 1;
184 $end = $#$inputs if $end > $#$inputs;
185 my $out = $self->_forward([ @$inputs[ $start .. $end ] ]);
186 push @classes, @{ $self->{backend}->argmax_rows($out, $self->{n_out}) };
187 }
188 return \@classes;
189 }
190  
191 sub classify ($self, $x) { return $self->classify_all([$x])->[0] }
192  
193 # The summed loss of [input, target] pairs, without touching the gradients.
194 sub loss ($self, $pairs) {
195 my $total = 0;
196 for (my $start = 0; $start < @$pairs; $start += $EVAL_BATCH) {
197 my $end = $start + $EVAL_BATCH - 1;
198 $end = $#$pairs if $end > $#$pairs;
199 my @chunk = @$pairs[ $start .. $end ];
200 my ($loss) = $self->_loss($self->_forward([ map { $_->[0] } @chunk ]), \@chunk);
201 $total += $loss;
202 }
203 return $total;
204 }
205  
206 # Forward and backward for a batch of [input, target] pairs: leaves the
207 # batch's summed gradient with the layers and returns its summed loss.
208 sub backprop ($self, $pairs) {
209 my $out = $self->_forward([ map { $_->[0] } @$pairs ]);
210 my ($loss, $grad) = $self->_loss($out, $pairs);
211 for my $layer (reverse @{ $self->{layers} }) {
212 $grad = $layer->backward($grad);
213 }
214 return $loss;
215 }
216  
217 # The samples of a training run, left with a backend that can keep them: then
218 # a step picks its batch where the arithmetic is, and nothing is carried
219 # there and back for it (on a graphics card that carrying is what costs).
220 # Nothing if the backend does not keep samples, or not such as these.
221 sub _kept ($self, $data) {
222 my $backend = $self->{backend};
223 return if !$EPOCH_ON_BACKEND || !ref $self->{input} || $self->{loss} ne 'softmax' || !$backend->can('epoch_data');
224 return $backend->epoch_data([ map { @{ $_->[0] } } @$data ], $self->{input}{tokens}, [ map { $_->[1] } @$data ],
225 (grep { defined $_->[2] } @$data) ? [ map { $_->[2] // 1 } @$data ] : undef);
226 }
227  
228 # One training step on the samples $order->[$start .. $end]: forward,
229 # backward, update. Returns the batch's loss, or nothing where the samples
230 # are kept by the backend and their losses with them.
231 sub _step ($self, $optimizer, $kept, $data, $order, $start, $end) {
232 my $loss = 0;
233 if ($kept) {
234 my $backend = $self->{backend};
235 my $x = $backend->batch_at($kept, $start, $end - $start + 1);
236 $x = $_->forward($x) for @{ $self->{layers} };
237 my $grad = $backend->softmax_ce_at($x, $kept, $start);
238 $grad = $_->backward($grad) for reverse @{ $self->{layers} };
239 }
240 else { $loss = $self->backprop([ @$data[ @$order[ $start .. $end ] ] ]) }
241 $optimizer->step(1 / ($end - $start + 1));
242 $self->{generation}++;
243 return $loss;
244 }
245  
246 # The network on another backend, with these weights.
247 sub _move_to ($self, $backend, $weights) {
248 $self->{backend} = $backend;
249 $_->{backend} = $backend for @{ $self->{layers} };
250 my @params = $self->params;
251 for my $p (0 .. $#params) {
252 my ($layer, $name) = @{ $params[$p] };
253 $layer->{$name} = $backend->tensor($weights->[$p], $layer->param_cols($name));
254 }
255 $self->{generation}++;
256 return;
257 }
258  
259 # Seconds a training step takes on the backend the network is on now, timed
260 # on the first samples: the fastest of a few rounds of steps. The weights are
261 # afterwards what they were. With $beat, a time per step to beat, it gives
262 # up once it is hopeless, and tries one step on a handful of samples before
263 # anything else: a step on the whole batch cannot be faster than that.
264 sub _seconds_per_step ($self, $data, $batch, $weights, $beat = undef) {
265 my $backend = $self->{backend};
266 my $last = $batch * ($TRIAL_WARM + $TRIAL_STEPS) - 1;
267 $last = $#$data if $last > $#$data;
268 my @sample = @$data[ 0 .. $last ];
269 my @order = 0 .. $last;
270 my $optimizer = Peta::NN::Optimizer->new(kind => 'adam', backend => $backend, params => [ $self->params ]);
271 # Reading a bias back waits for whatever a card has been sent.
272 my $wait = sub { $backend->flat(($self->params)[-1][0]{ ($self->params)[-1][1] }) };
273 my $best;
274 if (defined $beat && $batch > $PROBE) {
275 $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]); # the first step of all sets things up
276 my $began = time;
277 $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]);
278 $wait->();
279 $best = 9**9**9 if time - $began > $HOPELESS * $beat;
280 }
281 if (!defined $best) {
282 my $kept = $self->_kept(\@sample);
283 $backend->epoch_order($kept, \@order) if $kept;
284 my $steps = sub ($from, $count) {
285 $self->_step($optimizer, $kept, \@sample, \@order, $_ * $batch, ($_ + 1) * $batch - 1) for $from .. $from + $count - 1;
286 };
287 $steps->(0, $TRIAL_WARM);
288 my ($stale, $since) = (0, time);
289 while (1) { # the same samples round after round: it is the time that is wanted
290 $wait->();
291 my $began = time;
292 $steps->($TRIAL_WARM, $TRIAL_STEPS);
293 $wait->();
294 my $round = (time - $began) / $TRIAL_STEPS;
295 $stale = defined $best && $round > $best * (1 - $SETTLED) ? $stale + 1 : 0;
296 $best = $round if !defined $best || $round < $best;
297 my $spent = time - $since;
298 last if $spent >= $TRIAL_TIME[1] || $stale >= 2 && $spent >= $TRIAL_TIME[0] || defined $beat && $best > $HOPELESS * $beat;
299 }
300 }
301 $self->_move_to($backend, $weights);
302 return $best;
303 }
304  
305 # Settle the backend of a network that was left the choice, for a run of
306 # this many samples, at this batch, for this many epochs.
307 sub _settle ($self, $data, $batch, $epochs) {
308 # The same work as the last time it was settled: the same answer.
309 my $steps = $epochs * (int($#$data / $batch) + 1);
310 my $work = join ' ', scalar @$data, $batch, $steps;
311 return if $self->{settled} && $self->{settled}{work} eq $work;
312 delete $self->{settled};
313 return if @$data < $batch * ($TRIAL_WARM + $TRIAL_STEPS); # too few steps to time a round
314 my $weights = $self->weights;
315 my $count = $self->{generation};
316 my $here = $self->{backend};
317 my %seconds = ($here->name => $self->_seconds_per_step($data, $batch, $weights));
318 my ($best, $on) = ($seconds{ $here->name }, $here);
319 if ($best * $steps >= $WORTH) {
320 for my $name (grep { $_ ne $here->name } Peta::NN::Backend::names()) {
321 my $other = Peta::NN::Backend::try($name) or next;
322 $self->_move_to($other, $weights);
323 $seconds{$name} = $self->_seconds_per_step($data, $batch, $weights, $best);
324 ($best, $on) = ($seconds{$name}, $other) if $seconds{$name} < $best;
325 }
326 # A close second is timed once more: the first time anything runs it is
327 # not at its best (a JIT has not compiled it yet, a card is not up to
328 # speed), and by how much differs from backend to backend.
329 my %backend = ($here->name => $here);
330 for my $name (grep { $seconds{$_} <= $HOPELESS * $best } sort keys %seconds) {
331 $backend{$name} //= Peta::NN::Backend::try($name);
332 $self->_move_to($backend{$name}, $weights);
333 my $again = $self->_seconds_per_step($data, $batch, $weights);
334 $seconds{$name} = $again if $again < $seconds{$name};
335 }
336 my ($fastest) = sort { $seconds{$a} <=> $seconds{$b} } keys %seconds;
337 ($best, $on) = ($seconds{$fastest}, $backend{$fastest} // Peta::NN::Backend::try($fastest));
338 }
339 $self->_move_to($on, $weights);
340 $self->{generation} = $count; # nothing has changed: the weights are what they were
341 delete @seconds{ grep { $seconds{$_} == 9**9**9 } keys %seconds }; # those that were not worth timing
342 $self->{settled} = { backend => $on->name, steps => $steps, seconds_per_step => \%seconds, work => $work };
343 return;
344 }
345  
346 # Train on [input, target] or [input, target, weight] pairs. Returns the mean
347 # loss of the last epoch.
348 # epochs, batch how long, and how many samples per update
349 # optimizer, lr, ... passed to Peta::NN::Optimizer, weight_decay included
350 # lr_decay the learning rate is multiplied by this after every epoch
351 # validate pairs that are not trained on; their loss is measured every epoch
352 # patience stop after this many epochs without improvement of the
353 # validation loss (the training loss if there is no
354 # validation set), and return to the best epoch's weights
355 # min_delta what counts as an improvement; default any
356 # watch sub (training loss, validation loss) returning the number
357 # patience watches instead; lower is better
358 # target_loss stop once an epoch's mean loss is at or below this
359 # on_epoch called with (epoch, mean loss); a false return stops.
360 # history() has the validation loss as well
361 sub train ($self, %arg) {
362 my $data = $arg{data} // die "data => [[input, target], ...] is required\n";
363 my $epochs = $arg{epochs} // 10;
364 my $batch = $arg{batch} // 16;
365 $self->_settle($data, $batch, $epochs) if $self->{auto};
366 my $optimizer = Peta::NN::Optimizer->new(
367 kind => $arg{optimizer} // 'adam',
368 backend => $self->{backend},
369 params => [ $self->params ],
370 map { $_ => $arg{$_} } grep { defined $arg{$_} } qw(lr momentum beta1 beta2 epsilon weight_decay),
371 );
372  
373 my $validate = $arg{validate};
374 my $patience = $arg{patience};
375 my $min_delta = $arg{min_delta} // 0;
376 my ($best, $best_weights, $stale) = (undef, undef, 0);
377  
378 my $backend = $self->{backend};
379 my $kept = $self->_kept($data);
380  
381 my @order = 0 .. $#$data;
382 my $mean = 0;
383 $self->{history} = [];
384 for my $epoch (1 .. $epochs) {
385 $self->{rng}->shuffle(\@order);
386 my $total = 0;
387 $backend->epoch_order($kept, \@order) if $kept;
388 for (my $start = 0; $start < @order; $start += $batch) {
389 my $end = $start + $batch - 1;
390 $end = $#order if $end > $#order;
391 $total += $self->_step($optimizer, $kept, $data, \@order, $start, $end);
392 }
393 $total = $backend->epoch_loss($kept) if $kept;
394 $mean = $total / @order;
395 my $held = $validate && @$validate ? $self->loss($validate) / @$validate : undef;
396 push @{ $self->{history} }, { epoch => $epoch, loss => $mean, validation => $held };
397  
398 # The loss that is watched: the validation set's if there is one. The
399 # weights of its best epoch are kept and put back at the end.
400 if (defined $patience) {
401 my $watched = $arg{watch} ? $arg{watch}->($mean, $held) : $held // $mean;
402 if (!defined $best || $watched < $best - $min_delta) {
403 ($best, $best_weights, $stale) = ($watched, $self->weights, 0);
404 }
405 elsif (++$stale >= $patience) { last }
406 }
407 last if $arg{on_epoch} && !$arg{on_epoch}->($epoch, $mean);
408 last if defined $arg{target_loss} && $mean <= $arg{target_loss};
409 $optimizer->{lr} *= $arg{lr_decay} if defined $arg{lr_decay};
410 }
411 $self->set_weights($best_weights) if $best_weights;
412 return $mean;
413 }
414  
415 # What every epoch of the last train() measured: { epoch, loss, validation }.
416 sub history ($self) { return @{ $self->{history} // [] } }
417  
418 # Share of [input, class index] pairs classified correctly.
419 sub accuracy ($self, $pairs) {
420 my $classes = $self->classify_all([ map { $_->[0] } @$pairs ]);
421 my $right = grep { $classes->[$_] == $pairs->[$_][1] } 0 .. $#$pairs;
422 return $right / @$pairs;
423 }
424  
425 # What it takes to rebuild this net: its definition and its weights. The
426 # backend is not part of it; a net trained on one loads on any other.
427 sub state ($self) {
428 return {
429 version => $VERSION,
430 input => $self->{input},
431 loss => $self->{loss},
432 seed => $self->{seed},
433 layers => [ map { $_->spec } @{ $self->{layers} } ],
434 weights => $self->weights,
435 };
436 }
437  
438 sub from_state ($class, $state, %arg) {
439 return $class->new(%$state{qw(input loss seed layers)}, %arg)->set_weights($state->{weights});
440 }
441  
442 # Storable's portable format writes a double as decimal text and loses its
443 # last bits. Packed little-endian it stays exact, and portable all the same.
444 sub freeze_state ($state) {
445 return { %$state, weights => [ map { pack 'd<*', @$_ } @{ $state->{weights} } ] };
446 }
447  
448 sub thaw_state ($frozen) {
449 return { %$frozen, weights => [ map { [ unpack 'd<*', $_ ] } @{ $frozen->{weights} } ] };
450 }
451  
452 # A network's state file, by convention *.net.
453 sub save ($self, $file) {
454 Storable::nstore({ %{ freeze_state($self->state) }, format => $STATE_FORMAT }, $file);
455 return $self;
456 }
457  
458 # The file is read as plain data, and weights that do not fit the definition
459 # in it are refused.
460 sub load ($class, $file, %arg) {
461 my $frozen = Peta::NN::Inference::read_file($file, $STATE_FORMAT);
462 die "$file is a malformed network state\n"
463 if ref $frozen->{layers} ne 'ARRAY' || ref $frozen->{weights} ne 'ARRAY' || grep { ref } @{ $frozen->{weights} };
464 return $class->from_state(thaw_state($frozen), %arg);
465 }
466  
467 1;
468  
469 __END__