| 1 | package Peta::NN; | ||
| 2 | # ABSTRACT: a small neural network, defined, trained and saved in Perl | ||
| 3 | |||
| 4 | # A small feed-forward neural network: a list of layers, a loss, and a | ||
| 5 | # training loop. A mini-batch goes through as one tensor; what a tensor is, | ||
| 6 | # and how the arithmetic on it is done, is the backend's business. | ||
| 7 | |||
| 8 | use v5.36; | ||
| 9 | |||
| 10 | use Storable (); | ||
| 11 | use Time::HiRes qw(time); | ||
| 12 | |||
| 13 | use Peta::NN::RNG; | ||
| 14 | use Peta::NN::Backend; | ||
| 15 | use Peta::NN::Inference; | ||
| 16 | use Peta::NN::Optimizer; | ||
| 17 | use Peta::NN::Layer::Dense; | ||
| 18 | use Peta::NN::Layer::Embed; | ||
| 19 | use Peta::NN::Layer::Activation; | ||
| 20 | |||
| 21 | our $VERSION = '0.2610090'; | ||
| 22 | |||
| 23 | my %IS_ACTIVATION = map { $_ => 1 } Peta::NN::Layer::Activation::kinds(); | ||
| 24 | my %IS_LOSS = map { $_ => 1 } qw(softmax mse); | ||
| 25 | |||
| 26 | my $EVAL_BATCH = 256; # samples per forward pass when only predicting | ||
| 27 | |||
| 28 | # Whether train() leaves the samples with a backend that can keep them. On | ||
| 29 | # unless a test wants the two ways compared. | ||
| 30 | our $EPOCH_ON_BACKEND = 1; | ||
| 31 | |||
| 32 | # A network that was left the choice of its backend ('auto') settles it when | ||
| 33 | # it is trained: only then is it known how much work there is. If the run | ||
| 34 | # would take the backend it stands on $WORTH seconds or more, a few steps | ||
| 35 | # are timed on every backend this perl and this machine have, and the run | ||
| 36 | # goes to the fastest. Nothing is assumed about which that is: plain Perl | ||
| 37 | # under a JIT, PDL and a graphics card each win somewhere, and each may be | ||
| 38 | # missing. $TRIAL_WARM steps are run before the $TRIAL_STEPS that are timed. | ||
| 39 | our $WORTH = 2; | ||
| 40 | our $TRIAL_STEPS = 6; # steps in a timed round | ||
| 41 | our $TRIAL_WARM = 2; | ||
| 42 | my @TRIAL_TIME = (0.3, 1); # seconds of timed rounds, at least and at most; the fastest round counts. The | ||
| 43 | # first steps of anything are slow, and a card that sat idle takes a few tenths | ||
| 44 | # of a second to come up to speed | ||
| 45 | my $SETTLED = 0.05; # two rounds in a row that do not beat the best by this share: it is found | ||
| 46 | my $PROBE = 16; # samples in the one small step that is tried first | ||
| 47 | my $HOPELESS = 3; # a backend this many times slower than the best so far is not timed to the end | ||
| 48 | |||
| 49 | my $STATE_FORMAT = 'Peta::NN network state'; | ||
| 50 | |||
| 51 | # A layer is written [dense => N], [embed => DIM], or an activation's name. | ||
| 52 | sub _build_layer ($spec) { | ||
| 53 | if (!ref $spec) { | ||
| 54 | die "unknown layer '$spec'\n" if !$IS_ACTIVATION{$spec}; | ||
| 55 | return Peta::NN::Layer::Activation->new(kind => $spec); | ||
| 56 | } | ||
| 57 | my ($kind, $size) = @$spec; | ||
| 58 | return Peta::NN::Layer::Dense->new(out => $size) if $kind eq 'dense'; | ||
| 59 | return Peta::NN::Layer::Embed->new(dim => $size) if $kind eq 'embed'; | ||
| 60 | die "unknown layer '$kind'\n"; | ||
| 61 | } | ||
| 62 | |||
| 63 | sub new ($class, %arg) { | ||
| 64 | my $self = bless { | ||
| 65 | input => $arg{input} // die("input => size, or { tokens =>, vocab => }, is required\n"), | ||
| 66 | loss => $arg{loss} // 'softmax', | ||
| 67 | seed => $arg{seed} // 1, | ||
| 68 | }, $class; | ||
| 69 | die "unknown loss '$self->{loss}' (have: @{[ sort keys %IS_LOSS ]})\n" if !$IS_LOSS{ $self->{loss} }; | ||
| 70 | my $backend = $arg{backend} // $ENV{PETA_NN_BACKEND} // 'plain'; | ||
| 71 | $self->{auto} = $backend eq 'auto'; | ||
| 72 | $self->{backend} = Peta::NN::Backend::create($backend); | ||
| 73 | $self->{rng} = Peta::NN::RNG->new($self->{seed}); | ||
| 74 | $self->{generation} = 0; | ||
| 75 | $self->{layers} = [ map { _build_layer($_) } @{ $arg{layers} // die "layers => [...] is required\n" } ]; | ||
| 76 | |||
| 77 | my $size = $self->{input}; | ||
| 78 | $size = $_->init($size, $self->{rng}, $self->{backend}) for @{ $self->{layers} }; | ||
| 79 | die "the network needs at least one layer\n" if ref $size; | ||
| 80 | $self->{n_out} = $size; | ||
| 81 | |||
| 82 | # Nothing consumes the first layer's input gradient. | ||
| 83 | my $first = $self->{layers}[0]; | ||
| 84 | $first->{need_dx} = 0 if $first->type eq 'dense'; | ||
| 85 | return $self; | ||
| 86 | } | ||
| 87 | |||
| 88 | sub n_out ($self) { return $self->{n_out} } | ||
| 89 | |||
| 90 | # A number that changes whenever the weights do, for whoever caches something | ||
| 91 | # derived from them. | ||
| 92 | sub generation ($self) { return $self->{generation} } | ||
| 93 | sub layers ($self) { return @{ $self->{layers} } } | ||
| 94 | sub backend ($self) { return $self->{backend}->name } | ||
| 95 | |||
| 96 | # How the backend was settled by the last train(), for a network that was | ||
| 97 | # left the choice: { backend, steps, seconds_per_step => { backend => seconds } }, | ||
| 98 | # the timings being those that were taken. Nothing for any other network, or | ||
| 99 | # for a run of too few steps to time. | ||
| 100 | sub settled ($self) { return $self->{settled} } | ||
| 101 | |||
| 102 | # Every parameter as [layer, name], in layer order: the tensor is | ||
| 103 | # $layer->{name}, its gradient after backprop() is $layer->{"g$name"}. | ||
| 104 | sub params ($self) { | ||
| 105 | return map { my $layer = $_; map { [ $layer, $_ ] } $layer->param_names } @{ $self->{layers} }; | ||
| 106 | } | ||
| 107 | |||
| 108 | # The parameters, and their gradients, as flat Perl lists in params() order. | ||
| 109 | sub weights ($self) { | ||
| 110 | return [ map { $self->{backend}->flat($_->[0]{ $_->[1] }) } $self->params ]; | ||
| 111 | } | ||
| 112 | |||
| 113 | sub gradients ($self) { | ||
| 114 | return [ map { $self->{backend}->flat($_->[0]{"g$_->[1]"}) } $self->params ]; | ||
| 115 | } | ||
| 116 | |||
| 117 | sub set_weights ($self, $lists) { | ||
| 118 | my @params = $self->params; | ||
| 119 | die "these weights do not fit this network\n" if @params != @$lists; | ||
| 120 | for my $p (0 .. $#params) { | ||
| 121 | my ($layer, $name) = @{ $params[$p] }; | ||
| 122 | die "these weights do not fit this network\n" | ||
| 123 | if @{ $lists->[$p] } != @{ $self->{backend}->flat($layer->{$name}) }; | ||
| 124 | $layer->{$name} = $self->{backend}->tensor($lists->[$p], $layer->param_cols($name)); | ||
| 125 | } | ||
| 126 | $self->{generation}++; | ||
| 127 | return $self; | ||
| 128 | } | ||
| 129 | |||
| 130 | sub n_params ($self) { | ||
| 131 | my $n = 0; | ||
| 132 | $n += @$_ for @{ $self->weights }; | ||
| 133 | return $n; | ||
| 134 | } | ||
| 135 | |||
| 136 | # A list of samples as the backend's batch: rows of numbers, or of tokens. | ||
| 137 | sub _batch ($self, $inputs) { | ||
| 138 | my $input = $self->{input}; | ||
| 139 | return ref $input | ||
| 140 | ? $self->{backend}->tokens([ map { @$_ } @$inputs ], $input->{tokens}) | ||
| 141 | : $self->{backend}->tensor([ map { @$_ } @$inputs ], $input); | ||
| 142 | } | ||
| 143 | |||
| 144 | sub _forward ($self, $inputs) { | ||
| 145 | my $x = $self->_batch($inputs); | ||
| 146 | $x = $_->forward($x) for @{ $self->{layers} }; | ||
| 147 | return $x; | ||
| 148 | } | ||
| 149 | |||
| 150 | # The loss of a batch of [input, target] or [input, target, weight] pairs, | ||
| 151 | # given the network's output for them. A weight multiplies that sample's loss | ||
| 152 | # and gradient; if no pair has one, none is passed on. | ||
| 153 | sub _loss ($self, $out, $pairs) { | ||
| 154 | my @targets = map { $_->[1] } @$pairs; | ||
| 155 | my $weights = (grep { defined $_->[2] } @$pairs) ? [ map { $_->[2] // 1 } @$pairs ] : undef; | ||
| 156 | return $self->{loss} eq 'softmax' | ||
| 157 | ? $self->{backend}->softmax_ce($out, \@targets, $weights) | ||
| 158 | : $self->{backend}->mse($out, [ map { @$_ } @targets ], $weights); | ||
| 159 | } | ||
| 160 | |||
| 161 | # The raw output for one sample: logits under the softmax loss. | ||
| 162 | sub forward ($self, $x) { | ||
| 163 | return $self->{backend}->flat($self->_forward([$x])); | ||
| 164 | } | ||
| 165 | |||
| 166 | # What a caller wants to see: class probabilities under the softmax loss, | ||
| 167 | # the output itself otherwise. | ||
| 168 | sub predict ($self, $x) { | ||
| 169 | my $out = $self->forward($x); | ||
| 170 | return $out if $self->{loss} ne 'softmax'; | ||
| 171 | my $max = $out->[0]; | ||
| 172 | for my $v (@$out) { $max = $v if $v > $max } | ||
| 173 | my @p = map { exp($_ - $max) } @$out; | ||
| 174 | my $sum = 0; | ||
| 175 | $sum += $_ for @p; | ||
| 176 | return [ map { $_ / $sum } @p ]; | ||
| 177 | } | ||
| 178 | |||
| 179 | # The most probable class of each of many samples. | ||
| 180 | sub classify_all ($self, $inputs) { | ||
| 181 | my @classes; | ||
| 182 | for (my $start = 0; $start < @$inputs; $start += $EVAL_BATCH) { | ||
| 183 | my $end = $start + $EVAL_BATCH - 1; | ||
| 184 | $end = $#$inputs if $end > $#$inputs; | ||
| 185 | my $out = $self->_forward([ @$inputs[ $start .. $end ] ]); | ||
| 186 | push @classes, @{ $self->{backend}->argmax_rows($out, $self->{n_out}) }; | ||
| 187 | } | ||
| 188 | return \@classes; | ||
| 189 | } | ||
| 190 | |||
| 191 | sub classify ($self, $x) { return $self->classify_all([$x])->[0] } | ||
| 192 | |||
| 193 | # The summed loss of [input, target] pairs, without touching the gradients. | ||
| 194 | sub loss ($self, $pairs) { | ||
| 195 | my $total = 0; | ||
| 196 | for (my $start = 0; $start < @$pairs; $start += $EVAL_BATCH) { | ||
| 197 | my $end = $start + $EVAL_BATCH - 1; | ||
| 198 | $end = $#$pairs if $end > $#$pairs; | ||
| 199 | my @chunk = @$pairs[ $start .. $end ]; | ||
| 200 | my ($loss) = $self->_loss($self->_forward([ map { $_->[0] } @chunk ]), \@chunk); | ||
| 201 | $total += $loss; | ||
| 202 | } | ||
| 203 | return $total; | ||
| 204 | } | ||
| 205 | |||
| 206 | # Forward and backward for a batch of [input, target] pairs: leaves the | ||
| 207 | # batch's summed gradient with the layers and returns its summed loss. | ||
| 208 | sub backprop ($self, $pairs) { | ||
| 209 | my $out = $self->_forward([ map { $_->[0] } @$pairs ]); | ||
| 210 | my ($loss, $grad) = $self->_loss($out, $pairs); | ||
| 211 | for my $layer (reverse @{ $self->{layers} }) { | ||
| 212 | $grad = $layer->backward($grad); | ||
| 213 | } | ||
| 214 | return $loss; | ||
| 215 | } | ||
| 216 | |||
| 217 | # The samples of a training run, left with a backend that can keep them: then | ||
| 218 | # a step picks its batch where the arithmetic is, and nothing is carried | ||
| 219 | # there and back for it (on a graphics card that carrying is what costs). | ||
| 220 | # Nothing if the backend does not keep samples, or not such as these. | ||
| 221 | sub _kept ($self, $data) { | ||
| 222 | my $backend = $self->{backend}; | ||
| 223 | return if !$EPOCH_ON_BACKEND || !ref $self->{input} || $self->{loss} ne 'softmax' || !$backend->can('epoch_data'); | ||
| 224 | return $backend->epoch_data([ map { @{ $_->[0] } } @$data ], $self->{input}{tokens}, [ map { $_->[1] } @$data ], | ||
| 225 | (grep { defined $_->[2] } @$data) ? [ map { $_->[2] // 1 } @$data ] : undef); | ||
| 226 | } | ||
| 227 | |||
| 228 | # One training step on the samples $order->[$start .. $end]: forward, | ||
| 229 | # backward, update. Returns the batch's loss, or nothing where the samples | ||
| 230 | # are kept by the backend and their losses with them. | ||
| 231 | sub _step ($self, $optimizer, $kept, $data, $order, $start, $end) { | ||
| 232 | my $loss = 0; | ||
| 233 | if ($kept) { | ||
| 234 | my $backend = $self->{backend}; | ||
| 235 | my $x = $backend->batch_at($kept, $start, $end - $start + 1); | ||
| 236 | $x = $_->forward($x) for @{ $self->{layers} }; | ||
| 237 | my $grad = $backend->softmax_ce_at($x, $kept, $start); | ||
| 238 | $grad = $_->backward($grad) for reverse @{ $self->{layers} }; | ||
| 239 | } | ||
| 240 | else { $loss = $self->backprop([ @$data[ @$order[ $start .. $end ] ] ]) } | ||
| 241 | $optimizer->step(1 / ($end - $start + 1)); | ||
| 242 | $self->{generation}++; | ||
| 243 | return $loss; | ||
| 244 | } | ||
| 245 | |||
| 246 | # The network on another backend, with these weights. | ||
| 247 | sub _move_to ($self, $backend, $weights) { | ||
| 248 | $self->{backend} = $backend; | ||
| 249 | $_->{backend} = $backend for @{ $self->{layers} }; | ||
| 250 | my @params = $self->params; | ||
| 251 | for my $p (0 .. $#params) { | ||
| 252 | my ($layer, $name) = @{ $params[$p] }; | ||
| 253 | $layer->{$name} = $backend->tensor($weights->[$p], $layer->param_cols($name)); | ||
| 254 | } | ||
| 255 | $self->{generation}++; | ||
| 256 | return; | ||
| 257 | } | ||
| 258 | |||
| 259 | # Seconds a training step takes on the backend the network is on now, timed | ||
| 260 | # on the first samples: the fastest of a few rounds of steps. The weights are | ||
| 261 | # afterwards what they were. With $beat, a time per step to beat, it gives | ||
| 262 | # up once it is hopeless, and tries one step on a handful of samples before | ||
| 263 | # anything else: a step on the whole batch cannot be faster than that. | ||
| 264 | sub _seconds_per_step ($self, $data, $batch, $weights, $beat = undef) { | ||
| 265 | my $backend = $self->{backend}; | ||
| 266 | my $last = $batch * ($TRIAL_WARM + $TRIAL_STEPS) - 1; | ||
| 267 | $last = $#$data if $last > $#$data; | ||
| 268 | my @sample = @$data[ 0 .. $last ]; | ||
| 269 | my @order = 0 .. $last; | ||
| 270 | my $optimizer = Peta::NN::Optimizer->new(kind => 'adam', backend => $backend, params => [ $self->params ]); | ||
| 271 | # Reading a bias back waits for whatever a card has been sent. | ||
| 272 | my $wait = sub { $backend->flat(($self->params)[-1][0]{ ($self->params)[-1][1] }) }; | ||
| 273 | my $best; | ||
| 274 | if (defined $beat && $batch > $PROBE) { | ||
| 275 | $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]); # the first step of all sets things up | ||
| 276 | my $began = time; | ||
| 277 | $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]); | ||
| 278 | $wait->(); | ||
| 279 | $best = 9**9**9 if time - $began > $HOPELESS * $beat; | ||
| 280 | } | ||
| 281 | if (!defined $best) { | ||
| 282 | my $kept = $self->_kept(\@sample); | ||
| 283 | $backend->epoch_order($kept, \@order) if $kept; | ||
| 284 | my $steps = sub ($from, $count) { | ||
| 285 | $self->_step($optimizer, $kept, \@sample, \@order, $_ * $batch, ($_ + 1) * $batch - 1) for $from .. $from + $count - 1; | ||
| 286 | }; | ||
| 287 | $steps->(0, $TRIAL_WARM); | ||
| 288 | my ($stale, $since) = (0, time); | ||
| 289 | while (1) { # the same samples round after round: it is the time that is wanted | ||
| 290 | $wait->(); | ||
| 291 | my $began = time; | ||
| 292 | $steps->($TRIAL_WARM, $TRIAL_STEPS); | ||
| 293 | $wait->(); | ||
| 294 | my $round = (time - $began) / $TRIAL_STEPS; | ||
| 295 | $stale = defined $best && $round > $best * (1 - $SETTLED) ? $stale + 1 : 0; | ||
| 296 | $best = $round if !defined $best || $round < $best; | ||
| 297 | my $spent = time - $since; | ||
| 298 | last if $spent >= $TRIAL_TIME[1] || $stale >= 2 && $spent >= $TRIAL_TIME[0] || defined $beat && $best > $HOPELESS * $beat; | ||
| 299 | } | ||
| 300 | } | ||
| 301 | $self->_move_to($backend, $weights); | ||
| 302 | return $best; | ||
| 303 | } | ||
| 304 | |||
| 305 | # Settle the backend of a network that was left the choice, for a run of | ||
| 306 | # this many samples, at this batch, for this many epochs. | ||
| 307 | sub _settle ($self, $data, $batch, $epochs) { | ||
| 308 | # The same work as the last time it was settled: the same answer. | ||
| 309 | my $steps = $epochs * (int($#$data / $batch) + 1); | ||
| 310 | my $work = join ' ', scalar @$data, $batch, $steps; | ||
| 311 | return if $self->{settled} && $self->{settled}{work} eq $work; | ||
| 312 | delete $self->{settled}; | ||
| 313 | return if @$data < $batch * ($TRIAL_WARM + $TRIAL_STEPS); # too few steps to time a round | ||
| 314 | my $weights = $self->weights; | ||
| 315 | my $count = $self->{generation}; | ||
| 316 | my $here = $self->{backend}; | ||
| 317 | my %seconds = ($here->name => $self->_seconds_per_step($data, $batch, $weights)); | ||
| 318 | my ($best, $on) = ($seconds{ $here->name }, $here); | ||
| 319 | if ($best * $steps >= $WORTH) { | ||
| 320 | for my $name (grep { $_ ne $here->name } Peta::NN::Backend::names()) { | ||
| 321 | my $other = Peta::NN::Backend::try($name) or next; | ||
| 322 | $self->_move_to($other, $weights); | ||
| 323 | $seconds{$name} = $self->_seconds_per_step($data, $batch, $weights, $best); | ||
| 324 | ($best, $on) = ($seconds{$name}, $other) if $seconds{$name} < $best; | ||
| 325 | } | ||
| 326 | # A close second is timed once more: the first time anything runs it is | ||
| 327 | # not at its best (a JIT has not compiled it yet, a card is not up to | ||
| 328 | # speed), and by how much differs from backend to backend. | ||
| 329 | my %backend = ($here->name => $here); | ||
| 330 | for my $name (grep { $seconds{$_} <= $HOPELESS * $best } sort keys %seconds) { | ||
| 331 | $backend{$name} //= Peta::NN::Backend::try($name); | ||
| 332 | $self->_move_to($backend{$name}, $weights); | ||
| 333 | my $again = $self->_seconds_per_step($data, $batch, $weights); | ||
| 334 | $seconds{$name} = $again if $again < $seconds{$name}; | ||
| 335 | } | ||
| 336 | my ($fastest) = sort { $seconds{$a} <=> $seconds{$b} } keys %seconds; | ||
| 337 | ($best, $on) = ($seconds{$fastest}, $backend{$fastest} // Peta::NN::Backend::try($fastest)); | ||
| 338 | } | ||
| 339 | $self->_move_to($on, $weights); | ||
| 340 | $self->{generation} = $count; # nothing has changed: the weights are what they were | ||
| 341 | delete @seconds{ grep { $seconds{$_} == 9**9**9 } keys %seconds }; # those that were not worth timing | ||
| 342 | $self->{settled} = { backend => $on->name, steps => $steps, seconds_per_step => \%seconds, work => $work }; | ||
| 343 | return; | ||
| 344 | } | ||
| 345 | |||
| 346 | # Train on [input, target] or [input, target, weight] pairs. Returns the mean | ||
| 347 | # loss of the last epoch. | ||
| 348 | # epochs, batch how long, and how many samples per update | ||
| 349 | # optimizer, lr, ... passed to Peta::NN::Optimizer, weight_decay included | ||
| 350 | # lr_decay the learning rate is multiplied by this after every epoch | ||
| 351 | # validate pairs that are not trained on; their loss is measured every epoch | ||
| 352 | # patience stop after this many epochs without improvement of the | ||
| 353 | # validation loss (the training loss if there is no | ||
| 354 | # validation set), and return to the best epoch's weights | ||
| 355 | # min_delta what counts as an improvement; default any | ||
| 356 | # watch sub (training loss, validation loss) returning the number | ||
| 357 | # patience watches instead; lower is better | ||
| 358 | # target_loss stop once an epoch's mean loss is at or below this | ||
| 359 | # on_epoch called with (epoch, mean loss); a false return stops. | ||
| 360 | # history() has the validation loss as well | ||
| 361 | sub train ($self, %arg) { | ||
| 362 | my $data = $arg{data} // die "data => [[input, target], ...] is required\n"; | ||
| 363 | my $epochs = $arg{epochs} // 10; | ||
| 364 | my $batch = $arg{batch} // 16; | ||
| 365 | $self->_settle($data, $batch, $epochs) if $self->{auto}; | ||
| 366 | my $optimizer = Peta::NN::Optimizer->new( | ||
| 367 | kind => $arg{optimizer} // 'adam', | ||
| 368 | backend => $self->{backend}, | ||
| 369 | params => [ $self->params ], | ||
| 370 | map { $_ => $arg{$_} } grep { defined $arg{$_} } qw(lr momentum beta1 beta2 epsilon weight_decay), | ||
| 371 | ); | ||
| 372 | |||
| 373 | my $validate = $arg{validate}; | ||
| 374 | my $patience = $arg{patience}; | ||
| 375 | my $min_delta = $arg{min_delta} // 0; | ||
| 376 | my ($best, $best_weights, $stale) = (undef, undef, 0); | ||
| 377 | |||
| 378 | my $backend = $self->{backend}; | ||
| 379 | my $kept = $self->_kept($data); | ||
| 380 | |||
| 381 | my @order = 0 .. $#$data; | ||
| 382 | my $mean = 0; | ||
| 383 | $self->{history} = []; | ||
| 384 | for my $epoch (1 .. $epochs) { | ||
| 385 | $self->{rng}->shuffle(\@order); | ||
| 386 | my $total = 0; | ||
| 387 | $backend->epoch_order($kept, \@order) if $kept; | ||
| 388 | for (my $start = 0; $start < @order; $start += $batch) { | ||
| 389 | my $end = $start + $batch - 1; | ||
| 390 | $end = $#order if $end > $#order; | ||
| 391 | $total += $self->_step($optimizer, $kept, $data, \@order, $start, $end); | ||
| 392 | } | ||
| 393 | $total = $backend->epoch_loss($kept) if $kept; | ||
| 394 | $mean = $total / @order; | ||
| 395 | my $held = $validate && @$validate ? $self->loss($validate) / @$validate : undef; | ||
| 396 | push @{ $self->{history} }, { epoch => $epoch, loss => $mean, validation => $held }; | ||
| 397 | |||
| 398 | # The loss that is watched: the validation set's if there is one. The | ||
| 399 | # weights of its best epoch are kept and put back at the end. | ||
| 400 | if (defined $patience) { | ||
| 401 | my $watched = $arg{watch} ? $arg{watch}->($mean, $held) : $held // $mean; | ||
| 402 | if (!defined $best || $watched < $best - $min_delta) { | ||
| 403 | ($best, $best_weights, $stale) = ($watched, $self->weights, 0); | ||
| 404 | } | ||
| 405 | elsif (++$stale >= $patience) { last } | ||
| 406 | } | ||
| 407 | last if $arg{on_epoch} && !$arg{on_epoch}->($epoch, $mean); | ||
| 408 | last if defined $arg{target_loss} && $mean <= $arg{target_loss}; | ||
| 409 | $optimizer->{lr} *= $arg{lr_decay} if defined $arg{lr_decay}; | ||
| 410 | } | ||
| 411 | $self->set_weights($best_weights) if $best_weights; | ||
| 412 | return $mean; | ||
| 413 | } | ||
| 414 | |||
| 415 | # What every epoch of the last train() measured: { epoch, loss, validation }. | ||
| 416 | sub history ($self) { return @{ $self->{history} // [] } } | ||
| 417 | |||
| 418 | # Share of [input, class index] pairs classified correctly. | ||
| 419 | sub accuracy ($self, $pairs) { | ||
| 420 | my $classes = $self->classify_all([ map { $_->[0] } @$pairs ]); | ||
| 421 | my $right = grep { $classes->[$_] == $pairs->[$_][1] } 0 .. $#$pairs; | ||
| 422 | return $right / @$pairs; | ||
| 423 | } | ||
| 424 | |||
| 425 | # What it takes to rebuild this net: its definition and its weights. The | ||
| 426 | # backend is not part of it; a net trained on one loads on any other. | ||
| 427 | sub state ($self) { | ||
| 428 | return { | ||
| 429 | version => $VERSION, | ||
| 430 | input => $self->{input}, | ||
| 431 | loss => $self->{loss}, | ||
| 432 | seed => $self->{seed}, | ||
| 433 | layers => [ map { $_->spec } @{ $self->{layers} } ], | ||
| 434 | weights => $self->weights, | ||
| 435 | }; | ||
| 436 | } | ||
| 437 | |||
| 438 | sub from_state ($class, $state, %arg) { | ||
| 439 | return $class->new(%$state{qw(input loss seed layers)}, %arg)->set_weights($state->{weights}); | ||
| 440 | } | ||
| 441 | |||
| 442 | # Storable's portable format writes a double as decimal text and loses its | ||
| 443 | # last bits. Packed little-endian it stays exact, and portable all the same. | ||
| 444 | sub freeze_state ($state) { | ||
| 445 | return { %$state, weights => [ map { pack 'd<*', @$_ } @{ $state->{weights} } ] }; | ||
| 446 | } | ||
| 447 | |||
| 448 | sub thaw_state ($frozen) { | ||
| 449 | return { %$frozen, weights => [ map { [ unpack 'd<*', $_ ] } @{ $frozen->{weights} } ] }; | ||
| 450 | } | ||
| 451 | |||
| 452 | # A network's state file, by convention *.net. | ||
| 453 | sub save ($self, $file) { | ||
| 454 | Storable::nstore({ %{ freeze_state($self->state) }, format => $STATE_FORMAT }, $file); | ||
| 455 | return $self; | ||
| 456 | } | ||
| 457 | |||
| 458 | # The file is read as plain data, and weights that do not fit the definition | ||
| 459 | # in it are refused. | ||
| 460 | sub load ($class, $file, %arg) { | ||
| 461 | my $frozen = Peta::NN::Inference::read_file($file, $STATE_FORMAT); | ||
| 462 | die "$file is a malformed network state\n" | ||
| 463 | if ref $frozen->{layers} ne 'ARRAY' || ref $frozen->{weights} ne 'ARRAY' || grep { ref } @{ $frozen->{weights} }; | ||
| 464 | return $class->from_state(thaw_state($frozen), %arg); | ||
| 465 | } | ||
| 466 | |||
| 467 | 1; | ||
| 468 | |||
| 469 | __END__ |