lib/Peta/NN.pm

filter: Statementstmt 95.5% 302 / 316 Branchbran 86.0% 86 / 100 Conditioncond 71.9% 64 / 89 MC/DCmcdc 45.8% 22 / 48 Subroutinesub 97.8% 46 / 47 Podpod 100% 26 / 26 totaltotal 87.2% 546 / 626 CC 101 SCAR 23.8
CC
101
Cov
90%
train
27.2
_kept
18.1
_seconds_per_step
17.0
CRAP 10.9
« lib/Peta/NN/Model.pm ↑ Summary lib/Peta/NN/Chain.pm »
1 package Peta::NN;
2 # ABSTRACT: a small neural network, defined, trained and saved in Perl
3  
4 # A small feed-forward neural network: a list of layers, a loss, and a
5 # training loop. A mini-batch goes through as one tensor; what a tensor is,
6 # and how the arithmetic on it is done, is the backend's business.
7  
8 27 ▶use v5.36;
Subroutine BEGIN: called CC=1 SCAR=0.0
9  
10 27 ▶use Storable ();
Subroutine BEGIN: called CC=1 SCAR=0.0
11 27 ▶use Time::HiRes qw(time);
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 27 ▶use Peta::NN::RNG;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 27 ▶use Peta::NN::Backend;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 27 ▶use Peta::NN::Inference;
Subroutine BEGIN: called CC=1 SCAR=0.0
16 27 ▶use Peta::NN::Optimizer;
Subroutine BEGIN: called CC=1 SCAR=0.0
17 27 ▶use Peta::NN::Layer::Dense;
Subroutine BEGIN: called CC=1 SCAR=0.0
18 27 ▶use Peta::NN::Layer::Embed;
Subroutine BEGIN: called CC=1 SCAR=0.0
19 27 ▶use Peta::NN::Layer::Activation;
Subroutine BEGIN: called CC=1 SCAR=0.0
20  
21 27 our $VERSION = '0.2610090';
22  
23 27 my %IS_ACTIVATION = map { $_ => 1 } Peta::NN::Layer::Activation::kinds();
24 27 my %IS_LOSS = map { $_ => 1 } qw(softmax mse);
25  
26 27 my $EVAL_BATCH = 256; # samples per forward pass when only predicting
27  
28 # Whether train() leaves the samples with a backend that can keep them. On
29 # unless a test wants the two ways compared.
30 27 our $EPOCH_ON_BACKEND = 1;
31  
32 # A network that was left the choice of its backend ('auto') settles it when
33 # it is trained: only then is it known how much work there is. If the run
34 # would take the backend it stands on $WORTH seconds or more, a few steps
35 # are timed on every backend this perl and this machine have, and the run
36 # goes to the fastest. Nothing is assumed about which that is: plain Perl
37 # under a JIT, PDL and a graphics card each win somewhere, and each may be
38 # missing. $TRIAL_WARM steps are run before the $TRIAL_STEPS that are timed.
39 27 our $WORTH = 2;
40 27 our $TRIAL_STEPS = 6; # steps in a timed round
41 27 our $TRIAL_WARM = 2;
42 27 my @TRIAL_TIME = (0.3, 1); # seconds of timed rounds, at least and at most; the fastest round counts. The
43 # first steps of anything are slow, and a card that sat idle takes a few tenths
44 # of a second to come up to speed
45 27 my $SETTLED = 0.05; # two rounds in a row that do not beat the best by this share: it is found
46 27 my $PROBE = 16; # samples in the one small step that is tried first
47 27 my $HOPELESS = 3; # a backend this many times slower than the best so far is not timed to the end
48  
49 27 my $STATE_FORMAT = 'Peta::NN network state';
50  
51 # A layer is written [dense => N], [embed => DIM], or an activation's name.
52 572 ▶sub _build_layer ($spec) {
Subroutine _build_layer: called CC=5 SCAR=16.1
53 572 ▶ if (!ref $spec) {
Branch: 1 branch
BranchTrueFalseTotal
unless (ref $spec) 153 419 572
54 153 ▶ die "unknown layer '$spec'\n" if !$IS_ACTIVATION{$spec};
Branch: 1 branch
BranchTrueFalseTotal
unless $IS_ACTIVATION{$spec} 1 152 153
55 152 return Peta::NN::Layer::Activation->new(kind => $spec);
56 }
57 419 my ($kind, $size) = @$spec;
58 419 ▶ return Peta::NN::Layer::Dense->new(out => $size) if $kind eq 'dense';
Branch: 1 branch
BranchTrueFalseTotal
if $kind eq "dense" 302 117 419
59 117 ▶ return Peta::NN::Layer::Embed->new(dim => $size) if $kind eq 'embed';
Branch: 1 branch
BranchTrueFalseTotal
if $kind eq "embed" 116 1 117
60 1 die "unknown layer '$kind'\n";
61 }
62  
63 156 ▶sub new ($class, %arg) {
Subroutine new: called CC=11 SCAR=24.1
Pod documented
64 my $self = bless {
65 input => $arg{input} // die("input => size, or { tokens =>, vocab => }, is required\n"),
66 loss => $arg{loss} // 'softmax',
67 156 ▶ seed => $arg{seed} // 1,
Condition83%5/6
$arg{"input"} // die("input => size, or { tokens =>, vocab => }, is required\n")
l!l
1560
$arg{"loss"} // "softmax"
l!l
13125
$arg{"seed"} // 1
l!l
1497
MC/DC0%0/1
$arg{"input"} // die("input => size, or { tokens =>, vocab => }, is required\n")
$arg{"input"}
MC/DC100%1/1
$arg{"loss"} // "softmax"
$arg{"loss"}
MC/DC100%1/1
$arg{"seed"} // 1
$arg{"seed"}
Truth table50%1/2
A
A= $arg{"input"}
Aresult
11
00
Truth table100%2/2
A
A= $arg{"loss"}
Aresult
11
00
Truth table100%2/2
A
A= $arg{"seed"}
Aresult
11
00
68 }, $class;
69 156 ▶ die "unknown loss '$self->{loss}' (have: @{[ sort keys %IS_LOSS ]})\n" if !$IS_LOSS{ $self->{loss} };
Branch: 1 branch
BranchTrueFalseTotal
unless $IS_LOSS{$self->{"loss"}} 1 155 156
70 155 ▶ my $backend = $arg{backend} // $ENV{PETA_NN_BACKEND} // 'plain';
Condition80%4/5
($arg{"backend"} // $ENV{"PETA_NN_BACKEND"}) // "plain"
l!l
9065
$arg{"backend"} // $ENV{"PETA_NN_BACKEND"}
l!l&&r!l&&!r
90065
MC/DC50%1/2
($arg{"backend"} // $ENV{"PETA_NN_BACKEND"}) // "plain"
$arg{"backend"} $ENV{"PETA_NN_BACKEND"}
Truth table66%2/3
A // B
A= $arg{"backend"}B= $ENV{"PETA_NN_BACKEND"}
ABresult
1X1
011
000
71 155 $self->{auto} = $backend eq 'auto';
72 155 $self->{backend} = Peta::NN::Backend::create($backend);
73 154 $self->{rng} = Peta::NN::RNG->new($self->{seed});
74 154 $self->{generation} = 0;
75 154 ▶ $self->{layers} = [ map { _build_layer($_) } @{ $arg{layers} // die "layers => [...] is required\n" } ];
Condition50%1/2
$arg{"layers"} // die("layers => [...] is required\n")
l!l
1540
MC/DC0%0/1
$arg{"layers"} // die("layers => [...] is required\n")
$arg{"layers"}
Truth table50%1/2
A
A= $arg{"layers"}
Aresult
11
00
76  
77 152 my $size = $self->{input};
78 152 $size = $_->init($size, $self->{rng}, $self->{backend}) for @{ $self->{layers} };
79 150 ▶ die "the network needs at least one layer\n" if ref $size;
Branch: 1 branch
BranchTrueFalseTotal
if ref $size 0 150 150
80 150 $self->{n_out} = $size;
81  
82 # Nothing consumes the first layer's input gradient.
83 150 my $first = $self->{layers}[0];
84 150 ▶ $first->{need_dx} = 0 if $first->type eq 'dense';
Branch: 1 branch
BranchTrueFalseTotal
if $first->type eq "dense" 35 115 150
85 150 return $self;
86 }
87  
88 0 ▶sub n_out ($self) { return $self->{n_out} }
Subroutine n_out: not called CC=1 SCAR=6.9
Pod documented
89  
90 # A number that changes whenever the weights do, for whoever caches something
91 # derived from them.
92 6423 ▶sub generation ($self) { return $self->{generation} }
Subroutine generation: called CC=1 SCAR=0.0
Pod documented
93 788 ▶sub layers ($self) { return @{ $self->{layers} } }
Subroutine layers: called CC=1 SCAR=0.0
Pod documented
94 1481 ▶sub backend ($self) { return $self->{backend}->name }
Subroutine backend: called CC=1 SCAR=0.0
Pod documented
95  
96 # How the backend was settled by the last train(), for a network that was
97 # left the choice: { backend, steps, seconds_per_step => { backend => seconds } },
98 # the timings being those that were taken. Nothing for any other network, or
99 # for a run of too few steps to time.
100 11 ▶sub settled ($self) { return $self->{settled} }
Subroutine settled: called CC=1 SCAR=0.0
Pod documented
101  
102 # Every parameter as [layer, name], in layer order: the tensor is
103 # $layer->{name}, its gradient after backprop() is $layer->{"g$name"}.
104 16561 ▶sub params ($self) {
Subroutine params: called CC=1 SCAR=0.0
Pod documented
105 16561 return map { my $layer = $_; map { [ $layer, $_ ] } $layer->param_names } @{ $self->{layers} };
106 }
107  
108 # The parameters, and their gradients, as flat Perl lists in params() order.
109 1140 ▶sub weights ($self) {
Subroutine weights: called CC=1 SCAR=0.0
Pod documented
110 1140 return [ map { $self->{backend}->flat($_->[0]{ $_->[1] }) } $self->params ];
111 }
112  
113 30 ▶sub gradients ($self) {
Subroutine gradients: called CC=1 SCAR=0.0
Pod documented
114 30 return [ map { $self->{backend}->flat($_->[0]{"g$_->[1]"}) } $self->params ];
115 }
116  
117 1696 ▶sub set_weights ($self, $lists) {
Subroutine set_weights: called CC=4 SCAR=13.9
Pod documented
118 1696 my @params = $self->params;
119 1696 ▶ die "these weights do not fit this network\n" if @params != @$lists;
Branch: 1 branch
BranchTrueFalseTotal
if @params != @$lists 0 1696 1696
120 1696 for my $p (0 .. $#params) {
121 7871 my ($layer, $name) = @{ $params[$p] };
122 die "these weights do not fit this network\n"
123 7871 ▶ if @{ $lists->[$p] } != @{ $self->{backend}->flat($layer->{$name}) };
Branch: 1 branch
BranchTrueFalseTotal
if @{$lists->[$p];} != @{$self->{"backend"}->flat($layer->{$name});} 1 7870 7871
124 7870 $layer->{$name} = $self->{backend}->tensor($lists->[$p], $layer->param_cols($name));
125 }
126 1695 $self->{generation}++;
127 1695 return $self;
128 }
129  
130 62 ▶sub n_params ($self) {
Subroutine n_params: called CC=2 SCAR=6.9
Pod documented
131 62 my $n = 0;
132 62 $n += @$_ for @{ $self->weights };
133 62 return $n;
134 }
135  
136 # A list of samples as the backend's batch: rows of numbers, or of tokens.
137 95387 ▶sub _batch ($self, $inputs) {
Subroutine _batch: called CC=2 SCAR=6.9
138 95387 my $input = $self->{input};
139 return ref $input
140 1186853 ? $self->{backend}->tokens([ map { @$_ } @$inputs ], $input->{tokens})
141 95387 ▶ : $self->{backend}->tensor([ map { @$_ } @$inputs ], $input);
Branch: 1 branch
BranchTrueFalseTotal
ref $input ? : 70245 25142 95387
142 }
143  
144 95387 ▶sub _forward ($self, $inputs) {
Subroutine _forward: called CC=2 SCAR=6.9
145 95387 my $x = $self->_batch($inputs);
146 95387 $x = $_->forward($x) for @{ $self->{layers} };
147 95387 return $x;
148 }
149  
150 # The loss of a batch of [input, target] or [input, target, weight] pairs,
151 # given the network's output for them. A weight multiplies that sample's loss
152 # and gradient; if no pair has one, none is passed on.
153 95251 ▶sub _loss ($self, $out, $pairs) {
Subroutine _loss: called CC=4 SCAR=13.9
154 95251 my @targets = map { $_->[1] } @$pairs;
155 95251 ▶ my $weights = (grep { defined $_->[2] } @$pairs) ? [ map { $_->[2] // 1 } @$pairs ] : undef;
Branch: 1 branch
BranchTrueFalseTotal
grep({defined $_->[2];} @$pairs) ? : 16562 78689 95251
Condition100%2/2
$_->[2] // 1
l!l
2629492
MC/DC100%1/1
$_->[2] // 1
$_->[2]
Truth table100%2/2
A
A= $_->[2]
Aresult
11
00
156 return $self->{loss} eq 'softmax'
157 ? $self->{backend}->softmax_ce($out, \@targets, $weights)
158 95251 ▶ : $self->{backend}->mse($out, [ map { @$_ } @targets ], $weights);
Branch: 1 branch
BranchTrueFalseTotal
$self->{'loss'} eq 'softmax' ? : 78607 16644 95251
159 }
160  
161 # The raw output for one sample: logits under the softmax loss.
162 130 ▶sub forward ($self, $x) {
Subroutine forward: called CC=1 SCAR=0.0
Pod documented
163 130 return $self->{backend}->flat($self->_forward([$x]));
164 }
165  
166 # What a caller wants to see: class probabilities under the softmax loss,
167 # the output itself otherwise.
168 122 ▶sub predict ($self, $x) {
Subroutine predict: called CC=5 SCAR=16.1
Pod documented
169 122 my $out = $self->forward($x);
170 122 ▶ return $out if $self->{loss} ne 'softmax';
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"loss"} ne "softmax" 120 2 122
171 2 my $max = $out->[0];
172 2 ▶ for my $v (@$out) { $max = $v if $v > $max }
Branch: 1 branch
BranchTrueFalseTotal
if $v > $max 2 2 4
173 2 my @p = map { exp($_ - $max) } @$out;
174 2 my $sum = 0;
175 2 $sum += $_ for @p;
176 2 return [ map { $_ / $sum } @p ];
177 }
178  
179 # The most probable class of each of many samples.
180 6 ▶sub classify_all ($self, $inputs) {
Subroutine classify_all: called CC=3 SCAR=11.0
Pod documented
181 6 my @classes;
182 6 ▶ for (my $start = 0; $start < @$inputs; $start += $EVAL_BATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if ($start < @$inputs) 6 6 12
183 6 my $end = $start + $EVAL_BATCH - 1;
184 6 ▶ $end = $#$inputs if $end > $#$inputs;
Branch: 1 branch
BranchTrueFalseTotal
if $end > $inputs->$#* 6 0 6
185 6 my $out = $self->_forward([ @$inputs[ $start .. $end ] ]);
186 6 push @classes, @{ $self->{backend}->argmax_rows($out, $self->{n_out}) };
187 }
188 6 return \@classes;
189 }
190  
191 2 ▶sub classify ($self, $x) { return $self->classify_all([$x])->[0] }
Subroutine classify: called CC=1 SCAR=0.0
Pod documented
192  
193 # The summed loss of [input, target] pairs, without touching the gradients.
194 2176 ▶sub loss ($self, $pairs) {
Subroutine loss: called CC=3 SCAR=11.0
Pod documented
195 2176 my $total = 0;
196 2176 ▶ for (my $start = 0; $start < @$pairs; $start += $EVAL_BATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if ($start < @$pairs) 2191 2176 4367
197 2191 my $end = $start + $EVAL_BATCH - 1;
198 2191 ▶ $end = $#$pairs if $end > $#$pairs;
Branch: 1 branch
BranchTrueFalseTotal
if $end > $pairs->$#* 2176 15 2191
199 2191 my @chunk = @$pairs[ $start .. $end ];
200 2191 my ($loss) = $self->_loss($self->_forward([ map { $_->[0] } @chunk ]), \@chunk);
201 2191 $total += $loss;
202 }
203 2176 return $total;
204 }
205  
206 # Forward and backward for a batch of [input, target] pairs: leaves the
207 # batch's summed gradient with the layers and returns its summed loss.
208 93060 ▶sub backprop ($self, $pairs) {
Subroutine backprop: called CC=2 SCAR=6.9
Pod documented
209 93060 my $out = $self->_forward([ map { $_->[0] } @$pairs ]);
210 93060 my ($loss, $grad) = $self->_loss($out, $pairs);
211 93060 for my $layer (reverse @{ $self->{layers} }) {
212 348484 $grad = $layer->backward($grad);
213 }
214 93060 return $loss;
215 }
216  
217 # The samples of a training run, left with a backend that can keep them: then
218 # a step picks its batch where the arithmetic is, and nothing is carried
219 # there and back for it (on a graphics card that carrying is what costs).
220 # Nothing if the backend does not keep samples, or not such as these.
221 240 ▶sub _kept ($self, $data) {
Subroutine _kept: called CC=7 SCAR=28.9
222 240 my $backend = $self->{backend};
223 240 ▶ return if !$EPOCH_ON_BACKEND || !ref $self->{input} || $self->{loss} ne 'softmax' || !$backend->can('epoch_data');
Branch: 1 branch
BranchTrueFalseTotal
if not $Peta::NN::EPOCH_ON_BACKEND && ref $self->{"input"} or $self->{"loss"} ne "softmax" or not $backend->can("epoch_data") 240 0 240
Condition66%6/9
not $Peta::NN::EPOCH_ON_BACKEND && ref $self->{"input"} or $self->{"loss"} ne "softmax" or not $backend->can("epoch_data")
l!l&&r!l&&!r
212190
not $Peta::NN::EPOCH_ON_BACKEND && ref $self->{"input"} or $self->{"loss"} ne "softmax"
l!l&&r!l&&!r
210219
$Peta::NN::EPOCH_ON_BACKEND && ref $self->{'input'}
!ll&&!rl&&r
021219
MC/DC0%0/4
not $Peta::NN::EPOCH_ON_BACKEND && ref $self->{"input"} or $self->{"loss"} ne "softmax" or not $backend->can("epoch_data")
$Peta::NN::EPOCH_ON_BACKEND ref $self->{'input'} $self->{"loss"} ne "softmax" not $backend->can("epoch_data")
Truth table40%2/5
not(A && B) or C or not(D)
A= $Peta::NN::EPOCH_ON_BACKENDB= ref $self->{'input'}C= $self->{"loss"} ne "softmax"D= not $backend->can("epoch_data")
ABCDresult
0XXX1
10XX1
111X1
11011
11000
224 0 ▶ return $backend->epoch_data([ map { @{ $_->[0] } } @$data ], $self->{input}{tokens}, [ map { $_->[1] } @$data ],
Branch: 1 branch
BranchTrueFalseTotal
grep({defined $_->[2];} @$data) ? : 0 0 0
225 0 ▶ (grep { defined $_->[2] } @$data) ? [ map { $_->[2] // 1 } @$data ] : undef);
Condition0%0/2
$_->[2] // 1
l!l
00
MC/DC0%0/1
$_->[2] // 1
$_->[2]
Truth table0%0/2
A
A= $_->[2]
Aresult
11
00
226 }
227  
228 # One training step on the samples $order->[$start .. $end]: forward,
229 # backward, update. Returns the batch's loss, or nothing where the samples
230 # are kept by the backend and their losses with them.
231 93028 ▶sub _step ($self, $optimizer, $kept, $data, $order, $start, $end) {
Subroutine _step: called CC=4 SCAR=18.6
232 93028 my $loss = 0;
233 93028 ▶ if ($kept) {
Branch: 1 branch
BranchTrueFalseTotal
if ($kept) { } 0 93028 93028
234 0 my $backend = $self->{backend};
235 0 my $x = $backend->batch_at($kept, $start, $end - $start + 1);
236 0 $x = $_->forward($x) for @{ $self->{layers} };
237 0 my $grad = $backend->softmax_ce_at($x, $kept, $start);
238 0 $grad = $_->backward($grad) for reverse @{ $self->{layers} };
239 }
240 93028 else { $loss = $self->backprop([ @$data[ @$order[ $start .. $end ] ] ]) }
241 93028 $optimizer->step(1 / ($end - $start + 1));
242 93028 $self->{generation}++;
243 93028 return $loss;
244 }
245  
246 # The network on another backend, with these weights.
247 188 ▶sub _move_to ($self, $backend, $weights) {
Subroutine _move_to: called CC=3 SCAR=11.0
248 188 $self->{backend} = $backend;
249 188 $_->{backend} = $backend for @{ $self->{layers} };
250 188 my @params = $self->params;
251 188 for my $p (0 .. $#params) {
252 932 my ($layer, $name) = @{ $params[$p] };
253 932 $layer->{$name} = $backend->tensor($weights->[$p], $layer->param_cols($name));
254 }
255 188 $self->{generation}++;
256 188 return;
257 }
258  
259 # Seconds a training step takes on the backend the network is on now, timed
260 # on the first samples: the fastest of a few rounds of steps. The weights are
261 # afterwards what they were. With $beat, a time per step to beat, it gives
262 # up once it is hopeless, and tries one step on a handful of samples before
263 # anything else: a step on the whole batch cannot be faster than that.
264 94 ▶sub _seconds_per_step ($self, $data, $batch, $weights, $beat = undef) {
Subroutine _seconds_per_step: called CC=17 SCAR=28.4
265 94 my $backend = $self->{backend};
266 94 my $last = $batch * ($TRIAL_WARM + $TRIAL_STEPS) - 1;
267 94 ▶ $last = $#$data if $last > $#$data;
Branch: 1 branch
BranchTrueFalseTotal
if $last > $data->$#* 0 94 94
268 94 my @sample = @$data[ 0 .. $last ];
269 94 my @order = 0 .. $last;
270 94 my $optimizer = Peta::NN::Optimizer->new(kind => 'adam', backend => $backend, params => [ $self->params ]);
271 # Reading a bias back waits for whatever a card has been sent.
272 94 ▶ my $wait = sub { $backend->flat(($self->params)[-1][0]{ ($self->params)[-1][1] }) };
Subroutine __ANON__: called CC=1 SCAR=0.0
273 94 my $best;
274 94 ▶ if (defined $beat && $batch > $PROBE) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $beat and $batch > $PROBE) 1 93 94
Condition100%3/3
defined $beat and $batch > $PROBE
!ll&&!rl&&r
8581
MC/DC100%2/2
defined $beat and $batch > $PROBE
defined $beat $batch > $PROBE
Truth table100%3/3
A and B
A= defined $beatB= $batch > $PROBE
ABresult
0X0
100
111
275 1 $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]); # the first step of all sets things up
276 1 my $began = time;
277 1 $self->backprop([ @sample[ 0 .. $PROBE - 1 ] ]);
278 1 $wait->();
279 1 ▶ $best = 9**9**9 if time - $began > $HOPELESS * $beat;
Branch: 1 branch
BranchTrueFalseTotal
if &Peta::NN::time() - $began > $HOPELESS * $beat 1 0 1
280 }
281 94 ▶ if (!defined $best) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $best) 93 1 94
282 93 my $kept = $self->_kept(\@sample);
283 93 ▶ $backend->epoch_order($kept, \@order) if $kept;
Branch: 1 branch
BranchTrueFalseTotal
if $kept 0 93 93
284 3409 ▶ my $steps = sub ($from, $count) {
Subroutine __ANON__: called CC=2 SCAR=6.9
285 3409 $self->_step($optimizer, $kept, \@sample, \@order, $_ * $batch, ($_ + 1) * $batch - 1) for $from .. $from + $count - 1;
286 93 };
287 93 $steps->(0, $TRIAL_WARM);
288 93 my ($stale, $since) = (0, time);
289 93 while (1) { # the same samples round after round: it is the time that is wanted
290 3316 $wait->();
291 3316 my $began = time;
292 3316 $steps->($TRIAL_WARM, $TRIAL_STEPS);
293 3316 $wait->();
294 3316 my $round = (time - $began) / $TRIAL_STEPS;
295 3316 ▶ $stale = defined $best && $round > $best * (1 - $SETTLED) ? $stale + 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
defined $best && $round > $best * (1 - $SETTLED) ? : 3207 109 3316
Condition100%3/3
defined $best and $round > $best * (1 - $SETTLED)
!ll&&!rl&&r
93163207
MC/DC100%2/2
defined $best and $round > $best * (1 - $SETTLED)
defined $best $round > $best * (1 - $SETTLED)
Truth table100%3/3
A and B
A= defined $bestB= $round > $best * (1 - $SETTLED)
ABresult
0X0
100
111
296 3316 ▶ $best = $round if !defined $best || $round < $best;
Branch: 1 branch
BranchTrueFalseTotal
if not defined $best or $round < $best 472 2844 3316
Condition100%3/3
not defined $best or $round < $best
l!l&&r!l&&!r
933792844
MC/DC100%2/2
not defined $best or $round < $best
not defined $best $round < $best
Truth table100%3/3
not(A) or B
A= not defined $bestB= $round < $best
ABresult
1X1
011
000
297 3316 my $spent = time - $since;
298 3316 ▶ last if $spent >= $TRIAL_TIME[1] || $stale >= 2 && $spent >= $TRIAL_TIME[0] || defined $beat && $best > $HOPELESS * $beat;
Branch: 1 branch
BranchTrueFalseTotal
if $spent >= $TRIAL_TIME[1] or $stale >= 2 and $spent >= $TRIAL_TIME[0] or defined $beat and $best > $HOPELESS * $beat 93 3223 3316
Condition91%11/12
$spent >= $TRIAL_TIME[1] or $stale >= 2 and $spent >= $TRIAL_TIME[0] or defined $beat and $best > $HOPELESS * $beat
l!l&&r!l&&!r
8943223
$spent >= $TRIAL_TIME[1] or $stale >= 2 and $spent >= $TRIAL_TIME[0]
l!l&&r!l&&!r
0893227
$stale >= 2 and $spent >= $TRIAL_TIME[0]
!ll&&!rl&&r
204302389
defined $beat and $best > $HOPELESS * $beat
!ll&&!rl&&r
30641594
MC/DC80%4/5
$spent >= $TRIAL_TIME[1] or $stale >= 2 and $spent >= $TRIAL_TIME[0] or defined $beat and $best > $HOPELESS * $beat
$spent >= $TRIAL_TIME[1] $stale >= 2 $spent >= $TRIAL_TIME[0] defined $beat $best > $HOPELESS * $beat
Truth table75%6/8
A or B and C or D and E
A= $spent >= $TRIAL_TIME[1]B= $stale >= 2C= $spent >= $TRIAL_TIME[0]D= defined $beatE= $best > $HOPELESS * $beat
ABCDEresult
1XXXX1
011XX1
00X111
010111
00X0X0
00X100
0100X0
010100
299 }
300 }
301 94 $self->_move_to($backend, $weights);
302 94 return $best;
303 }
304  
305 # Settle the backend of a network that was left the choice, for a run of
306 # this many samples, at this batch, for this many epochs.
307 75 ▶sub _settle ($self, $data, $batch, $epochs) {
Subroutine _settle: called CC=12 SCAR=24.9
308 # The same work as the last time it was settled: the same answer.
309 75 my $steps = $epochs * (int($#$data / $batch) + 1);
310 75 my $work = join ' ', scalar @$data, $batch, $steps;
311 75 ▶ return if $self->{settled} && $self->{settled}{work} eq $work;
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"settled"} and $self->{"settled"}{"work"} eq $work 2 73 75
Condition100%3/3
$self->{"settled"} and $self->{"settled"}{"work"} eq $work
!ll&&!rl&&r
43302
MC/DC100%2/2
$self->{"settled"} and $self->{"settled"}{"work"} eq $work
$self->{"settled"} $self->{"settled"}{"work"} eq $work
Truth table100%3/3
A and B
A= $self->{"settled"}B= $self->{"settled"}{"work"} eq $work
ABresult
0X0
100
111
312 73 delete $self->{settled};
313 73 ▶ return if @$data < $batch * ($TRIAL_WARM + $TRIAL_STEPS); # too few steps to time a round
Branch: 1 branch
BranchTrueFalseTotal
if @$data < $batch * ($Peta::NN::TRIAL_WARM + $Peta::NN::TRIAL_STEPS) 1 72 73
314 72 my $weights = $self->weights;
315 72 my $count = $self->{generation};
316 72 my $here = $self->{backend};
317 72 my %seconds = ($here->name => $self->_seconds_per_step($data, $batch, $weights));
318 72 my ($best, $on) = ($seconds{ $here->name }, $here);
319 72 ▶ if ($best * $steps >= $WORTH) {
Branch: 1 branch
BranchTrueFalseTotal
if ($best * $steps >= $Peta::NN::WORTH) 9 63 72
320 9 for my $name (grep { $_ ne $here->name } Peta::NN::Backend::names()) {
321 18 ▶ my $other = Peta::NN::Backend::try($name) or next;
Branch: 1 branch
BranchTrueFalseTotal
my $other = Peta::NN::Backend::try($name) or next 9 9 18
322 9 $self->_move_to($other, $weights);
323 9 $seconds{$name} = $self->_seconds_per_step($data, $batch, $weights, $best);
324 9 ▶ ($best, $on) = ($seconds{$name}, $other) if $seconds{$name} < $best;
Branch: 1 branch
BranchTrueFalseTotal
if $seconds{$name} < $best 1 8 9
325 }
326 # A close second is timed once more: the first time anything runs it is
327 # not at its best (a JIT has not compiled it yet, a card is not up to
328 # speed), and by how much differs from backend to backend.
329 9 my %backend = ($here->name => $here);
330 9 for my $name (grep { $seconds{$_} <= $HOPELESS * $best } sort keys %seconds) {
331 13 ▶ $backend{$name} //= Peta::NN::Backend::try($name);
Condition66%2/3
$backend{$name} //= Peta::NN::Backend::try($name)
l!l&&r!l&&!r
940
MC/DC0%0/2
$backend{$name} //= Peta::NN::Backend::try($name)
$backend{$name} Peta::NN::Backend::try($name)
Truth table66%2/3
A //= B
A= $backend{$name}B= Peta::NN::Backend::try($name)
ABresult
1X1
011
000
332 13 $self->_move_to($backend{$name}, $weights);
333 13 my $again = $self->_seconds_per_step($data, $batch, $weights);
334 13 ▶ $seconds{$name} = $again if $again < $seconds{$name};
Branch: 1 branch
BranchTrueFalseTotal
if $again < $seconds{$name} 3 10 13
335 }
336 9 my ($fastest) = sort { $seconds{$a} <=> $seconds{$b} } keys %seconds;
337 9 ▶ ($best, $on) = ($seconds{$fastest}, $backend{$fastest} // Peta::NN::Backend::try($fastest));
Condition33%1/3
$backend{$fastest} // Peta::NN::Backend::try($fastest)
l!l&&r!l&&!r
900
MC/DC0%0/2
$backend{$fastest} // Peta::NN::Backend::try($fastest)
$backend{$fastest} Peta::NN::Backend::try($fastest)
Truth table33%1/3
A // B
A= $backend{$fastest}B= Peta::NN::Backend::try($fastest)
ABresult
1X1
011
000
338 }
339 72 $self->_move_to($on, $weights);
340 72 $self->{generation} = $count; # nothing has changed: the weights are what they were
341 72 delete @seconds{ grep { $seconds{$_} == 9**9**9 } keys %seconds }; # those that were not worth timing
342 72 $self->{settled} = { backend => $on->name, steps => $steps, seconds_per_step => \%seconds, work => $work };
343 72 return;
344 }
345  
346 # Train on [input, target] or [input, target, weight] pairs. Returns the mean
347 # loss of the last epoch.
348 # epochs, batch how long, and how many samples per update
349 # optimizer, lr, ... passed to Peta::NN::Optimizer, weight_decay included
350 # lr_decay the learning rate is multiplied by this after every epoch
351 # validate pairs that are not trained on; their loss is measured every epoch
352 # patience stop after this many epochs without improvement of the
353 # validation loss (the training loss if there is no
354 # validation set), and return to the best epoch's weights
355 # min_delta what counts as an improvement; default any
356 # watch sub (training loss, validation loss) returning the number
357 # patience watches instead; lower is better
358 # target_loss stop once an epoch's mean loss is at or below this
359 # on_epoch called with (epoch, mean loss); a false return stops.
360 # history() has the validation loss as well
361 147 ▶sub train ($self, %arg) {
Subroutine train: called CC=26 SCAR=33.0
Pod documented
362 147 ▶ my $data = $arg{data} // die "data => [[input, target], ...] is required\n";
Condition50%1/2
$arg{"data"} // die("data => [[input, target], ...] is required\n")
l!l
1470
MC/DC0%0/1
$arg{"data"} // die("data => [[input, target], ...] is required\n")
$arg{"data"}
Truth table50%1/2
A
A= $arg{"data"}
Aresult
11
00
363 147 ▶ my $epochs = $arg{epochs} // 10;
Condition50%1/2
$arg{"epochs"} // 10
l!l
1470
MC/DC0%0/1
$arg{"epochs"} // 10
$arg{"epochs"}
Truth table50%1/2
A
A= $arg{"epochs"}
Aresult
11
00
364 147 ▶ my $batch = $arg{batch} // 16;
Condition100%2/2
$arg{"batch"} // 16
l!l
1452
MC/DC100%1/1
$arg{"batch"} // 16
$arg{"batch"}
Truth table100%2/2
A
A= $arg{"batch"}
Aresult
11
00
365 147 ▶ $self->_settle($data, $batch, $epochs) if $self->{auto};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"auto"} 75 72 147
366 my $optimizer = Peta::NN::Optimizer->new(
367 kind => $arg{optimizer} // 'adam',
368 backend => $self->{backend},
369 params => [ $self->params ],
370 147 ▶ map { $_ => $arg{$_} } grep { defined $arg{$_} } qw(lr momentum beta1 beta2 epsilon weight_decay),
Condition50%1/2
$arg{"optimizer"} // "adam"
l!l
0147
MC/DC0%0/1
$arg{"optimizer"} // "adam"
$arg{"optimizer"}
Truth table50%1/2
A
A= $arg{"optimizer"}
Aresult
11
00
371 );
372  
373 147 my $validate = $arg{validate};
374 147 my $patience = $arg{patience};
375 147 ▶ my $min_delta = $arg{min_delta} // 0;
Condition50%1/2
$arg{"min_delta"} // 0
l!l
0147
MC/DC0%0/1
$arg{"min_delta"} // 0
$arg{"min_delta"}
Truth table50%1/2
A
A= $arg{"min_delta"}
Aresult
11
00
376 147 my ($best, $best_weights, $stale) = (undef, undef, 0);
377  
378 147 my $backend = $self->{backend};
379 147 my $kept = $self->_kept($data);
380  
381 147 my @order = 0 .. $#$data;
382 147 my $mean = 0;
383 147 $self->{history} = [];
384 147 for my $epoch (1 .. $epochs) {
385 5149 $self->{rng}->shuffle(\@order);
386 5149 my $total = 0;
387 5149 ▶ $backend->epoch_order($kept, \@order) if $kept;
Branch: 1 branch
BranchTrueFalseTotal
if $kept 0 5149 5149
388 5149 ▶ for (my $start = 0; $start < @order; $start += $batch) {
Branch: 1 branch
BranchTrueFalseTotal
if ($start < @order) 72946 5149 78095
389 72946 my $end = $start + $batch - 1;
390 72946 ▶ $end = $#order if $end > $#order;
Branch: 1 branch
BranchTrueFalseTotal
if $end > $#order 2296 70650 72946
391 72946 $total += $self->_step($optimizer, $kept, $data, \@order, $start, $end);
392 }
393 5149 ▶ $total = $backend->epoch_loss($kept) if $kept;
Branch: 1 branch
BranchTrueFalseTotal
if $kept 0 5149 5149
394 5149 $mean = $total / @order;
395 5149 ▶ my $held = $validate && @$validate ? $self->loss($validate) / @$validate : undef;
Branch: 1 branch
BranchTrueFalseTotal
$validate && @$validate ? : 573 4576 5149
Condition66%2/3
$validate and @$validate
!ll&&!rl&&r
45760573
MC/DC50%1/2
$validate and @$validate
$validate @$validate
Truth table66%2/3
A and B
A= $validateB= @$validate
ABresult
0X0
100
111
396 5149 push @{ $self->{history} }, { epoch => $epoch, loss => $mean, validation => $held };
397  
398 # The loss that is watched: the validation set's if there is one. The
399 # weights of its best epoch are kept and put back at the end.
400 5149 ▶ if (defined $patience) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $patience) 554 4595 5149
401 554 ▶ my $watched = $arg{watch} ? $arg{watch}->($mean, $held) : $held // $mean;
Branch: 1 branch
BranchTrueFalseTotal
$arg{'watch'} ? : 522 32 554
Condition33%1/3
$held // $mean
l!l&&r!l&&!r
3200
MC/DC0%0/2
$held // $mean
$held $mean
Truth table33%1/3
A // B
A= $heldB= $mean
ABresult
1X1
011
000
402 554 ▶ if (!defined $best || $watched < $best - $min_delta) {
Branch: 2 branches
BranchTrueFalseTotal
if (not defined $best or $watched < $best - $min_delta) { } 151 403 554
elsif (++$stale >= $patience) { } 2 401 403
Condition100%3/3
not defined $best or $watched < $best - $min_delta
l!l&&r!l&&!r
5190413
MC/DC100%2/2
not defined $best or $watched < $best - $min_delta
not defined $best $watched < $best - $min_delta
Truth table100%3/3
not(A) or B
A= not defined $bestB= $watched < $best - $min_delta
ABresult
1X1
011
000
403 151 ($best, $best_weights, $stale) = ($watched, $self->weights, 0);
404 }
405 2 elsif (++$stale >= $patience) { last }
406 }
407 5147 ▶ last if $arg{on_epoch} && !$arg{on_epoch}->($epoch, $mean);
Branch: 1 branch
BranchTrueFalseTotal
if $arg{"on_epoch"} and not $arg{"on_epoch"}->($epoch, $mean) 46 5101 5147
Condition100%3/3
$arg{"on_epoch"} and not $arg{"on_epoch"}->($epoch, $mean)
!ll&&!rl&&r
461848346
MC/DC100%2/2
$arg{"on_epoch"} and not $arg{"on_epoch"}->($epoch, $mean)
$arg{"on_epoch"} not $arg{"on_epoch"}->($epoch, $mean)
Truth table100%3/3
A and not(B)
A= $arg{"on_epoch"}B= not $arg{"on_epoch"}->($epoch, $mean)
ABresult
0X0
100
111
408 5101 ▶ last if defined $arg{target_loss} && $mean <= $arg{target_loss};
Branch: 1 branch
BranchTrueFalseTotal
if defined $arg{"target_loss"} and $mean <= $arg{"target_loss"} 0 5101 5101
Condition66%2/3
defined $arg{"target_loss"} and $mean <= $arg{"target_loss"}
!ll&&!rl&&r
350116000
MC/DC0%0/2
defined $arg{"target_loss"} and $mean <= $arg{"target_loss"}
defined $arg{"target_loss"} $mean <= $arg{"target_loss"}
Truth table66%2/3
A and B
A= defined $arg{"target_loss"}B= $mean <= $arg{"target_loss"}
ABresult
0X0
100
111
409 5101 ▶ $optimizer->{lr} *= $arg{lr_decay} if defined $arg{lr_decay};
Branch: 1 branch
BranchTrueFalseTotal
if defined $arg{"lr_decay"} 2096 3005 5101
410 }
411 147 ▶ $self->set_weights($best_weights) if $best_weights;
Branch: 1 branch
BranchTrueFalseTotal
if $best_weights 51 96 147
412 147 return $mean;
413 }
414  
415 # What every epoch of the last train() measured: { epoch, loss, validation }.
416 48 ▶sub history ($self) { return @{ $self->{history} // [] } }
Subroutine history: called CC=2 SCAR=7.2
Pod documented
Condition50%1/2
$self->{"history"} // []
l!l
480
MC/DC0%0/1
$self->{"history"} // []
$self->{"history"}
Truth table50%1/2
A
A= $self->{"history"}
Aresult
11
00
417  
418 # Share of [input, class index] pairs classified correctly.
419 4 ▶sub accuracy ($self, $pairs) {
Subroutine accuracy: called CC=1 SCAR=0.0
Pod documented
420 4 my $classes = $self->classify_all([ map { $_->[0] } @$pairs ]);
421 4 my $right = grep { $classes->[$_] == $pairs->[$_][1] } 0 .. $#$pairs;
422 4 return $right / @$pairs;
423 }
424  
425 # What it takes to rebuild this net: its definition and its weights. The
426 # backend is not part of it; a net trained on one loads on any other.
427 10 ▶sub state ($self) {
Subroutine state: called CC=1 SCAR=0.0
Pod documented
428 return {
429 version => $VERSION,
430 input => $self->{input},
431 loss => $self->{loss},
432 seed => $self->{seed},
433 10 layers => [ map { $_->spec } @{ $self->{layers} } ],
434 weights => $self->weights,
435 };
436 }
437  
438 12 ▶sub from_state ($class, $state, %arg) {
Subroutine from_state: called CC=1 SCAR=0.0
Pod documented
439 12 return $class->new(%$state{qw(input loss seed layers)}, %arg)->set_weights($state->{weights});
440 }
441  
442 # Storable's portable format writes a double as decimal text and loses its
443 # last bits. Packed little-endian it stays exact, and portable all the same.
444 9 ▶sub freeze_state ($state) {
Subroutine freeze_state: called CC=1 SCAR=0.0
Pod documented
445 9 return { %$state, weights => [ map { pack 'd<*', @$_ } @{ $state->{weights} } ] };
446 }
447  
448 11 ▶sub thaw_state ($frozen) {
Subroutine thaw_state: called CC=1 SCAR=0.0
Pod documented
449 11 return { %$frozen, weights => [ map { [ unpack 'd<*', $_ ] } @{ $frozen->{weights} } ] };
450 }
451  
452 # A network's state file, by convention *.net.
453 1 ▶sub save ($self, $file) {
Subroutine save: called CC=1 SCAR=0.0
Pod documented
454 1 Storable::nstore({ %{ freeze_state($self->state) }, format => $STATE_FORMAT }, $file);
455 1 return $self;
456 }
457  
458 # The file is read as plain data, and weights that do not fit the definition
459 # in it are refused.
460 2 ▶sub load ($class, $file, %arg) {
Subroutine load: called CC=4 SCAR=15.9
Pod documented
461 2 my $frozen = Peta::NN::Inference::read_file($file, $STATE_FORMAT);
462 die "$file is a malformed network state\n"
463 1 ▶ if ref $frozen->{layers} ne 'ARRAY' || ref $frozen->{weights} ne 'ARRAY' || grep { ref } @{ $frozen->{weights} };
Branch: 1 branch
BranchTrueFalseTotal
if ref $frozen->{"layers"} ne "ARRAY" or ref $frozen->{"weights"} ne "ARRAY" or grep {ref $_;} @{$$frozen{"weights"};} 0 1 1
Condition33%2/6
ref $frozen->{"layers"} ne "ARRAY" or ref $frozen->{"weights"} ne "ARRAY" or grep {ref $_;} @{$$frozen{"weights"};}
l!l&&r!l&&!r
001
ref $frozen->{"layers"} ne "ARRAY" or ref $frozen->{"weights"} ne "ARRAY"
l!l&&r!l&&!r
001
MC/DC0%0/3
ref $frozen->{"layers"} ne "ARRAY" or ref $frozen->{"weights"} ne "ARRAY" or grep {ref $_;} @{$$frozen{"weights"};}
ref $frozen->{"layers"} ne "ARRAY" ref $frozen->{"weights"} ne "ARRAY" grep {ref $_;} @{$$frozen{"weights"};}
Truth table25%1/4
A or B or C
A= ref $frozen->{"layers"} ne "ARRAY"B= ref $frozen->{"weights"} ne "ARRAY"C= grep {ref $_;} @{$$frozen{"weights"};}
ABCresult
1XX1
01X1
0011
0000
464 1 return $class->from_state(thaw_state($frozen), %arg);
465 }
466  
467 27 1;
468  
469 __END__
« lib/Peta/NN/Model.pm ↑ Summary lib/Peta/NN/Chain.pm »