lib/Peta/NN/Job.pm

filter: Statementstmt 98.5% 268 / 272 Branchbran 78.0% 103 / 132 Conditioncond 70.3% 83 / 118 MC/DCmcdc 49.2% 33 / 67 Subroutinesub 100% 29 / 29 Podpod 100% 9 / 9 totaltotal 83.7% 525 / 627 CC 125 SCAR 26.7
CC
125
Cov
87%
run
25.8
new
25.7
_stage
21.2
CRAP 14.4
« bin/peta-nn-info ↑ Summary blib/lib/Peta/NN/Data.pm »
1 package Peta::NN::Job;
2 # ABSTRACT: train a model until it meets given fidelity thresholds, at the smallest size that can
3  
4 # An optimisation job: say what a model must achieve and what it may cost,
5 # and get the smallest model that does.
6 #
7 # A job trains ONE model until it has got there. It does not run from a model
8 # that falls short to another one:
9 #
10 # train until every threshold is met, for as long as the model keeps
11 # getting closer to them. What is watched is the distance to the
12 # thresholds, not the loss: a model that still has three of its
13 # exceptions to learn is not done because its loss has gone flat.
14 # Training on is gentler than training: a lower learning rate,
15 # so that the model keeps what it has while it learns the rest.
16 # weigh a model that has stopped getting closer while a subset it must
17 # retain is still short is trained on with more weight on that
18 # subset
19 # widen a model that has stopped getting closer all the same has
20 # learned what its size allows. It is given wider hidden layers,
21 # which changes none of its answers (Peta::NN::Model's widen),
22 # and trains on with what it knows
23 #
24 # The job starts at the smallest width it is allowed and widens by `grow`, so
25 # the model that gets there is the smallest that does, to within that factor.
26 # A second seed then has to get there as well, from that width: one lucky
27 # initialisation is not an answer.
28 #
29 # What no training can reach is taken out of what must be retained, and
30 # reported: pairs the model reads exactly as it reads another pair with a
31 # different answer, because they differ only outside its window. Chasing
32 # those would widen the model for ever.
33 #
34 # The data is split three ways. Thresholds are checked on the validation part
35 # while training, with the subsets that must be retained on the training
36 # part; the test part is measured once, on the model returned.
37  
38 2 ▶use v5.36;
Subroutine BEGIN: called CC=1 SCAR=0.0
39 2 ▶use utf8;
Subroutine BEGIN: called CC=1 SCAR=0.0
40  
41 2 ▶use Time::HiRes qw(time);
Subroutine BEGIN: called CC=1 SCAR=0.0
42  
43 2 ▶use Peta::NN::Model;
Subroutine BEGIN: called CC=1 SCAR=0.0
44 2 ▶use Peta::NN::RNG;
Subroutine BEGIN: called CC=1 SCAR=0.0
45  
46 2 our $VERSION = '0.2610090';
47  
48 2 my %DEFAULT_SEARCH = (scale => [ 8, 256 ], depth => [1], grow => 2);
49 # epochs is the most one stage trains before the job looks at it again;
50 # patience is how many epochs a model may go without getting closer to its
51 # thresholds before it counts as stuck.
52 2 my %DEFAULT_TRAIN = (epochs => 60, batch => 32, lr => 0.01, lr_decay => 0.95, patience => 20);
53 2 my %DEFAULT_BUDGET = (runs => 40);
54 2 my @DEFAULT_SPLIT = (0.70, 0.15, 0.15);
55  
56 2 my $BOOST = 30; # the weight of the pairs of a subset that has to be retained and is short
57 2 my $FINE = 0.3; # a model that is trained on is trained at this share of the learning rate:
58 # at the full rate it first unlearns, and takes a dozen epochs to be where it was
59 2 my $TIMED_ITEMS = 256;
60 2 my $TIMED_REPEATS = 3;
61  
62 # The default network for a scale: an embedding that grows slowly with it,
63 # and `depth` hidden layers of `scale` units. A job takes the embedding of
64 # its starting width and keeps it; widening changes the hidden layers.
65 19 ▶sub default_shape ($scale, $depth) {
Subroutine default_shape: called CC=3 SCAR=11.0
Pod documented
66 19 my $embed = int($scale / 4);
67 19 ▶ $embed = 4 if $embed < 4;
Branch: 1 branch
BranchTrueFalseTotal
if $embed < 4 12 7 19
68 19 ▶ $embed = 16 if $embed > 16;
Branch: 1 branch
BranchTrueFalseTotal
if $embed > 16 0 19 19
69 19 return [ [ embed => $embed ], map { ([ dense => $scale ], 'relu') } 1 .. $depth ];
70 }
71  
72 15 ▶sub new ($class, %arg) {
Subroutine new: called CC=21 SCAR=32.5
Pod documented
73 my $self = bless {
74 model => $arg{model} // die("model => { kind => ..., window => ... } is required\n"),
75 pairs => $arg{pairs} // die("pairs => [[input, output], ...] is required\n"),
76 fidelity => $arg{fidelity} // die("fidelity => { all => 0.97, ... } is required\n"),
77 subsets => $arg{subsets} // {},
78 11513 ▶ group => $arg{group} // sub ($pair) { $pair->[0] },
Subroutine __ANON__: called CC=1 SCAR=0.0
79 always_train => $arg{always_train},
80 split => $arg{split} // [@DEFAULT_SPLIT],
81 shape => $arg{shape} // \&default_shape,
82 15 ▶ search => { %DEFAULT_SEARCH, %{ $arg{search} // {} } },
Condition50%1/2
$arg{"search"} // {}
l!l
150
MC/DC0%0/1
$arg{"search"} // {}
$arg{"search"}
Truth table50%1/2
A
A= $arg{"search"}
Aresult
11
00
83 15 ▶ train => { %DEFAULT_TRAIN, %{ $arg{train} // {} } },
Condition50%1/2
$arg{"train"} // {}
l!l
150
MC/DC0%0/1
$arg{"train"} // {}
$arg{"train"}
Truth table50%1/2
A
A= $arg{"train"}
Aresult
11
00
84 15 ▶ budget => { %DEFAULT_BUDGET, %{ $arg{budget} // {} } },
Condition88%8/9
$arg{"budget"} // {}
l!l
312
$arg{"seed"} // 1
l!l
312
($arg{"backend"} // $ENV{"PETA_NN_BACKEND"}) // "auto"
l!l
312
$arg{"backend"} // $ENV{"PETA_NN_BACKEND"}
l!l&&r!l&&!r
3012
MC/DC100%1/1
$arg{"budget"} // {}
$arg{"budget"}
MC/DC100%1/1
$arg{"seed"} // 1
$arg{"seed"}
MC/DC50%1/2
($arg{"backend"} // $ENV{"PETA_NN_BACKEND"}) // "auto"
$arg{"backend"} $ENV{"PETA_NN_BACKEND"}
Truth table100%2/2
A
A= $arg{"budget"}
Aresult
11
00
Truth table100%2/2
A
A= $arg{"seed"}
Aresult
11
00
Truth table66%2/3
A // B
A= $arg{"backend"}B= $ENV{"PETA_NN_BACKEND"}
ABresult
1X1
011
000
85 seed => $arg{seed} // 1,
86 15 ▶ backend => $arg{backend} // $ENV{PETA_NN_BACKEND} // 'auto',
Condition46%7/15
$arg{"model"} // die("model => { kind => ..., window => ... } is required\n")
l!l
150
$arg{"pairs"} // die("pairs => [[input, output], ...] is required\n")
l!l
150
$arg{"fidelity"} // die("fidelity => { all => 0.97, ... } is required\n")
l!l
150
$arg{"subsets"} // {}
l!l
150
$arg{"group"} // sub ($pair) { $pair->[0]; }
l!l&&r!l&&!r
0150
$arg{"split"} // [@DEFAULT_SPLIT]
l!l
015
$arg{"shape"} // \&Peta::NN::Job::default_shape
l!l
015
MC/DC0%0/1
$arg{"model"} // die("model => { kind => ..., window => ... } is required\n")
$arg{"model"}
MC/DC0%0/1
$arg{"pairs"} // die("pairs => [[input, output], ...] is required\n")
$arg{"pairs"}
MC/DC0%0/1
$arg{"fidelity"} // die("fidelity => { all => 0.97, ... } is required\n")
$arg{"fidelity"}
MC/DC0%0/1
$arg{"subsets"} // {}
$arg{"subsets"}
MC/DC0%0/2
$arg{"group"} // sub ($pair) { $pair->[0]; }
$arg{"group"} sub ($pair) { $pair->[0]; }
MC/DC0%0/1
$arg{"split"} // [@DEFAULT_SPLIT]
$arg{"split"}
MC/DC0%0/1
$arg{"shape"} // \&Peta::NN::Job::default_shape
$arg{"shape"}
Truth table50%1/2
A
A= $arg{"model"}
Aresult
11
00
Truth table50%1/2
A
A= $arg{"pairs"}
Aresult
11
00
Truth table50%1/2
A
A= $arg{"fidelity"}
Aresult
11
00
Truth table50%1/2
A
A= $arg{"subsets"}
Aresult
11
00
Truth table33%1/3
A // B
A= $arg{"group"}B= sub ($pair) { $pair->[0]; }
ABresult
1X1
011
000
Truth table50%1/2
A
A= $arg{"split"}
Aresult
11
00
Truth table50%1/2
A
A= $arg{"shape"}
Aresult
11
00
87 attempts => [],
88 }, $class;
89 15 for my $name (keys %{ $self->{fidelity} }) {
90 die "fidelity names '$name', which is neither 'all' nor one of the subsets\n"
91 36 ▶ if $name ne 'all' && !$self->{subsets}{$name};
Branch: 1 branch
BranchTrueFalseTotal
if $name ne "all" and not $self->{"subsets"}{$name} 1 35 36
Condition100%3/3
$name ne "all" and not $self->{"subsets"}{$name}
!ll&&!rl&&r
14211
MC/DC100%2/2
$name ne "all" and not $self->{"subsets"}{$name}
$name ne "all" not $self->{"subsets"}{$name}
Truth table100%3/3
A and not(B)
A= $name ne "all"B= not $self->{"subsets"}{$name}
ABresult
0X0
100
111
92 }
93 14 my ($min, $max) = @{ $self->{search}{scale} };
94 14 ▶ $self->{search}{start} //= $min;
Condition66%2/3
$self->{"search"}{"start"} //= $min
l!l&&r!l&&!r
4100
MC/DC0%0/2
$self->{"search"}{"start"} //= $min
$self->{"search"}{"start"} $min
Truth table66%2/3
A //= B
A= $self->{"search"}{"start"}B= $min
ABresult
1X1
011
000
95 die "the search starts at a width of $self->{search}{start}, outside its scale of $min to $max\n"
96 14 ▶ if $self->{search}{start} < $min || $self->{search}{start} > $max;
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"search"}{"start"} < $min or $self->{"search"}{"start"} > $max 1 13 14
Condition66%2/3
$self->{"search"}{"start"} < $min or $self->{"search"}{"start"} > $max
l!l&&r!l&&!r
0113
MC/DC50%1/2
$self->{"search"}{"start"} < $min or $self->{"search"}{"start"} > $max
$self->{"search"}{"start"} < $min $self->{"search"}{"start"} > $max
Truth table66%2/3
A or B
A= $self->{"search"}{"start"} < $minB= $self->{"search"}{"start"} > $max
ABresult
1X1
011
000
97 13 ▶ die "grow is a factor above 1, not '$self->{search}{grow}'\n" if $self->{search}{grow} <= 1;
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"search"}{"grow"} <= 1 1 12 13
98 12 $self->_drop_contradictions;
99 12 $self->_split;
100 12 return $self;
101 }
102  
103 # Two pairs that give one input (with the same parameters) different outputs cannot both be
104 # answered right, by any model. Such an input keeps its first pair; the
105 # others are set aside and listed by contradictions(), so that a threshold is
106 # not missed for a reason no training can remove.
107 12 ▶sub _drop_contradictions ($self) {
Subroutine _drop_contradictions: called CC=4 SCAR=13.9
108 12 my (%first, @kept, @dropped);
109 12 for my $pair (@{ $self->{pairs} }) {
110 11514 my $key = join "\x{1f}", @$pair[ 0, 2 .. $#$pair ];
111 11514 ▶ if (!exists $first{$key}) { $first{$key} = $pair->[1]; push @kept, $pair }
Branch: 2 branches
BranchTrueFalseTotal
if (not exists $first{$key}) { } 10948 566 11514
elsif ($first{$key} eq $pair->[1]) { } 565 1 566
112 565 elsif ($first{$key} eq $pair->[1]) { push @kept, $pair }
113 1 else { push @dropped, [ $pair, $first{$key} ] }
114 }
115 12 @$self{qw(pairs contradictions)} = (\@kept, \@dropped);
116 12 return;
117 }
118  
119 # [the pair, the output that was kept for its input] for every pair set aside.
120 9 ▶sub contradictions ($self) { return @{ $self->{contradictions} } }
Subroutine contradictions: called CC=1 SCAR=0.0
Pod documented
121  
122 # Whole groups go to one part: all forms of one word stay together, so the
123 # held-out parts hold words the model has never seen in any form.
124 12 ▶sub _split ($self) {
Subroutine _split: called CC=7 SCAR=19.5
125 12 my (%group, @order);
126 12 for my $pair (@{ $self->{pairs} }) {
127 11513 my $key = $self->{group}->($pair);
128 11513 ▶ push @order, $key if !$group{$key};
Branch: 1 branch
BranchTrueFalseTotal
unless $group{$key} 9820 1693 11513
129 11513 push @{ $group{$key} }, $pair;
130 }
131 12 my $forced = $self->{always_train};
132 12 ▶ my @fixed = grep { $forced && grep { $forced->($_) } @{ $group{$_} } } @order;
Condition100%3/3
$forced && grep({&$forced($_);} @{$group{$_};})
!ll&&!rl&&r
11288585107
MC/DC100%2/2
$forced && grep({&$forced($_);} @{$group{$_};})
$forced grep({&$forced($_);} @{$group{$_};})
Truth table100%3/3
A && B
A= $forcedB= grep({&$forced($_);} @{$group{$_};})
ABresult
0X0
100
111
133 12 ▶ my @free = grep { !($forced && grep { $forced->($_) } @{ $group{$_} }) } @order;
Condition100%3/3
$forced && grep({&$forced($_);} @{$group{$_};})
!ll&&!rl&&r
11288585107
MC/DC100%2/2
$forced && grep({&$forced($_);} @{$group{$_};})
$forced grep({&$forced($_);} @{$group{$_};})
Truth table100%3/3
A && B
A= $forcedB= grep({&$forced($_);} @{$group{$_};})
ABresult
0X0
100
111
134 12 Peta::NN::RNG->new($self->{seed})->shuffle(\@free);
135  
136 12 my ($train_share, $validation_share) = @{ $self->{split} };
137 12 my $n_train = int(@free * $train_share);
138 12 my $n_validation = int(@free * $validation_share);
139 12 my %part = (
140 train => [ @fixed, @free[ 0 .. $n_train - 1 ] ],
141 validation => [ @free[ $n_train .. $n_train + $n_validation - 1 ] ],
142 test => [ @free[ $n_train + $n_validation .. $#free ] ],
143 );
144 12 $self->{part}{$_} = [ map { @{ $group{$_} } } @{ $part{$_} } ] for keys %part;
145 12 ▶ die "the split leaves the validation part empty\n" if !@{ $self->{part}{validation} };
Branch: 1 branch
BranchTrueFalseTotal
unless @{$self->{"part"}{"validation"};} 0 12 12
146 12 return;
147 }
148  
149 7 ▶sub part ($self, $name) { return @{ $self->{part}{$name} } }
Subroutine part: called CC=1 SCAR=0.0
Pod documented
150  
151 # The pairs a threshold is measured on: 'all' is the whole held-out part; a
152 # subset is the pairs its `where` accepts, of that part or, with of =>
153 # 'train', of the training part (what was shown and must be retained).
154 1714 ▶sub _measured_on ($self, $name, $held_out) {
Subroutine _measured_on: called CC=6 SCAR=17.9
155 1714 ▶ return $self->{part}{$held_out} if $name eq 'all';
Branch: 1 branch
BranchTrueFalseTotal
if $name eq "all" 633 1081 1714
156 1081 my $subset = $self->{subsets}{$name};
157 1081 ▶ return [ grep { $subset->{where}->($_) } @{ $self->{part}{$held_out} } ] if ($subset->{of} // '') ne 'train';
Branch: 1 branch
BranchTrueFalseTotal
if ($subset->{"of"} // "") ne "train" 516 565 1081
Condition100%2/2
$subset->{"of"} // ""
l!l
565516
MC/DC100%1/1
$subset->{"of"} // ""
$subset->{"of"}
Truth table100%2/2
A
A= $subset->{"of"}
Aresult
11
00
158 # What must be retained, but for what the model cannot tell apart.
159 565 ▶ return $self->{retained}{$name} //= [ grep { $subset->{where}->($_) && !$self->{is_indistinct}{$_} } @{ $self->{part}{train} } ];
Condition100%5/5
$self->{"retained"}{$name} //= [grep({$subset->{"where"}->($_) and not $self->{"is_indistinct"}{$_};} @{$$self{"part"}{"train"};})]
l!l
5578
$subset->{'where'}->($_) && !$self->{'is_indistinct'}{$_}
!ll&&!rl&&r
50883177
MC/DC100%1/1
$self->{"retained"}{$name} //= [grep({$subset->{"where"}->($_) and not $self->{"is_indistinct"}{$_};} @{$$self{"part"}{"train"};})]
$self->{"retained"}{$name}
MC/DC100%2/2
$subset->{'where'}->($_) && !$self->{'is_indistinct'}{$_}
$subset->{'where'}->($_) !$self->{'is_indistinct'}{$_}
Truth table100%2/2
A
A= $self->{"retained"}{$name}
Aresult
11
00
Truth table100%3/3
A && not(B)
A= $subset->{'where'}->($_)B= !$self->{'is_indistinct'}{$_}
ABresult
0X0
100
111
160 }
161  
162 # The pairs of the training part that the model reads as it reads another
163 # pair with a different answer; see Peta::NN::Model's indistinct. They are
164 # found once, with the first model that has been fitted: what a model reads
165 # does not depend on its width.
166 19 ▶sub _find_indistinct ($self, $model) {
Subroutine _find_indistinct: called CC=2 SCAR=6.9
167 19 ▶ return if $self->{indistinct};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"indistinct"} 8 11 19
168 11 $self->{indistinct} = [ $model->indistinct($self->{part}{train}) ];
169 11 $self->{is_indistinct} = { map { $_->[0] => 1 } @{ $self->{indistinct} } };
170 11 delete $self->{retained};
171 11 return;
172 }
173  
174 # [pair, another answer, a pair that has it] for every training pair the
175 # model cannot tell from one with a different answer.
176 10 ▶sub indistinct ($self) { return @{ $self->{indistinct} // [] } }
Subroutine indistinct: called CC=2 SCAR=7.2
Pod documented
Condition50%1/2
$self->{"indistinct"} // []
l!l
100
MC/DC0%0/1
$self->{"indistinct"} // []
$self->{"indistinct"}
Truth table50%1/2
A
A= $self->{"indistinct"}
Aresult
11
00
177  
178 # { name => share of pairs answered exactly } for every threshold.
179 # With $quick the model is measured where it is trained: on the graphics card,
180 # if that is where, which is what watching every epoch can afford. What a
181 # stage records, and what decides, is measured by the inference leg.
182 633 ▶sub _fidelity ($self, $model, $held_out, $quick = 0) {
Subroutine _fidelity: called CC=4 SCAR=13.9
183 633 my %value;
184 633 for my $name (sort keys %{ $self->{fidelity} }) {
185 1714 my $pairs = $self->_measured_on($name, $held_out);
186 1714 ▶ $value{$name} = @$pairs ? $model->accuracy($pairs, where_trained => $quick) : 1;
Branch: 1 branch
BranchTrueFalseTotal
@$pairs ? : 1714 0 1714
187 }
188 633 return \%value;
189 }
190  
191 # Milliseconds per item on the inference leg, which is what a user of the
192 # model runs: the best of a few timed passes over a batch of held-out inputs.
193 46 ▶sub _ms_per_item ($self, $model) {
Subroutine _ms_per_item: called CC=5 SCAR=16.1
194 46 my @pairs = @{ $self->{part}{validation} };
195 # One pass has one set of parameters: those of the first pair.
196 46 my @values = @{ $pairs[0] }[ 2 .. $#{ $pairs[0] } ];
197 46 @pairs = grep { "@$_[ 2 .. $#$_ ]" eq "@values" } @pairs;
198 46 ▶ $#pairs = $TIMED_ITEMS - 1 if @pairs > $TIMED_ITEMS;
Branch: 1 branch
BranchTrueFalseTotal
if @pairs > $TIMED_ITEMS 0 46 46
199 46 my @inputs = map { $_->[0] } @pairs;
200 46 my $best;
201 46 for (1 .. $TIMED_REPEATS) {
202 138 my $start = time;
203 138 my @answers = @{ $model->inference->answers(\@inputs, @values) };
204 138 my $seconds = time - $start;
205 138 ▶ $best = $seconds if !defined $best || $seconds < $best;
Branch: 1 branch
BranchTrueFalseTotal
if not defined $best or $seconds < $best 89 49 138
Condition100%3/3
not defined $best or $seconds < $best
l!l&&r!l&&!r
464349
MC/DC100%2/2
not defined $best or $seconds < $best
not defined $best $seconds < $best
Truth table100%3/3
not(A) or B
A= not defined $bestB= $seconds < $best
ABresult
1X1
011
000
206 }
207 46 return 1000 * $best / @inputs;
208 }
209  
210 # How much a pair counts in training: the pairs of the subsets in %$boost
211 # count that many times.
212 46 ▶sub _weight ($self, $boost) {
Subroutine _weight: called CC=2 SCAR=6.9
213 46 ▶ return if !%$boost;
Branch: 1 branch
BranchTrueFalseTotal
unless %$boost 22 24 46
214 24 my $subsets = $self->{subsets};
215 15198 ▶ return sub (@pair) {
Subroutine __ANON__: called CC=3 SCAR=11.0
216 15198 my $weight = 1;
217 15198 for my $name (keys %$boost) {
218 15198 ▶ $weight *= $boost->{$name} if $subsets->{$name}{where}->(\@pair);
Branch: 1 branch
BranchTrueFalseTotal
if $subsets->{$name}{"where"}->(\@pair) 78 15120 15198
219 }
220 15198 return $weight;
221 24 };
222 }
223  
224 77 ▶sub _out_of_budget ($self) {
Subroutine _out_of_budget: called CC=4 SCAR=14.5
225 77 my $budget = $self->{budget};
226 77 ▶ return 'run limit reached' if @{ $self->{attempts} } >= $budget->{runs};
Branch: 1 branch
BranchTrueFalseTotal
if @{$self->{"attempts"};} >= $budget->{"runs"} 2 75 77
227 75 ▶ return 'time limit reached' if defined $budget->{seconds} && time - $self->{started} >= $budget->{seconds};
Branch: 1 branch
BranchTrueFalseTotal
if defined $budget->{"seconds"} and &Peta::NN::Job::time() - $self->{"started"} >= $budget->{"seconds"} 0 75 75
Condition33%1/3
defined $budget->{"seconds"} and &Peta::NN::Job::time() - $self->{"started"} >= $budget->{"seconds"}
!ll&&!rl&&r
7500
MC/DC0%0/2
defined $budget->{"seconds"} and &Peta::NN::Job::time() - $self->{"started"} >= $budget->{"seconds"}
defined $budget->{"seconds"} &Peta::NN::Job::time() - $self->{"started"} >= $budget->{"seconds"}
Truth table33%1/3
A and B
A= defined $budget->{"seconds"}B= &Peta::NN::Job::time() - $self->{"started"} >= $budget->{"seconds"}
ABresult
0X0
100
111
228 75 return;
229 }
230  
231 # How far a model is from the thresholds: the shortfalls added up, 0 when
232 # every one is met.
233 576 ▶sub _distance ($self, $fidelity) {
Subroutine _distance: called CC=3 SCAR=11.0
234 576 my $distance = 0;
235 576 for my $name (keys %$fidelity) {
236 1572 my $gap = $self->{fidelity}{$name} - $fidelity->{$name};
237 1572 ▶ $distance += $gap if $gap > 0;
Branch: 1 branch
BranchTrueFalseTotal
if $gap > 0 755 817 1572
238 }
239 576 return $distance;
240 }
241  
242 # One stage of training: the model is trained, from nothing if it is new and
243 # on from where it is if not, until every threshold is met, or it has gone
244 # `patience` epochs without getting closer to them, or the stage's epochs or
245 # the job's time are used up. The weights it is left with are those of the
246 # epoch that was closest, or the ones it came with if no epoch got closer
247 # than it already was. The record says what the stage came to; `ended` is
248 # 'met', 'stuck', 'epochs' or 'time'.
249 46 ▶sub _stage ($self, $model, $stage, $boost, %opt) {
Subroutine _stage: called CC=20 SCAR=30.5
250 46 my $start = time;
251 46 my $train = $self->{train};
252 46 my $patience = $train->{patience};
253 46 my $limit = $self->{budget}{seconds};
254 46 my ($closest, $stale, $ended) = (undef, 0, 'epochs');
255  
256 # A model that has been trained comes with a distance of its own to beat.
257 46 ▶ my $came_with = $model->net ? { distance => $self->_distance($self->_fidelity($model, 'validation')), weights => $model->net->weights } : undef;
Branch: 1 branch
BranchTrueFalseTotal
$model->net ? : 27 19 46
258 46 ▶ $closest = $came_with->{distance} if $came_with;
Branch: 1 branch
BranchTrueFalseTotal
if $came_with 27 19 46
259  
260 # What training watches, epoch by epoch: the distance to the thresholds.
261 522 ▶ my $watch = sub ($loss, $held) {
Subroutine __ANON__: called CC=3 SCAR=11.0
262 522 my $distance = $self->_distance($self->_fidelity($model, 'validation', 1));
263 522 ▶ if (!defined $closest || $distance < $closest) { ($closest, $stale) = ($distance, 0) }
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $closest or $distance < $closest) { } 102 420 522
Condition100%3/3
not defined $closest or $distance < $closest
l!l&&r!l&&!r
1983420
MC/DC100%2/2
not defined $closest or $distance < $closest
not defined $closest $distance < $closest
Truth table100%3/3
not(A) or B
A= not defined $closestB= $distance < $closest
ABresult
1X1
011
000
264 420 else { $stale++ }
265 522 return $distance;
266 46 };
267 522 ▶ my $go_on = sub ($epoch, $loss) {
Subroutine __ANON__: called CC=5 SCAR=17.2
268 522 ▶ if ($closest == 0) { $ended = 'met' }
Branch: 3 branches
BranchTrueFalseTotal
if ($closest == 0) { } 16 506 522
elsif ($stale >= $patience) { } 29 477 506
elsif (defined $limit and &Peta::NN::Job::time() - $self->{'started'} >= $limit) { } 0 954 954
269 29 elsif ($stale >= $patience) { $ended = 'stuck' }
270 0 ▶ elsif (defined $limit && time - $self->{started} >= $limit) { $ended = 'time' }
Condition33%1/3
defined $limit and &Peta::NN::Job::time() - $self->{"started"} >= $limit
!ll&&!rl&&r
47700
MC/DC0%0/2
defined $limit and &Peta::NN::Job::time() - $self->{"started"} >= $limit
defined $limit &Peta::NN::Job::time() - $self->{"started"} >= $limit
Truth table33%1/3
A and B
A= defined $limitB= &Peta::NN::Job::time() - $self->{"started"} >= $limit
ABresult
0X0
100
111
271 522 return $ended eq 'epochs';
272 46 };
273 46 my $weight = $self->_weight($boost);
274 my @how = (validate => $self->{part}{validation}, ($weight ? (weight => $weight) : ()),
275 %$train, ($came_with ? (lr => $train->{lr} * $FINE) : ()),
276 46 ▶ patience => $train->{epochs} + 1, watch => $watch, on_epoch => $go_on);
Branch: 2 branches
BranchTrueFalseTotal
$weight ? : 24 22 46
$came_with ? : 27 19 46
277 46 ▶ if ($model->net) { $model->tune($self->{part}{train}, @how) }
Branch: 1 branch
BranchTrueFalseTotal
if ($model->net) { } 27 19 46
278 else {
279 # A new model is fitted for one epoch first, so that what it cannot
280 # tell apart is known before anything is asked of it.
281 19 $model->fit($self->{part}{train}, @how, epochs => 1, watch => undef, on_epoch => undef, patience => undef);
282 19 $self->_find_indistinct($model);
283 19 $model->tune($self->{part}{train}, @how, epochs => $train->{epochs} - 1);
284 }
285 46 ▶ my $epochs = ($came_with ? 0 : 1) + (() = $model->net->history);
Branch: 1 branch
BranchTrueFalseTotal
$came_with ? : 27 19 46
286 46 ▶ if ($came_with && $self->_distance($self->_fidelity($model, 'validation')) > $came_with->{distance}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($came_with and $self->_distance($self->_fidelity($model, "validation")) > $came_with->{"distance"}) 0 46 46
Condition66%2/3
$came_with and $self->_distance($self->_fidelity($model, "validation")) > $came_with->{"distance"}
!ll&&!rl&&r
19270
MC/DC0%0/2
$came_with and $self->_distance($self->_fidelity($model, "validation")) > $came_with->{"distance"}
$came_with $self->_distance($self->_fidelity($model, "validation")) > $came_with->{"distance"}
Truth table66%2/3
A and B
A= $came_withB= $self->_distance($self->_fidelity($model, "validation")) > $came_with->{"distance"}
ABresult
0X0
100
111
287 0 $model->net->set_weights($came_with->{weights});
288 0 ▶ $ended = 'stuck' if $ended eq 'epochs';
Branch: 1 branch
BranchTrueFalseTotal
if $ended eq "epochs" 0 0 0
289 }
290  
291 46 my $fidelity = $self->_fidelity($model, 'validation');
292 46 ▶ my ($width) = map { $_->[1] } grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} };
Condition100%3/3
ref $_ && $_->[0] eq 'dense'
!ll&&!rl&&r
464646
MC/DC100%2/2
ref $_ && $_->[0] eq 'dense'
ref $_ $_->[0] eq 'dense'
Truth table100%3/3
A && B
A= ref $_B= $_->[0] eq 'dense'
ABresult
0X0
100
111
293 my $record = {
294 stage => $stage,
295 scale => $width,
296 138 ▶ depth => scalar(grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} }),
Condition100%3/3
ref $_ && $_->[0] eq 'dense'
!ll&&!rl&&r
464646
MC/DC100%2/2
ref $_ && $_->[0] eq 'dense'
ref $_ $_->[0] eq 'dense'
Truth table100%3/3
A && B
A= ref $_B= $_->[0] eq 'dense'
ABresult
0X0
100
111
297 seed => $model->{seed},
298 params => $model->net->n_params,
299 epochs => $epochs,
300 fidelity => $fidelity,
301 116 ▶ missed => [ grep { $fidelity->{$_} < $self->{fidelity}{$_} } sort keys %$fidelity ],
Branch: 1 branch
BranchTrueFalseTotal
$opt{'confirmation'} ? : 16 30 46
302 ended => $ended,
303 ms => $self->_ms_per_item($model),
304 seconds => time - $start,
305 46 confirmation => $opt{confirmation} ? 1 : 0,
306 backend => $model->net->backend,
307 };
308 46 my $budget = $self->{budget};
309 $record->{over} = join ', ',
310 (defined $budget->{params} && $record->{params} > $budget->{params} ? 'parameters' : ()),
311 46 ▶ (defined $budget->{ms} && $record->{ms} > $budget->{ms} ? 'milliseconds' : ());
Branch: 2 branches
BranchTrueFalseTotal
defined $budget->{'params'} && $record->{'params'} > $budget->{'params'} ? : 1 45 46
defined $budget->{'ms'} && $record->{'ms'} > $budget->{'ms'} ? : 0 46 46
Condition66%4/6
defined $budget->{"params"} and $record->{"params"} > $budget->{"params"}
!ll&&!rl&&r
3961
defined $budget->{"ms"} and $record->{"ms"} > $budget->{"ms"}
!ll&&!rl&&r
4600
MC/DC100%2/2
defined $budget->{"params"} and $record->{"params"} > $budget->{"params"}
defined $budget->{"params"} $record->{"params"} > $budget->{"params"}
MC/DC0%0/2
defined $budget->{"ms"} and $record->{"ms"} > $budget->{"ms"}
defined $budget->{"ms"} $record->{"ms"} > $budget->{"ms"}
Truth table100%3/3
A and B
A= defined $budget->{"params"}B= $record->{"params"} > $budget->{"params"}
ABresult
0X0
100
111
Truth table33%1/3
A and B
A= defined $budget->{"ms"}B= $record->{"ms"} > $budget->{"ms"}
ABresult
0X0
100
111
312 46 ▶ $record->{passed} = !@{ $record->{missed} } && !$record->{over} ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
!(@{$record->{'missed'};} || $record->{'over'}) ? : 16 30 46
Condition66%2/3
@{$record->{'missed'};} || $record->{'over'}
l!l&&r!l&&!r
30016
MC/DC50%1/2
@{$record->{'missed'};} || $record->{'over'}
@{$record->{'missed'};} $record->{'over'}
Truth table66%2/3
A || B
A= @{$record->{'missed'};}B= $record->{'over'}
ABresult
1X1
011
000
313 46 push @{ $self->{attempts} }, $record;
314 46 ▶ $self->{progress}->($record) if $self->{progress};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"progress"} 0 46 46
315 46 return $record;
316 }
317  
318 # Train one model until it has got there, giving it weight on what is short
319 # and, with $widen, more width when it is stuck. Returns the model and the
320 # record of its last stage.
321 19 ▶sub _grow ($self, $width, $depth, $seed, %opt) {
Subroutine _grow: called CC=14 SCAR=26.4
322 19 my ($min, $max) = @{ $self->{search}{scale} };
323 19 my $model = Peta::NN::Model->new(%{ $self->{model} }, layers => $self->{shape}->($width, $depth), seed => $seed,
324 19 backend => $self->{backend});
325 19 my (%boost, $record);
326 19 my $stage = 'train';
327 19 ▶ while (!$self->_out_of_budget) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->_out_of_budget) 46 1 47
328 46 $record = $self->_stage($model, $stage, \%boost, confirmation => $opt{confirmation});
329 46 ▶ last if $record->{passed} || $record->{over} || $record->{ended} eq 'time';
Branch: 1 branch
BranchTrueFalseTotal
if $record->{"passed"} or $record->{"over"} or $record->{"ended"} eq "time" 17 29 46
Condition83%5/6
$record->{"passed"} or $record->{"over"} or $record->{"ended"} eq "time"
l!l&&r!l&&!r
17029
$record->{"passed"} or $record->{"over"}
l!l&&r!l&&!r
16129
MC/DC66%2/3
$record->{"passed"} or $record->{"over"} or $record->{"ended"} eq "time"
$record->{"passed"} $record->{"over"} $record->{"ended"} eq "time"
Truth table75%3/4
A or B or C
A= $record->{"passed"}B= $record->{"over"}C= $record->{"ended"} eq "time"
ABCresult
1XX1
01X1
0011
0000
330 29 ▶ if ($record->{ended} eq 'epochs') { $stage = 'train on'; next } # still getting closer: it only needs more
Branch: 1 branch
BranchTrueFalseTotal
if ($record->{"ended"} eq "epochs") 1 28 29
331  
332 # Stuck. First more weight on what must be retained and is not, once
333 # per subset; then more room.
334 28 ▶ my @short = grep { $_ ne 'all' && ($self->{subsets}{$_}{of} // '') eq 'train' && !$boost{$_} } @{ $record->{missed} };
Condition100%8/8
$_ ne 'all' && ($self->{'subsets'}{$_}{'of'} // '') eq 'train' && !$boost{$_}
!ll&&!rl&&r
101410
$_ ne "all" and ($self->{"subsets"}{$_}{"of"} // "") eq "train"
!ll&&!rl&&r
4624
$self->{"subsets"}{$_}{"of"} // ""
l!l
246
MC/DC100%3/3
$_ ne 'all' && ($self->{'subsets'}{$_}{'of'} // '') eq 'train' && !$boost{$_}
$_ ne "all" $self->{"subsets"}{$_}{"of"} !$boost{$_}
Truth table100%4/4
A and B && not(C)
A= $_ ne "all"B= $self->{"subsets"}{$_}{"of"}C= !$boost{$_}
ABCresult
0XX0
10X0
1100
1111
335 28 ▶ if (@short) {
Branch: 1 branch
BranchTrueFalseTotal
if (@short) 10 18 28
336 10 $boost{$_} = $BOOST for @short;
337 10 $stage = "more weight on: @short";
338 10 next;
339 }
340 18 ▶ last if !$opt{widen} || $width >= $max;
Branch: 1 branch
BranchTrueFalseTotal
if not $opt{"widen"} or $width >= $max 1 17 18
Condition66%2/3
not $opt{"widen"} or $width >= $max
l!l&&r!l&&!r
0117
MC/DC50%1/2
not $opt{"widen"} or $width >= $max
not $opt{"widen"} $width >= $max
Truth table66%2/3
not(A) or B
A= not $opt{"widen"}B= $width >= $max
ABresult
1X1
011
000
341 17 my $wider = int($width * $self->{search}{grow} + 0.5);
342 17 ▶ $width = $wider > $max ? $max : $wider;
Branch: 1 branch
BranchTrueFalseTotal
$wider > $max ? : 0 17 17
343 17 $model->widen($width);
344 17 $stage = "widened to $width";
345 }
346 19 return ($model, $record);
347 }
348  
349 # Find the smallest model that meets the thresholds, and return it. If none
350 # does within the budget, the closest is returned; result() says which.
351 11 ▶sub run ($self, %arg) {
Subroutine run: called CC=18 SCAR=32.5
Pod documented
352 11 $self->{progress} = $arg{progress};
353 11 $self->{started} = time;
354  
355 # One model per depth, grown from the starting width until it gets there.
356 11 my ($chosen, $model);
357 11 for my $depth (@{ $self->{search}{depth} }) {
358 11 ▶ last if $self->_out_of_budget;
Branch: 1 branch
BranchTrueFalseTotal
if $self->_out_of_budget 0 11 11
359 11 my ($grown, $record) = $self->_grow($self->{search}{start}, $depth, $self->{seed}, widen => 1);
360 11 ▶ next if !$record;
Branch: 1 branch
BranchTrueFalseTotal
unless $record 0 11 11
361 my $better = !$chosen
362 || $record->{passed} > $chosen->{passed}
363 || $record->{passed} == $chosen->{passed}
364 && ($record->{passed} ? $record->{params} < $chosen->{params}
365 11 ▶ : $self->_distance($record->{fidelity}) < $self->_distance($chosen->{fidelity}));
Branch: 1 branch
BranchTrueFalseTotal
$record->{'passed'} ? : 0 0 0
Condition22%2/9
!$chosen || $record->{'passed'} > $chosen->{'passed'} || $record->{'passed'} == $chosen->{'passed'} && ($record->{'passed'} ? $record->{'params'} < $chosen->{'params'} : $self->_distance($record->{'fidelity'}) < $self->_distance($chosen->{'fidelity'}))
l!l&&r!l&&!r
1100
not $chosen or $record->{"passed"} > $chosen->{"passed"}
l!l&&r!l&&!r
1100
$record->{"passed"} == $chosen->{"passed"} and $record->{"passed"} ? $record->{"params"} < $chosen->{"params"} : $self->_distance($record->{"fidelity"}) < $self->_distance($chosen->{"fidelity"})
!ll&&!rl&&r
000
MC/DC0%0/4
!$chosen || $record->{'passed'} > $chosen->{'passed'} || $record->{'passed'} == $chosen->{'passed'} && ($record->{'passed'} ? $record->{'params'} < $chosen->{'params'} : $self->_distance($record->{'fidelity'}) < $self->_distance($chosen->{'fidelity'}))
not $chosen $record->{"passed"} > $chosen->{"passed"} $record->{"passed"} == $chosen->{"passed"} $record->{"passed"} ? $record->{"params"} < $chosen->{"params"} : $self->_distance($record->{"fidelity"}) < $self->_distance($chosen->{"fidelity"})
Truth table20%1/5
not(A) or B || C and D
A= not $chosenB= $record->{"passed"} > $chosen->{"passed"}C= $record->{"passed"} == $chosen->{"passed"}D= $record->{"passed"} ? $record->{"params"} < $chosen->{"params"} : $self->_distance($record->{"fidelity"}) < $self->_distance($chosen->{"fidelity"})
ABCDresult
1XXX1
01XX1
00111
000X0
00100
366 11 ▶ ($chosen, $model) = ($record, $grown) if $better;
Branch: 1 branch
BranchTrueFalseTotal
if $better 11 0 11
367 11 ▶ last if $record->{passed};
Branch: 1 branch
BranchTrueFalseTotal
if $record->{"passed"} 8 3 11
368 }
369 11 ▶ die "the budget allows no training at all\n" if !$chosen;
Branch: 1 branch
BranchTrueFalseTotal
unless $chosen 0 11 11
370  
371 # Confirm it: a second seed, from nothing and from the size the first one
372 # reached, has to get there by the same rules. The model returned is the
373 # first seed's.
374 11 my $confirmed = 0;
375 11 ▶ if ($chosen->{passed} && !$self->_out_of_budget) {
Branch: 1 branch
BranchTrueFalseTotal
if ($chosen->{"passed"} and not $self->_out_of_budget) 8 3 11
Condition66%2/3
$chosen->{"passed"} and not $self->_out_of_budget
!ll&&!rl&&r
308
MC/DC50%1/2
$chosen->{"passed"} and not $self->_out_of_budget
$chosen->{"passed"} not $self->_out_of_budget
Truth table66%2/3
A and not(B)
A= $chosen->{"passed"}B= not $self->_out_of_budget
ABresult
0X0
100
111
376 8 my (undef, $again) = $self->_grow(@$chosen{qw(scale depth)}, $self->{seed} + 1, widen => 1, confirmation => 1);
377 8 ▶ $confirmed = 1 if $again && $again->{passed};
Branch: 1 branch
BranchTrueFalseTotal
if $again and $again->{"passed"} 8 0 8
Condition33%1/3
$again and $again->{"passed"}
!ll&&!rl&&r
008
MC/DC0%0/2
$again and $again->{"passed"}
$again $again->{"passed"}
Truth table33%1/3
A and B
A= $againB= $again->{"passed"}
ABresult
0X0
100
111
378 }
379  
380 $self->{result} = {
381 met => $chosen->{passed} ? 1 : 0,
382 confirmed => $confirmed,
383 %$chosen{qw(scale depth params ms fidelity missed over stage)},
384 test => $self->_fidelity($model, 'test'),
385 11 runs => scalar @{ $self->{attempts} },
386 11 ▶ epochs => eval { my $n = 0; $n += $_->{epochs} for grep { !$_->{confirmation} } @{ $self->{attempts} }; $n },
Condition100%2/2
$self->_out_of_budget // "done"
l!l
110
MC/DC100%1/1
$self->_out_of_budget // "done"
$self->_out_of_budget
Truth table100%2/2
A
A= $self->_out_of_budget
Aresult
11
00
387 seconds => time - $self->{started},
388 11 ▶ stopped => $self->_out_of_budget // 'done',
Branch: 1 branch
BranchTrueFalseTotal
$chosen->{'passed'} ? : 8 3 11
389 };
390 11 return $model;
391 }
392  
393 6 ▶sub attempts ($self) { return @{ $self->{attempts} } }
Subroutine attempts: called CC=1 SCAR=0.0
Pod documented
394 13 ▶sub result ($self) { return $self->{result} }
Subroutine result: called CC=1 SCAR=0.0
Pod documented
395  
396 # Every stage in the order it was run, and the outcome.
397 8 ▶sub report ($self) {
Subroutine report: called CC=17 SCAR=30.0
Pod documented
398 8 my @names = sort keys %{ $self->{fidelity} };
399 8 my @clash = $self->contradictions;
400 my $text = !@clash ? '' : sprintf "%d pairs set aside: their input already had another answer (%s%s)\n", scalar @clash,
401 8 ▶ join(', ', map { "$_->[0][0] → $_->[0][1], kept $_->[1]" } @clash[ 0 .. ($#clash < 2 ? $#clash : 2) ]), @clash > 3 ? ', ...' : '';
Branch: 3 branches
BranchTrueFalseTotal
!@clash ? : 8 0 8
$#clash < 2 ? : 0 0 0
@clash > 3 ? : 0 0 0
402 8 my @same = $self->indistinct;
403 $text .= sprintf "%d training pairs the model cannot tell from a pair with a different answer; a wider window would (%s%s)\n", scalar @same,
404 8 ▶ join(', ', map { "$_->[0][0] → $_->[0][1] as $_->[2][0] → $_->[2][1]" } @same[ 0 .. ($#same < 2 ? $#same : 2) ]), @same > 3 ? ', ...' : '' if @same;
Branch: 3 branches
BranchTrueFalseTotal
if @same 2 6 8
$#same < 2 ? : 0 2 2
@same > 3 ? : 2 0 2
405 $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s %s\n", '#', 'stage', 'width', 'params', 'epochs', 'on', 'ms',
406 8 join(' ', map { sprintf '%-12s', substr($_, 0, 12) } @names), 'outcome';
407 $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s\n", '', 'required', '', '', '', '', $self->{budget}{ms} // '',
408 8 ▶ join(' ', map { sprintf '%-12s', sprintf '>= %.1f%%', 100 * $self->{fidelity}{$_} } @names);
Condition50%1/2
$self->{"budget"}{"ms"} // ""
l!l
08
MC/DC0%0/1
$self->{"budget"}{"ms"} // ""
$self->{"budget"}{"ms"}
Truth table50%1/2
A
A= $self->{"budget"}{"ms"}
Aresult
11
00
409 8 my %ENDED = (stuck => 'no closer', epochs => 'still getting closer', time => 'out of time');
410 8 my $n = 0;
411 8 for my $a (@{ $self->{attempts} }) {
412 my $outcome = $a->{passed} ? ($a->{confirmation} ? 'confirmed' : 'met')
413 36 ▶ : $a->{over} ? "over budget: $a->{over}"
Branch: 3 branches
BranchTrueFalseTotal
$a->{'passed'} ? : 14 22 36
$a->{'confirmation'} ? : 7 7 14
$a->{'over'} ? : 0 22 22
414 22 : "$ENDED{ $a->{ended} }; short: @{ $a->{missed} }";
415 $text .= sprintf "%-3d %-24s %5d %7d %6d %-5s %7.3f %s %s\n", ++$n,
416 substr(($a->{confirmation} ? 'second seed: ' : '') . $a->{stage}, 0, 24), @$a{qw(scale params epochs backend ms)},
417 36 ▶ join(' ', map { sprintf '%-12s', sprintf '%.2f%%', 100 * $a->{fidelity}{$_} } @names), $outcome;
Branch: 1 branch
BranchTrueFalseTotal
$a->{'confirmation'} ? : 15 21 36
418 }
419 8 ▶ my $r = $self->{result} or return $text;
Branch: 1 branch
BranchTrueFalseTotal
my $r = $self->{"result"} or return $text 8 0 8
420 $text .= sprintf "\n%s: depth %d, width %d, %d parameters, %.3f ms per item. %s%d stages, %d epochs, %.0f s (%s).\n",
421 ($r->{met} ? 'the model meets the thresholds' : 'the model does NOT meet the thresholds; this is the closest it got'),
422 @$r{qw(depth scale params ms)},
423 ($r->{met} ? ($r->{confirmed} ? 'Confirmed on a second seed. ' : 'NOT confirmed on a second seed. ') : ''),
424 8 ▶ @$r{qw(runs epochs seconds stopped)};
Branch: 3 branches
BranchTrueFalseTotal
$r->{'met'} ? : 7 1 8
$r->{'met'} ? : 7 1 8
$r->{'confirmed'} ? : 7 0 7
425 $text .= sprintf "on the test part, measured once: %s\n",
426 8 join ', ', map { sprintf '%s %.1f%%', $_, 100 * $r->{test}{$_} } @names;
427 8 return $text;
428 }
429  
430 2 1;
431  
432 __END__
« bin/peta-nn-info ↑ Summary blib/lib/Peta/NN/Data.pm »