| 1 | package Peta::NN::Job; | ||
| 2 | # ABSTRACT: train a model until it meets given fidelity thresholds, at the smallest size that can | ||
| 3 | |||
| 4 | # An optimisation job: say what a model must achieve and what it may cost, | ||
| 5 | # and get the smallest model that does. | ||
| 6 | # | ||
| 7 | # A job trains ONE model until it has got there. It does not run from a model | ||
| 8 | # that falls short to another one: | ||
| 9 | # | ||
| 10 | # train until every threshold is met, for as long as the model keeps | ||
| 11 | # getting closer to them. What is watched is the distance to the | ||
| 12 | # thresholds, not the loss: a model that still has three of its | ||
| 13 | # exceptions to learn is not done because its loss has gone flat. | ||
| 14 | # Training on is gentler than training: a lower learning rate, | ||
| 15 | # so that the model keeps what it has while it learns the rest. | ||
| 16 | # weigh a model that has stopped getting closer while a subset it must | ||
| 17 | # retain is still short is trained on with more weight on that | ||
| 18 | # subset | ||
| 19 | # widen a model that has stopped getting closer all the same has | ||
| 20 | # learned what its size allows. It is given wider hidden layers, | ||
| 21 | # which changes none of its answers (Peta::NN::Model's widen), | ||
| 22 | # and trains on with what it knows | ||
| 23 | # | ||
| 24 | # The job starts at the smallest width it is allowed and widens by `grow`, so | ||
| 25 | # the model that gets there is the smallest that does, to within that factor. | ||
| 26 | # A second seed then has to get there as well, from that width: one lucky | ||
| 27 | # initialisation is not an answer. | ||
| 28 | # | ||
| 29 | # What no training can reach is taken out of what must be retained, and | ||
| 30 | # reported: pairs the model reads exactly as it reads another pair with a | ||
| 31 | # different answer, because they differ only outside its window. Chasing | ||
| 32 | # those would widen the model for ever. | ||
| 33 | # | ||
| 34 | # The data is split three ways. Thresholds are checked on the validation part | ||
| 35 | # while training, with the subsets that must be retained on the training | ||
| 36 | # part; the test part is measured once, on the model returned. | ||
| 37 | |||
| 38 | use v5.36; | ||
| 39 | use utf8; | ||
| 40 | |||
| 41 | use Time::HiRes qw(time); | ||
| 42 | |||
| 43 | use Peta::NN::Model; | ||
| 44 | use Peta::NN::RNG; | ||
| 45 | |||
| 46 | our $VERSION = '0.2610090'; | ||
| 47 | |||
| 48 | my %DEFAULT_SEARCH = (scale => [ 8, 256 ], depth => [1], grow => 2); | ||
| 49 | # epochs is the most one stage trains before the job looks at it again; | ||
| 50 | # patience is how many epochs a model may go without getting closer to its | ||
| 51 | # thresholds before it counts as stuck. | ||
| 52 | my %DEFAULT_TRAIN = (epochs => 60, batch => 32, lr => 0.01, lr_decay => 0.95, patience => 20); | ||
| 53 | my %DEFAULT_BUDGET = (runs => 40); | ||
| 54 | my @DEFAULT_SPLIT = (0.70, 0.15, 0.15); | ||
| 55 | |||
| 56 | my $BOOST = 30; # the weight of the pairs of a subset that has to be retained and is short | ||
| 57 | my $FINE = 0.3; # a model that is trained on is trained at this share of the learning rate: | ||
| 58 | # at the full rate it first unlearns, and takes a dozen epochs to be where it was | ||
| 59 | my $TIMED_ITEMS = 256; | ||
| 60 | my $TIMED_REPEATS = 3; | ||
| 61 | |||
| 62 | # The default network for a scale: an embedding that grows slowly with it, | ||
| 63 | # and `depth` hidden layers of `scale` units. A job takes the embedding of | ||
| 64 | # its starting width and keeps it; widening changes the hidden layers. | ||
| 65 | sub default_shape ($scale, $depth) { | ||
| 66 | my $embed = int($scale / 4); | ||
| 67 | $embed = 4 if $embed < 4; | ||
| 68 | $embed = 16 if $embed > 16; | ||
| 69 | return [ [ embed => $embed ], map { ([ dense => $scale ], 'relu') } 1 .. $depth ]; | ||
| 70 | } | ||
| 71 | |||
| 72 | sub new ($class, %arg) { | ||
| 73 | my $self = bless { | ||
| 74 | model => $arg{model} // die("model => { kind => ..., window => ... } is required\n"), | ||
| 75 | pairs => $arg{pairs} // die("pairs => [[input, output], ...] is required\n"), | ||
| 76 | fidelity => $arg{fidelity} // die("fidelity => { all => 0.97, ... } is required\n"), | ||
| 77 | subsets => $arg{subsets} // {}, | ||
| 78 | group => $arg{group} // sub ($pair) { $pair->[0] }, | ||
| 79 | always_train => $arg{always_train}, | ||
| 80 | split => $arg{split} // [@DEFAULT_SPLIT], | ||
| 81 | shape => $arg{shape} // \&default_shape, | ||
| 82 | search => { %DEFAULT_SEARCH, %{ $arg{search} // {} } }, | ||
| 83 | train => { %DEFAULT_TRAIN, %{ $arg{train} // {} } }, | ||
| 84 | budget => { %DEFAULT_BUDGET, %{ $arg{budget} // {} } }, | ||
| 85 | seed => $arg{seed} // 1, | ||
| 86 | backend => $arg{backend} // $ENV{PETA_NN_BACKEND} // 'auto', | ||
| 87 | attempts => [], | ||
| 88 | }, $class; | ||
| 89 | for my $name (keys %{ $self->{fidelity} }) { | ||
| 90 | die "fidelity names '$name', which is neither 'all' nor one of the subsets\n" | ||
| 91 | if $name ne 'all' && !$self->{subsets}{$name}; | ||
| 92 | } | ||
| 93 | my ($min, $max) = @{ $self->{search}{scale} }; | ||
| 94 | $self->{search}{start} //= $min; | ||
| 95 | die "the search starts at a width of $self->{search}{start}, outside its scale of $min to $max\n" | ||
| 96 | if $self->{search}{start} < $min || $self->{search}{start} > $max; | ||
| 97 | die "grow is a factor above 1, not '$self->{search}{grow}'\n" if $self->{search}{grow} <= 1; | ||
| 98 | $self->_drop_contradictions; | ||
| 99 | $self->_split; | ||
| 100 | return $self; | ||
| 101 | } | ||
| 102 | |||
| 103 | # Two pairs that give one input (with the same parameters) different outputs cannot both be | ||
| 104 | # answered right, by any model. Such an input keeps its first pair; the | ||
| 105 | # others are set aside and listed by contradictions(), so that a threshold is | ||
| 106 | # not missed for a reason no training can remove. | ||
| 107 | sub _drop_contradictions ($self) { | ||
| 108 | my (%first, @kept, @dropped); | ||
| 109 | for my $pair (@{ $self->{pairs} }) { | ||
| 110 | my $key = join "\x{1f}", @$pair[ 0, 2 .. $#$pair ]; | ||
| 111 | if (!exists $first{$key}) { $first{$key} = $pair->[1]; push @kept, $pair } | ||
| 112 | elsif ($first{$key} eq $pair->[1]) { push @kept, $pair } | ||
| 113 | else { push @dropped, [ $pair, $first{$key} ] } | ||
| 114 | } | ||
| 115 | @$self{qw(pairs contradictions)} = (\@kept, \@dropped); | ||
| 116 | return; | ||
| 117 | } | ||
| 118 | |||
| 119 | # [the pair, the output that was kept for its input] for every pair set aside. | ||
| 120 | sub contradictions ($self) { return @{ $self->{contradictions} } } | ||
| 121 | |||
| 122 | # Whole groups go to one part: all forms of one word stay together, so the | ||
| 123 | # held-out parts hold words the model has never seen in any form. | ||
| 124 | sub _split ($self) { | ||
| 125 | my (%group, @order); | ||
| 126 | for my $pair (@{ $self->{pairs} }) { | ||
| 127 | my $key = $self->{group}->($pair); | ||
| 128 | push @order, $key if !$group{$key}; | ||
| 129 | push @{ $group{$key} }, $pair; | ||
| 130 | } | ||
| 131 | my $forced = $self->{always_train}; | ||
| 132 | my @fixed = grep { $forced && grep { $forced->($_) } @{ $group{$_} } } @order; | ||
| 133 | my @free = grep { !($forced && grep { $forced->($_) } @{ $group{$_} }) } @order; | ||
| 134 | Peta::NN::RNG->new($self->{seed})->shuffle(\@free); | ||
| 135 | |||
| 136 | my ($train_share, $validation_share) = @{ $self->{split} }; | ||
| 137 | my $n_train = int(@free * $train_share); | ||
| 138 | my $n_validation = int(@free * $validation_share); | ||
| 139 | my %part = ( | ||
| 140 | train => [ @fixed, @free[ 0 .. $n_train - 1 ] ], | ||
| 141 | validation => [ @free[ $n_train .. $n_train + $n_validation - 1 ] ], | ||
| 142 | test => [ @free[ $n_train + $n_validation .. $#free ] ], | ||
| 143 | ); | ||
| 144 | $self->{part}{$_} = [ map { @{ $group{$_} } } @{ $part{$_} } ] for keys %part; | ||
| 145 | die "the split leaves the validation part empty\n" if !@{ $self->{part}{validation} }; | ||
| 146 | return; | ||
| 147 | } | ||
| 148 | |||
| 149 | sub part ($self, $name) { return @{ $self->{part}{$name} } } | ||
| 150 | |||
| 151 | # The pairs a threshold is measured on: 'all' is the whole held-out part; a | ||
| 152 | # subset is the pairs its `where` accepts, of that part or, with of => | ||
| 153 | # 'train', of the training part (what was shown and must be retained). | ||
| 154 | sub _measured_on ($self, $name, $held_out) { | ||
| 155 | return $self->{part}{$held_out} if $name eq 'all'; | ||
| 156 | my $subset = $self->{subsets}{$name}; | ||
| 157 | return [ grep { $subset->{where}->($_) } @{ $self->{part}{$held_out} } ] if ($subset->{of} // '') ne 'train'; | ||
| 158 | # What must be retained, but for what the model cannot tell apart. | ||
| 159 | return $self->{retained}{$name} //= [ grep { $subset->{where}->($_) && !$self->{is_indistinct}{$_} } @{ $self->{part}{train} } ]; | ||
| 160 | } | ||
| 161 | |||
| 162 | # The pairs of the training part that the model reads as it reads another | ||
| 163 | # pair with a different answer; see Peta::NN::Model's indistinct. They are | ||
| 164 | # found once, with the first model that has been fitted: what a model reads | ||
| 165 | # does not depend on its width. | ||
| 166 | sub _find_indistinct ($self, $model) { | ||
| 167 | return if $self->{indistinct}; | ||
| 168 | $self->{indistinct} = [ $model->indistinct($self->{part}{train}) ]; | ||
| 169 | $self->{is_indistinct} = { map { $_->[0] => 1 } @{ $self->{indistinct} } }; | ||
| 170 | delete $self->{retained}; | ||
| 171 | return; | ||
| 172 | } | ||
| 173 | |||
| 174 | # [pair, another answer, a pair that has it] for every training pair the | ||
| 175 | # model cannot tell from one with a different answer. | ||
| 176 | sub indistinct ($self) { return @{ $self->{indistinct} // [] } } | ||
| 177 | |||
| 178 | # { name => share of pairs answered exactly } for every threshold. | ||
| 179 | # With $quick the model is measured where it is trained: on the graphics card, | ||
| 180 | # if that is where, which is what watching every epoch can afford. What a | ||
| 181 | # stage records, and what decides, is measured by the inference leg. | ||
| 182 | sub _fidelity ($self, $model, $held_out, $quick = 0) { | ||
| 183 | my %value; | ||
| 184 | for my $name (sort keys %{ $self->{fidelity} }) { | ||
| 185 | my $pairs = $self->_measured_on($name, $held_out); | ||
| 186 | $value{$name} = @$pairs ? $model->accuracy($pairs, where_trained => $quick) : 1; | ||
| 187 | } | ||
| 188 | return \%value; | ||
| 189 | } | ||
| 190 | |||
| 191 | # Milliseconds per item on the inference leg, which is what a user of the | ||
| 192 | # model runs: the best of a few timed passes over a batch of held-out inputs. | ||
| 193 | sub _ms_per_item ($self, $model) { | ||
| 194 | my @pairs = @{ $self->{part}{validation} }; | ||
| 195 | # One pass has one set of parameters: those of the first pair. | ||
| 196 | my @values = @{ $pairs[0] }[ 2 .. $#{ $pairs[0] } ]; | ||
| 197 | @pairs = grep { "@$_[ 2 .. $#$_ ]" eq "@values" } @pairs; | ||
| 198 | $#pairs = $TIMED_ITEMS - 1 if @pairs > $TIMED_ITEMS; | ||
| 199 | my @inputs = map { $_->[0] } @pairs; | ||
| 200 | my $best; | ||
| 201 | for (1 .. $TIMED_REPEATS) { | ||
| 202 | my $start = time; | ||
| 203 | my @answers = @{ $model->inference->answers(\@inputs, @values) }; | ||
| 204 | my $seconds = time - $start; | ||
| 205 | $best = $seconds if !defined $best || $seconds < $best; | ||
| 206 | } | ||
| 207 | return 1000 * $best / @inputs; | ||
| 208 | } | ||
| 209 | |||
| 210 | # How much a pair counts in training: the pairs of the subsets in %$boost | ||
| 211 | # count that many times. | ||
| 212 | sub _weight ($self, $boost) { | ||
| 213 | return if !%$boost; | ||
| 214 | my $subsets = $self->{subsets}; | ||
| 215 | return sub (@pair) { | ||
| 216 | my $weight = 1; | ||
| 217 | for my $name (keys %$boost) { | ||
| 218 | $weight *= $boost->{$name} if $subsets->{$name}{where}->(\@pair); | ||
| 219 | } | ||
| 220 | return $weight; | ||
| 221 | }; | ||
| 222 | } | ||
| 223 | |||
| 224 | sub _out_of_budget ($self) { | ||
| 225 | my $budget = $self->{budget}; | ||
| 226 | return 'run limit reached' if @{ $self->{attempts} } >= $budget->{runs}; | ||
| 227 | return 'time limit reached' if defined $budget->{seconds} && time - $self->{started} >= $budget->{seconds}; | ||
| 228 | return; | ||
| 229 | } | ||
| 230 | |||
| 231 | # How far a model is from the thresholds: the shortfalls added up, 0 when | ||
| 232 | # every one is met. | ||
| 233 | sub _distance ($self, $fidelity) { | ||
| 234 | my $distance = 0; | ||
| 235 | for my $name (keys %$fidelity) { | ||
| 236 | my $gap = $self->{fidelity}{$name} - $fidelity->{$name}; | ||
| 237 | $distance += $gap if $gap > 0; | ||
| 238 | } | ||
| 239 | return $distance; | ||
| 240 | } | ||
| 241 | |||
| 242 | # One stage of training: the model is trained, from nothing if it is new and | ||
| 243 | # on from where it is if not, until every threshold is met, or it has gone | ||
| 244 | # `patience` epochs without getting closer to them, or the stage's epochs or | ||
| 245 | # the job's time are used up. The weights it is left with are those of the | ||
| 246 | # epoch that was closest, or the ones it came with if no epoch got closer | ||
| 247 | # than it already was. The record says what the stage came to; `ended` is | ||
| 248 | # 'met', 'stuck', 'epochs' or 'time'. | ||
| 249 | sub _stage ($self, $model, $stage, $boost, %opt) { | ||
| 250 | my $start = time; | ||
| 251 | my $train = $self->{train}; | ||
| 252 | my $patience = $train->{patience}; | ||
| 253 | my $limit = $self->{budget}{seconds}; | ||
| 254 | my ($closest, $stale, $ended) = (undef, 0, 'epochs'); | ||
| 255 | |||
| 256 | # A model that has been trained comes with a distance of its own to beat. | ||
| 257 | my $came_with = $model->net ? { distance => $self->_distance($self->_fidelity($model, 'validation')), weights => $model->net->weights } : undef; | ||
| 258 | $closest = $came_with->{distance} if $came_with; | ||
| 259 | |||
| 260 | # What training watches, epoch by epoch: the distance to the thresholds. | ||
| 261 | my $watch = sub ($loss, $held) { | ||
| 262 | my $distance = $self->_distance($self->_fidelity($model, 'validation', 1)); | ||
| 263 | if (!defined $closest || $distance < $closest) { ($closest, $stale) = ($distance, 0) } | ||
| 264 | else { $stale++ } | ||
| 265 | return $distance; | ||
| 266 | }; | ||
| 267 | my $go_on = sub ($epoch, $loss) { | ||
| 268 | if ($closest == 0) { $ended = 'met' } | ||
| 269 | elsif ($stale >= $patience) { $ended = 'stuck' } | ||
| 270 | elsif (defined $limit && time - $self->{started} >= $limit) { $ended = 'time' } | ||
| 271 | return $ended eq 'epochs'; | ||
| 272 | }; | ||
| 273 | my $weight = $self->_weight($boost); | ||
| 274 | my @how = (validate => $self->{part}{validation}, ($weight ? (weight => $weight) : ()), | ||
| 275 | %$train, ($came_with ? (lr => $train->{lr} * $FINE) : ()), | ||
| 276 | patience => $train->{epochs} + 1, watch => $watch, on_epoch => $go_on); | ||
| 277 | if ($model->net) { $model->tune($self->{part}{train}, @how) } | ||
| 278 | else { | ||
| 279 | # A new model is fitted for one epoch first, so that what it cannot | ||
| 280 | # tell apart is known before anything is asked of it. | ||
| 281 | $model->fit($self->{part}{train}, @how, epochs => 1, watch => undef, on_epoch => undef, patience => undef); | ||
| 282 | $self->_find_indistinct($model); | ||
| 283 | $model->tune($self->{part}{train}, @how, epochs => $train->{epochs} - 1); | ||
| 284 | } | ||
| 285 | my $epochs = ($came_with ? 0 : 1) + (() = $model->net->history); | ||
| 286 | if ($came_with && $self->_distance($self->_fidelity($model, 'validation')) > $came_with->{distance}) { | ||
| 287 | $model->net->set_weights($came_with->{weights}); | ||
| 288 | $ended = 'stuck' if $ended eq 'epochs'; | ||
| 289 | } | ||
| 290 | |||
| 291 | my $fidelity = $self->_fidelity($model, 'validation'); | ||
| 292 | my ($width) = map { $_->[1] } grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} }; | ||
| 293 | my $record = { | ||
| 294 | stage => $stage, | ||
| 295 | scale => $width, | ||
| 296 | depth => scalar(grep { ref && $_->[0] eq 'dense' } @{ $model->{layers} }), | ||
| 297 | seed => $model->{seed}, | ||
| 298 | params => $model->net->n_params, | ||
| 299 | epochs => $epochs, | ||
| 300 | fidelity => $fidelity, | ||
| 301 | missed => [ grep { $fidelity->{$_} < $self->{fidelity}{$_} } sort keys %$fidelity ], | ||
| 302 | ended => $ended, | ||
| 303 | ms => $self->_ms_per_item($model), | ||
| 304 | seconds => time - $start, | ||
| 305 | confirmation => $opt{confirmation} ? 1 : 0, | ||
| 306 | backend => $model->net->backend, | ||
| 307 | }; | ||
| 308 | my $budget = $self->{budget}; | ||
| 309 | $record->{over} = join ', ', | ||
| 310 | (defined $budget->{params} && $record->{params} > $budget->{params} ? 'parameters' : ()), | ||
| 311 | (defined $budget->{ms} && $record->{ms} > $budget->{ms} ? 'milliseconds' : ()); | ||
| 312 | $record->{passed} = !@{ $record->{missed} } && !$record->{over} ? 1 : 0; | ||
| 313 | push @{ $self->{attempts} }, $record; | ||
| 314 | $self->{progress}->($record) if $self->{progress}; | ||
| 315 | return $record; | ||
| 316 | } | ||
| 317 | |||
| 318 | # Train one model until it has got there, giving it weight on what is short | ||
| 319 | # and, with $widen, more width when it is stuck. Returns the model and the | ||
| 320 | # record of its last stage. | ||
| 321 | sub _grow ($self, $width, $depth, $seed, %opt) { | ||
| 322 | my ($min, $max) = @{ $self->{search}{scale} }; | ||
| 323 | my $model = Peta::NN::Model->new(%{ $self->{model} }, layers => $self->{shape}->($width, $depth), seed => $seed, | ||
| 324 | backend => $self->{backend}); | ||
| 325 | my (%boost, $record); | ||
| 326 | my $stage = 'train'; | ||
| 327 | while (!$self->_out_of_budget) { | ||
| 328 | $record = $self->_stage($model, $stage, \%boost, confirmation => $opt{confirmation}); | ||
| 329 | last if $record->{passed} || $record->{over} || $record->{ended} eq 'time'; | ||
| 330 | if ($record->{ended} eq 'epochs') { $stage = 'train on'; next } # still getting closer: it only needs more | ||
| 331 | |||
| 332 | # Stuck. First more weight on what must be retained and is not, once | ||
| 333 | # per subset; then more room. | ||
| 334 | my @short = grep { $_ ne 'all' && ($self->{subsets}{$_}{of} // '') eq 'train' && !$boost{$_} } @{ $record->{missed} }; | ||
| 335 | if (@short) { | ||
| 336 | $boost{$_} = $BOOST for @short; | ||
| 337 | $stage = "more weight on: @short"; | ||
| 338 | next; | ||
| 339 | } | ||
| 340 | last if !$opt{widen} || $width >= $max; | ||
| 341 | my $wider = int($width * $self->{search}{grow} + 0.5); | ||
| 342 | $width = $wider > $max ? $max : $wider; | ||
| 343 | $model->widen($width); | ||
| 344 | $stage = "widened to $width"; | ||
| 345 | } | ||
| 346 | return ($model, $record); | ||
| 347 | } | ||
| 348 | |||
| 349 | # Find the smallest model that meets the thresholds, and return it. If none | ||
| 350 | # does within the budget, the closest is returned; result() says which. | ||
| 351 | sub run ($self, %arg) { | ||
| 352 | $self->{progress} = $arg{progress}; | ||
| 353 | $self->{started} = time; | ||
| 354 | |||
| 355 | # One model per depth, grown from the starting width until it gets there. | ||
| 356 | my ($chosen, $model); | ||
| 357 | for my $depth (@{ $self->{search}{depth} }) { | ||
| 358 | last if $self->_out_of_budget; | ||
| 359 | my ($grown, $record) = $self->_grow($self->{search}{start}, $depth, $self->{seed}, widen => 1); | ||
| 360 | next if !$record; | ||
| 361 | my $better = !$chosen | ||
| 362 | || $record->{passed} > $chosen->{passed} | ||
| 363 | || $record->{passed} == $chosen->{passed} | ||
| 364 | && ($record->{passed} ? $record->{params} < $chosen->{params} | ||
| 365 | : $self->_distance($record->{fidelity}) < $self->_distance($chosen->{fidelity})); | ||
| 366 | ($chosen, $model) = ($record, $grown) if $better; | ||
| 367 | last if $record->{passed}; | ||
| 368 | } | ||
| 369 | die "the budget allows no training at all\n" if !$chosen; | ||
| 370 | |||
| 371 | # Confirm it: a second seed, from nothing and from the size the first one | ||
| 372 | # reached, has to get there by the same rules. The model returned is the | ||
| 373 | # first seed's. | ||
| 374 | my $confirmed = 0; | ||
| 375 | if ($chosen->{passed} && !$self->_out_of_budget) { | ||
| 376 | my (undef, $again) = $self->_grow(@$chosen{qw(scale depth)}, $self->{seed} + 1, widen => 1, confirmation => 1); | ||
| 377 | $confirmed = 1 if $again && $again->{passed}; | ||
| 378 | } | ||
| 379 | |||
| 380 | $self->{result} = { | ||
| 381 | met => $chosen->{passed} ? 1 : 0, | ||
| 382 | confirmed => $confirmed, | ||
| 383 | %$chosen{qw(scale depth params ms fidelity missed over stage)}, | ||
| 384 | test => $self->_fidelity($model, 'test'), | ||
| 385 | runs => scalar @{ $self->{attempts} }, | ||
| 386 | epochs => eval { my $n = 0; $n += $_->{epochs} for grep { !$_->{confirmation} } @{ $self->{attempts} }; $n }, | ||
| 387 | seconds => time - $self->{started}, | ||
| 388 | stopped => $self->_out_of_budget // 'done', | ||
| 389 | }; | ||
| 390 | return $model; | ||
| 391 | } | ||
| 392 | |||
| 393 | sub attempts ($self) { return @{ $self->{attempts} } } | ||
| 394 | sub result ($self) { return $self->{result} } | ||
| 395 | |||
| 396 | # Every stage in the order it was run, and the outcome. | ||
| 397 | sub report ($self) { | ||
| 398 | my @names = sort keys %{ $self->{fidelity} }; | ||
| 399 | my @clash = $self->contradictions; | ||
| 400 | my $text = !@clash ? '' : sprintf "%d pairs set aside: their input already had another answer (%s%s)\n", scalar @clash, | ||
| 401 | join(', ', map { "$_->[0][0] → $_->[0][1], kept $_->[1]" } @clash[ 0 .. ($#clash < 2 ? $#clash : 2) ]), @clash > 3 ? ', ...' : ''; | ||
| 402 | my @same = $self->indistinct; | ||
| 403 | $text .= sprintf "%d training pairs the model cannot tell from a pair with a different answer; a wider window would (%s%s)\n", scalar @same, | ||
| 404 | join(', ', map { "$_->[0][0] → $_->[0][1] as $_->[2][0] → $_->[2][1]" } @same[ 0 .. ($#same < 2 ? $#same : 2) ]), @same > 3 ? ', ...' : '' if @same; | ||
| 405 | $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s %s\n", '#', 'stage', 'width', 'params', 'epochs', 'on', 'ms', | ||
| 406 | join(' ', map { sprintf '%-12s', substr($_, 0, 12) } @names), 'outcome'; | ||
| 407 | $text .= sprintf "%-3s %-24s %5s %7s %6s %-5s %7s %s\n", '', 'required', '', '', '', '', $self->{budget}{ms} // '', | ||
| 408 | join(' ', map { sprintf '%-12s', sprintf '>= %.1f%%', 100 * $self->{fidelity}{$_} } @names); | ||
| 409 | my %ENDED = (stuck => 'no closer', epochs => 'still getting closer', time => 'out of time'); | ||
| 410 | my $n = 0; | ||
| 411 | for my $a (@{ $self->{attempts} }) { | ||
| 412 | my $outcome = $a->{passed} ? ($a->{confirmation} ? 'confirmed' : 'met') | ||
| 413 | : $a->{over} ? "over budget: $a->{over}" | ||
| 414 | : "$ENDED{ $a->{ended} }; short: @{ $a->{missed} }"; | ||
| 415 | $text .= sprintf "%-3d %-24s %5d %7d %6d %-5s %7.3f %s %s\n", ++$n, | ||
| 416 | substr(($a->{confirmation} ? 'second seed: ' : '') . $a->{stage}, 0, 24), @$a{qw(scale params epochs backend ms)}, | ||
| 417 | join(' ', map { sprintf '%-12s', sprintf '%.2f%%', 100 * $a->{fidelity}{$_} } @names), $outcome; | ||
| 418 | } | ||
| 419 | my $r = $self->{result} or return $text; | ||
| 420 | $text .= sprintf "\n%s: depth %d, width %d, %d parameters, %.3f ms per item. %s%d stages, %d epochs, %.0f s (%s).\n", | ||
| 421 | ($r->{met} ? 'the model meets the thresholds' : 'the model does NOT meet the thresholds; this is the closest it got'), | ||
| 422 | @$r{qw(depth scale params ms)}, | ||
| 423 | ($r->{met} ? ($r->{confirmed} ? 'Confirmed on a second seed. ' : 'NOT confirmed on a second seed. ') : ''), | ||
| 424 | @$r{qw(runs epochs seconds stopped)}; | ||
| 425 | $text .= sprintf "on the test part, measured once: %s\n", | ||
| 426 | join ', ', map { sprintf '%s %.1f%%', $_, 100 * $r->{test}{$_} } @names; | ||
| 427 | return $text; | ||
| 428 | } | ||
| 429 | |||
| 430 | 1; | ||
| 431 | |||
| 432 | __END__ |