| 1 | package Peta::NN::Model; | ||
| 2 | # ABSTRACT: train a micro model from string pairs, export it as a model file | ||
| 3 | |||
| 4 | # The training leg's view of a micro model: a small network with the string | ||
| 5 | # handling around it, trained from pairs of strings. What it produces is a | ||
| 6 | # model file for the inference leg (Peta::NN::Inference). | ||
| 7 | # | ||
| 8 | # Three kinds, differing only in what one decision of the network means: | ||
| 9 | # class the string as a whole gets a label | ||
| 10 | # edit one end of the string, or both, is rewritten; the label is the edit | ||
| 11 | # rewrite every character gets a replacement of its own | ||
| 12 | # | ||
| 13 | # A training pair is [input, output] followed by any number of parameters: | ||
| 14 | # [input, output, 'dative'], [input, output, 'feminine', 'comparative']. A | ||
| 15 | # parameter is an opaque value. The model learns what to do for each value | ||
| 16 | # it is shown; it knows nothing of what a value means, and every pair must | ||
| 17 | # have the same number of them. | ||
| 18 | # | ||
| 19 | # Answers always come from Peta::NN::Inference, here as after shipping, so | ||
| 20 | # what is measured during training is what a user of the model file gets. | ||
| 21 | |||
| 22 | use v5.36; | ||
| 23 | |||
| 24 | use Storable (); | ||
| 25 | |||
| 26 | use Peta::NN; | ||
| 27 | use Peta::NN::Codec qw(build_vocab edit_label edit2_label); | ||
| 28 | use Peta::NN::Inference; | ||
| 29 | use Peta::NN::RNG; | ||
| 30 | |||
| 31 | our $VERSION = '0.2610090'; | ||
| 32 | |||
| 33 | my %KIND = map { $_ => 1 } qw(class edit rewrite); | ||
| 34 | |||
| 35 | my @DEFAULT_LAYERS = ([ embed => 8 ], [ dense => 32 ], 'relu'); | ||
| 36 | my @CONFIG_KEYS = qw(kind window side radius layers seed given from to goal); # not `how`: a state is plain data, and that may hold subs | ||
| 37 | my @LEARNED_KEYS = qw(vocab labels params); | ||
| 38 | my @READING_KEYS = qw(kind window side radius vocab labels params given); # what inference needs besides the weights | ||
| 39 | my @META_KEYS = qw(name description source fidelity); | ||
| 40 | my $STATE_FORMAT = 'Peta::NN training state'; | ||
| 41 | my $EXPORT_BITS = 32; # model files carry 32-bit floats unless asked for 8; training state keeps doubles | ||
| 42 | my $EPOCHS = 30; # train() without a goal: at most so many epochs, | ||
| 43 | my $PATIENCE = 3; # and no more than so many without the held-out records being answered better | ||
| 44 | my %SEARCH = (scale => [ 16, 512 ], start => 64, depth => [1]); # train() to a goal: the widths a job tries | ||
| 45 | |||
| 46 | # What a model reads of a string, said in one table: { end => 8 }, { front => 2 }, | ||
| 47 | # { both => 5 } (so many characters of each end), or for a rewrite model | ||
| 48 | # { around => 2 } (so many either side of each character). | ||
| 49 | my %READS = (end => [ side => 'right' ], front => [ side => 'left' ], both => [ side => 'both' ], around => []); | ||
| 50 | |||
| 51 | sub new ($class, %arg) { | ||
| 52 | my $kind = $arg{kind} // die "kind => class, edit or rewrite is required\n"; | ||
| 53 | die "unknown kind '$kind'\n" if !$KIND{$kind}; | ||
| 54 | if (my $reads = delete $arg{reads}) { | ||
| 55 | my ($where, @more) = ref $reads eq 'HASH' ? keys %$reads : (); | ||
| 56 | die "reads => { end => N }, { front => N }, { both => N } or { around => N }\n" if !defined $where || @more || !$READS{$where}; | ||
| 57 | %arg = (%arg, @{ $READS{$where} }, ($where eq 'around' ? 'radius' : 'window') => $reads->{$where}); | ||
| 58 | } | ||
| 59 | my $side = $arg{side} // 'right'; | ||
| 60 | die "side is 'left', 'right', 'both' or 'auto'\n" if $side !~ /\A(?:left|right|both|auto)\z/; | ||
| 61 | die "side 'auto' is for edit models\n" if $kind ne 'edit' && $side eq 'auto'; | ||
| 62 | return bless { | ||
| 63 | kind => $kind, | ||
| 64 | window => $arg{window} // 6, # class, edit: characters read from an end | ||
| 65 | side => $side, | ||
| 66 | radius => $arg{radius} // 2, # rewrite: characters read either side | ||
| 67 | layers => $arg{layers} // [@DEFAULT_LAYERS], | ||
| 68 | seed => $arg{seed} // 1, | ||
| 69 | backend => $arg{backend}, # where training runs; see Peta::NN::Backend | ||
| 70 | given => $arg{given}, # the names of the parameters, in the order the pairs have them | ||
| 71 | from => $arg{from}, # the field of the data it reads, and the one it answers; train() | ||
| 72 | to => $arg{to}, | ||
| 73 | goal => $arg{goal}, # what it has to reach: { unseen => share, a mark of the data => share } | ||
| 74 | how => { map { $_ => $arg{$_} } grep { defined $arg{$_} } qw(train search budget) }, # how train() goes about it | ||
| 75 | sized => defined $arg{layers} ? 1 : 0, | ||
| 76 | }, $class; | ||
| 77 | } | ||
| 78 | |||
| 79 | sub net ($self) { return $self->{net} } | ||
| 80 | sub labels ($self) { return @{ $self->{labels} } } | ||
| 81 | |||
| 82 | # The values known for each parameter: one sorted list per position. | ||
| 83 | sub parameters ($self) { return map { [ sort keys %$_ ] } @{ $self->{params} // [] } } | ||
| 84 | |||
| 85 | # The names of the parameters, in their order, if they were given any. | ||
| 86 | sub given ($self) { return @{ $self->{given} // [] } } | ||
| 87 | |||
| 88 | # How many token positions the network reads per decision. | ||
| 89 | sub _tokens ($self) { | ||
| 90 | my $params = @{ $self->{params} // [] }; | ||
| 91 | return $params + 2 * $self->{radius} + 1 if $self->{kind} eq 'rewrite'; | ||
| 92 | return $params + ($self->{side} eq 'both' ? 2 : 1) * $self->{window}; | ||
| 93 | } | ||
| 94 | |||
| 95 | # How the output differs from the input, as the model's side sees it: at the | ||
| 96 | # end, at the front (an edit at the end of the reversed string), or at both. | ||
| 97 | sub _edit_label ($self, $in, $out) { | ||
| 98 | my $side = $self->{side}; | ||
| 99 | return edit2_label($in, $out) if $side eq 'both'; | ||
| 100 | return $side eq 'left' ? edit_label(scalar reverse($in), scalar reverse($out)) : edit_label($in, $out); | ||
| 101 | } | ||
| 102 | |||
| 103 | # One training pair becomes one or more (tokens, label) decisions. The | ||
| 104 | # windows are the inference leg's: both legs read a string the same way. | ||
| 105 | sub _decisions ($self, $in, $out, @values) { | ||
| 106 | my @chars = split //, $in; | ||
| 107 | my @tokens = Peta::NN::Inference::param_tokens($self->{params}, @values); | ||
| 108 | my $kind = $self->{kind}; | ||
| 109 | return [ Peta::NN::Inference::end_window($self, \@chars, @tokens), $out ] if $kind eq 'class'; | ||
| 110 | return [ Peta::NN::Inference::end_window($self, \@chars, @tokens), $self->_edit_label($in, $out) ] if $kind eq 'edit'; | ||
| 111 | |||
| 112 | my @target = split //, $out; | ||
| 113 | die "rewrite needs pairs of equal length: '$in' / '$out'\n" if @chars != @target; | ||
| 114 | return map { [ Peta::NN::Inference::char_window($self, \@chars, $_, @tokens), $target[$_] ] } 0 .. $#chars; | ||
| 115 | } | ||
| 116 | |||
| 117 | # Learn from [input, output, parameters...] string pairs, starting from | ||
| 118 | # nothing: the vocabulary, the parameter values, the labels and a fresh | ||
| 119 | # network all come from these pairs. Two of the remaining arguments are the | ||
| 120 | # model's own: | ||
| 121 | # validate string pairs that are not trained on; their loss is measured | ||
| 122 | # every epoch, which is what `patience` then watches | ||
| 123 | # weight sub ($in, $out, parameters...) giving how much a pair counts; | ||
| 124 | # 1 is plain | ||
| 125 | # Everything else goes to Peta::NN's train(). | ||
| 126 | sub fit ($self, $pairs, %train) { | ||
| 127 | # An edit model told to choose its side takes the end at which the pairs | ||
| 128 | # differ in fewer ways: a prefix shows as a handful of edits from the | ||
| 129 | # left and as one edit per word from the right. | ||
| 130 | if ($self->{side} eq 'auto') { | ||
| 131 | my %count; | ||
| 132 | for my $side (qw(right left)) { | ||
| 133 | $self->{side} = $side; | ||
| 134 | my %seen = map { $self->_edit_label(@$_[ 0, 1 ]) => 1 } @$pairs; | ||
| 135 | $count{$side} = keys %seen; | ||
| 136 | } | ||
| 137 | $self->{side} = $count{left} < $count{right} ? 'left' : 'right'; | ||
| 138 | } | ||
| 139 | |||
| 140 | die "there are no pairs to learn from\n" if !@$pairs; | ||
| 141 | $self->{vocab} = build_vocab([ map { $_->[0] } @$pairs ]); | ||
| 142 | |||
| 143 | # Parameter values are tokens after the characters, one embedding row | ||
| 144 | # each. A value has its own token per position: 'a' as first parameter | ||
| 145 | # and 'a' as second are different things to the model. | ||
| 146 | my $positions = @{ $pairs->[0] } - 2; | ||
| 147 | die "every pair must have the same number of parameters\n" if grep { @$_ - 2 != $positions } @$pairs; | ||
| 148 | die sprintf "the pairs have %d parameter%s, and the model is given %d name%s for them (%s)\n", $positions, $positions == 1 ? '' : 's', | ||
| 149 | scalar @{ $self->{given} }, @{ $self->{given} } == 1 ? '' : 's', join ', ', @{ $self->{given} } | ||
| 150 | if $self->{given} && @{ $self->{given} } != $positions; | ||
| 151 | my $next = 2 + keys %{ $self->{vocab} }; | ||
| 152 | $self->{params} = [ | ||
| 153 | map { | ||
| 154 | my $position = $_; | ||
| 155 | my %seen = map { ($_->[ 2 + $position ] // die "a parameter is undefined\n") => 1 } @$pairs; | ||
| 156 | +{ map { $_ => $next++ } sort keys %seen } | ||
| 157 | } 0 .. $positions - 1 | ||
| 158 | ]; | ||
| 159 | |||
| 160 | # Labels are numbered in order of first appearance. | ||
| 161 | my (%seen, @labels); | ||
| 162 | for my $label (map { $_->[1] } map { $self->_decisions(@$_) } @$pairs) { | ||
| 163 | push @labels, $label if !$seen{$label}++; | ||
| 164 | } | ||
| 165 | $self->{labels} = \@labels; | ||
| 166 | |||
| 167 | $self->{net} = $self->_network($self->{layers}); | ||
| 168 | return $self->_train($pairs, %train); | ||
| 169 | } | ||
| 170 | |||
| 171 | # Go on training the network the model has, on new pairs. What the model can | ||
| 172 | # read and answer stays as it is: a character it has never seen reads as | ||
| 173 | # unknown, and a pair whose answer is not one of its labels, or whose | ||
| 174 | # parameter value it does not know, cannot be learned this way and is | ||
| 175 | # refused. The arguments are fit()'s. | ||
| 176 | sub tune ($self, $pairs, %train) { | ||
| 177 | die "the model has not been trained; use fit\n" if !$self->{net}; | ||
| 178 | my %known = map { $_ => 1 } @{ $self->{labels} }; | ||
| 179 | my @foreign = grep { !$known{ $_->[1] } } map { $self->_decisions(@$_) } @$pairs; | ||
| 180 | die sprintf "%d of the pairs need an answer this model does not have (%s); that takes a fit from scratch\n", | ||
| 181 | scalar @foreign, join ', ', map { "'$_->[1]'" } @foreign[ 0 .. ($#foreign < 2 ? $#foreign : 2) ] if @foreign; | ||
| 182 | return $self->_train($pairs, %train); | ||
| 183 | } | ||
| 184 | |||
| 185 | # Make the hidden layers wider, to $width units each, without changing a | ||
| 186 | # single answer: a new unit reads with fresh random weights and is read with | ||
| 187 | # zeros, so the model computes what it did until training gives the new | ||
| 188 | # units something to say. A layer that is as wide already stays as it is. | ||
| 189 | # This is how a model that has learned all its size allows gets more room | ||
| 190 | # and goes on learning, instead of a larger one starting from nothing. | ||
| 191 | sub widen ($self, $width) { | ||
| 192 | die "the model has not been trained; a model is widened after fit\n" if !$self->{net}; | ||
| 193 | die "the width is a positive whole number, not '$width'\n" if $width !~ /\A[1-9][0-9]*\z/; | ||
| 194 | my @layers = $self->{net}->layers; | ||
| 195 | my @weights = @{ $self->{net}->weights }; | ||
| 196 | my $rng = Peta::NN::RNG->new($self->{seed} * 7919 + $width); | ||
| 197 | my ($last) = grep { $layers[$_]->type eq 'dense' } reverse 0 .. $#layers; # the output layer keeps its size | ||
| 198 | my $added = 0; # how many inputs the layer in hand has gained, at the end of each row | ||
| 199 | my (@new, @spec); | ||
| 200 | for my $at (0 .. $#layers) { | ||
| 201 | my $layer = $layers[$at]; | ||
| 202 | my @own = splice @weights, 0, scalar(() = $layer->param_names); | ||
| 203 | if ($layer->type ne 'dense') { push @new, @own; push @spec, $layer->spec if $at < $last; next } | ||
| 204 | |||
| 205 | my ($W, $b) = @own; | ||
| 206 | my $out = @$b; | ||
| 207 | my $in = @$W / $out; | ||
| 208 | my $more = $at < $last && $width > $out ? $width - $out : 0; | ||
| 209 | my $limit = sqrt(6 / ($in + $added + $out + $more)); | ||
| 210 | my @W = map { (@$W[ $_ * $in .. ($_ + 1) * $in - 1 ], (0.0) x $added) } 0 .. $out - 1; # what was there reads nothing new | ||
| 211 | push @W, map { (2 * $rng->uniform - 1) * $limit } 1 .. $more * ($in + $added); # a new unit reads everything | ||
| 212 | push @new, \@W, [ @$b, (0.0) x $more ]; | ||
| 213 | push @spec, [ dense => $out + $more ] if $at < $last; | ||
| 214 | $added = $more; | ||
| 215 | } | ||
| 216 | $self->{layers} = \@spec; | ||
| 217 | $self->{net} = $self->_network(\@spec)->set_weights(\@new); | ||
| 218 | delete @$self{qw(inference inference_of)}; | ||
| 219 | return $self; | ||
| 220 | } | ||
| 221 | |||
| 222 | # How many rows the embedding table needs: the two reserved tokens, the | ||
| 223 | # characters, and every value of every parameter. | ||
| 224 | sub _token_rows ($self) { | ||
| 225 | my $rows = 2 + keys %{ $self->{vocab} }; | ||
| 226 | $rows += keys %$_ for @{ $self->{params} // [] }; | ||
| 227 | return $rows; | ||
| 228 | } | ||
| 229 | |||
| 230 | # A network for this model's reading and labels: the given hidden layers and | ||
| 231 | # an output layer of one unit per label. | ||
| 232 | sub _network ($self, $hidden) { | ||
| 233 | return Peta::NN->new( | ||
| 234 | input => { tokens => $self->_tokens, vocab => $self->_token_rows }, | ||
| 235 | layers => [ @$hidden, [ dense => scalar @{ $self->{labels} } ] ], | ||
| 236 | loss => 'softmax', | ||
| 237 | seed => $self->{seed}, | ||
| 238 | backend => $self->{backend}, | ||
| 239 | ); | ||
| 240 | } | ||
| 241 | |||
| 242 | # Train the network on the pairs, with the labels the model has. | ||
| 243 | sub _train ($self, $pairs, %train) { | ||
| 244 | my $validate = delete $train{validate}; | ||
| 245 | my $weight = delete $train{weight}; | ||
| 246 | my %index; | ||
| 247 | @index{ @{ $self->{labels} } } = 0 .. $#{ $self->{labels} }; | ||
| 248 | |||
| 249 | # [tokens, class, weight]; one pair may be several decisions, all of its weight. | ||
| 250 | my @data = map { | ||
| 251 | my $counts = $weight ? $weight->(@$_) : undef; | ||
| 252 | map { [ $_->[0], $index{ $_->[1] }, $counts ] } $self->_decisions(@$_) | ||
| 253 | } @$pairs; | ||
| 254 | |||
| 255 | # A validation pair whose answer is a label the model does not have cannot | ||
| 256 | # be scored by the loss; accuracy() will count it as the miss it is. | ||
| 257 | my @held = map { [ $_->[0], $index{ $_->[1] } ] } grep { exists $index{ $_->[1] } } | ||
| 258 | map { $self->_decisions(@$_) } @{ $validate // [] }; | ||
| 259 | $self->{net}->train(data => \@data, (@held ? (validate => \@held) : ()), %train); | ||
| 260 | return $self; | ||
| 261 | } | ||
| 262 | |||
| 263 | # A trainable model from a model file: what export() wrote is enough to grow | ||
| 264 | # the training leg's side back. The architecture follows from the layers in | ||
| 265 | # the file; the weights are the file's, so as coarse as it stored them. | ||
| 266 | # tune() goes on from there. | ||
| 267 | sub from_model ($class, $file, %arg) { | ||
| 268 | my $data = Peta::NN::Inference::read_file($file, $Peta::NN::Inference::FORMAT); | ||
| 269 | my $read = eval { Peta::NN::Inference->new($data) } or die $@ =~ s/\Amalformed model:/$file is a malformed model:/r; | ||
| 270 | |||
| 271 | my @layers = @{ $read->{layers} }; # checked, and with their weights unpacked | ||
| 272 | my @hidden = map { $_->{type} eq 'embed' ? [ embed => $_->{dim} ] | ||
| 273 | : $_->{type} eq 'dense' ? [ dense => scalar @{ $_->{weights}[1] } ] | ||
| 274 | : $_->{type} } @layers; | ||
| 275 | die "$file does not end in a dense layer\n" if !ref $hidden[-1] || $hidden[-1][0] ne 'dense'; | ||
| 276 | pop @hidden; # the output layer: _network adds it, one unit per label | ||
| 277 | |||
| 278 | my $self = $class->new(%$data{qw(kind window side radius given)}, layers => \@hidden, %arg); | ||
| 279 | @$self{qw(vocab labels)} = @$data{qw(vocab labels)}; | ||
| 280 | $self->{params} = $data->{params} // []; | ||
| 281 | $self->{net} = $self->_network(\@hidden)->set_weights([ map { @{ $_->{weights} } } @layers ]); | ||
| 282 | return $self; | ||
| 283 | } | ||
| 284 | |||
| 285 | # --- trained on data, to a goal ---------------------------------------------------- | ||
| 286 | |||
| 287 | # The pairs this model learns from, of some data: what it reads, what it | ||
| 288 | # answers and what it is given are fields of the records. | ||
| 289 | sub _pairs_of ($self, $data) { | ||
| 290 | defined $self->{$_} or die "a model that is trained on data says which field it reads and which it answers: from => ..., to => ...\n" for qw(from to); | ||
| 291 | return $data->pairs(from => $self->{from}, to => $self->{to}, given => $self->{given} // []); | ||
| 292 | } | ||
| 293 | |||
| 294 | # Train the model on data (Peta::NN::Data): on the records that are not held | ||
| 295 | # out, reading the field `from`, answering the field `to`, given the fields | ||
| 296 | # `given`. A model with a goal is trained until it gets there, by a job | ||
| 297 | # (Peta::NN::Job): `unseen` is the share of records it was not shown that it | ||
| 298 | # has to answer right, and every other name is a mark of the data, the share | ||
| 299 | # of those records it has to. Without a goal it is fitted once, with the | ||
| 300 | # held-out records to say when to stop. Options: | ||
| 301 | # train what goes to the training itself (batch, lr, epochs, ...) | ||
| 302 | # search for a goal: the widths to try, { scale => [min, max], start => N } | ||
| 303 | # budget for a goal: { seconds => N, ms => N per answer } | ||
| 304 | # backend where; the fastest there is unless named | ||
| 305 | sub train ($self, $data, %how) { | ||
| 306 | # What the model was told when it was made, and over that what this call says. | ||
| 307 | %how = (%how, map { $_ => { %{ $self->{how}{$_} // {} }, %{ $how{$_} // {} } } } grep { $self->{how}{$_} || $how{$_} } qw(train search budget)); | ||
| 308 | my $pairs = $self->_pairs_of($data->shown); | ||
| 309 | die "there are no records to train on\n" if !@$pairs; | ||
| 310 | my $goal = $self->{goal}; | ||
| 311 | $self->{backend} = $how{backend} if defined $how{backend}; | ||
| 312 | $self->{backend} //= $ENV{PETA_NN_BACKEND} // 'auto'; | ||
| 313 | if (!$goal) { | ||
| 314 | my $held = $self->_pairs_of($data->held); | ||
| 315 | $self->fit($pairs, (@$held ? (validate => $held, patience => $PATIENCE) : ()), epochs => $EPOCHS, %{ $how{train} // {} }); | ||
| 316 | return $self; | ||
| 317 | } | ||
| 318 | |||
| 319 | require Peta::NN::Job; | ||
| 320 | my $key = sub ($pair) { join "\x{1f}", @$pair[ 0, 2 .. $#$pair ] }; | ||
| 321 | my %marked = map { $_ => { map { $key->($_) => 1 } @{ $self->_pairs_of($data->shown->marked($_)) } } } grep { $_ ne 'unseen' } keys %$goal; | ||
| 322 | my $job = Peta::NN::Job->new( | ||
| 323 | model => { map { $_ => $self->{$_} } grep { defined $self->{$_} } qw(kind window side radius given) }, | ||
| 324 | pairs => $pairs, | ||
| 325 | fidelity => { (defined $goal->{unseen} ? (all => $goal->{unseen}) : ()), map { $_ => $goal->{$_} } keys %marked }, | ||
| 326 | subsets => { map { my $set = $marked{$_}; ($_ => { of => 'train', where => sub ($pair) { $set->{ $key->($pair) } } }) } keys %marked }, | ||
| 327 | (%marked ? (always_train => sub ($pair) { my $k = $key->($pair); scalar grep { $_->{$k} } values %marked }) : ()), | ||
| 328 | backend => $self->{backend}, | ||
| 329 | seed => $self->{seed}, | ||
| 330 | search => { %SEARCH, %{ $how{search} // {} } }, | ||
| 331 | (map { $_ => $how{$_} } grep { defined $how{$_} } qw(budget train)), | ||
| 332 | ); | ||
| 333 | my $trained = $job->run; | ||
| 334 | @$self{qw(layers side vocab labels params net)} = @$trained{qw(layers side vocab labels params net)}; | ||
| 335 | delete @$self{qw(inference card)}; | ||
| 336 | $self->{report} = $job->report; | ||
| 337 | $self->{reached} = $job->result; | ||
| 338 | return $self; | ||
| 339 | } | ||
| 340 | |||
| 341 | # What the job that trained the model has to say, and what it measured; for | ||
| 342 | # a model trained to a goal. | ||
| 343 | sub report ($self) { return $self->{report} } | ||
| 344 | sub reached ($self) { return $self->{reached} } | ||
| 345 | |||
| 346 | # How the model does on data: the share of records it answers exactly, of | ||
| 347 | # those that are held out (`unseen`), and of each mark of the data, of its | ||
| 348 | # records that are not held out: those are the ones a goal is about. | ||
| 349 | sub score ($self, $data) { | ||
| 350 | my %of = (unseen => $data->held, map { $_ => $data->shown->marked($_) } $data->marks); | ||
| 351 | return { map { my $pairs = $self->_pairs_of($of{$_}); @$pairs ? ($_ => $self->accuracy($pairs)) : () } keys %of }; | ||
| 352 | } | ||
| 353 | |||
| 354 | # The model as the inference leg's data: how it reads a string, its labels, | ||
| 355 | # and its layers with their weights as lists. | ||
| 356 | sub data ($self) { | ||
| 357 | die "the model has not been trained\n" if !$self->{net}; | ||
| 358 | my @weights = @{ $self->{net}->weights }; # in layer order, as many lists as each layer has parameters | ||
| 359 | my @layers = map { | ||
| 360 | { type => $_->type, | ||
| 361 | ($_->type eq 'embed' ? (dim => $_->{dim}) : ()), | ||
| 362 | weights => [ splice @weights, 0, scalar(() = $_->param_names) ], | ||
| 363 | } | ||
| 364 | } $self->{net}->layers; | ||
| 365 | return { %$self{@READING_KEYS}, layers => \@layers }; | ||
| 366 | } | ||
| 367 | |||
| 368 | # The inference object for the weights as they are now. It is rebuilt when | ||
| 369 | # the network has been updated since, so it is safe to ask during training. | ||
| 370 | sub inference ($self) { | ||
| 371 | my $generation = $self->{net} ? $self->{net}->generation : die "the model has not been trained\n"; | ||
| 372 | if (!$self->{inference} || $self->{inference_of} != $generation) { | ||
| 373 | $self->{inference} = Peta::NN::Inference->new($self->data); | ||
| 374 | $self->{inference_of} = $generation; | ||
| 375 | } | ||
| 376 | return $self->{inference}; | ||
| 377 | } | ||
| 378 | |||
| 379 | # The model's answer for a string and its parameters; in list context also | ||
| 380 | # its confidence (for rewrite: that of its least certain character). | ||
| 381 | sub predict ($self, $text, @values) { return $self->inference->predict($text, @values) } | ||
| 382 | |||
| 383 | sub predict_all ($self, $texts, @values) { return $self->inference->predict_all($texts, @values) } | ||
| 384 | |||
| 385 | # Every answer the model considers, with its probability; see Peta::NN::Inference. | ||
| 386 | sub distribution ($self, $text, @values) { return $self->inference->distribution($text, @values) } | ||
| 387 | |||
| 388 | # One distribution for several strings together; class models only. | ||
| 389 | sub pooled ($self, $texts, @values) { return $self->inference->pooled($texts, @values) } | ||
| 390 | |||
| 391 | # This model as one that answers on the graphics card, when that is where it | ||
| 392 | # is trained and it is of a kind the card answers for (class models, and edit | ||
| 393 | # models that rewrite the end): the weights as they are now, in single | ||
| 394 | # precision. Nothing otherwise. | ||
| 395 | sub _on_card ($self) { | ||
| 396 | my $net = $self->{net} or die "the model has not been trained\n"; | ||
| 397 | return if $net->backend ne 'gpu' || $self->{kind} eq 'rewrite' || $self->{kind} eq 'edit' && $self->{side} ne 'right'; | ||
| 398 | if (!$self->{card} || $self->{card_of} != $net->generation) { | ||
| 399 | require Peta::NN::Fused; | ||
| 400 | my @params = map { \(my $n = $_) } 0 .. $#{ $self->{params} // [] }; | ||
| 401 | $self->{card} = Peta::NN::Fused->new( | ||
| 402 | models => { model => $self->data }, | ||
| 403 | steps => [ { model => 'model', params => \@params, ($self->{kind} eq 'class' ? (name => 'answer', classify => 1) : ()) } ], | ||
| 404 | engine => 'gpu', | ||
| 405 | ); | ||
| 406 | $self->{card_of} = $net->generation; | ||
| 407 | } | ||
| 408 | return $self->{card}; | ||
| 409 | } | ||
| 410 | |||
| 411 | # The pairs this model cannot get right together with all the others: those | ||
| 412 | # it reads exactly as it reads some other pair that has a different answer. | ||
| 413 | # The model sees a window of a string, not the string, so two strings that | ||
| 414 | # differ outside it are one and the same to it (with nine characters from | ||
| 415 | # the end, frühschichten and geschichten are both "schichten"). No training | ||
| 416 | # changes that; a wider window does. Returned as [pair, another answer, a | ||
| 417 | # pair that has it], one for each such pair. | ||
| 418 | sub indistinct ($self, $pairs) { | ||
| 419 | die "the model has not been trained\n" if !$self->{net}; | ||
| 420 | my (%answers, @read); | ||
| 421 | for my $pair (@$pairs) { | ||
| 422 | my @keys; | ||
| 423 | for my $decision ($self->_decisions(@$pair)) { | ||
| 424 | my $key = join ' ', @{ $decision->[0] }; | ||
| 425 | $answers{$key}{ $decision->[1] } //= $pair; | ||
| 426 | push @keys, [ $key, $decision->[1] ]; | ||
| 427 | } | ||
| 428 | push @read, [ $pair, \@keys ]; | ||
| 429 | } | ||
| 430 | my @found; | ||
| 431 | for my $read (@read) { | ||
| 432 | my ($pair, $keys) = @$read; | ||
| 433 | for my $key (@$keys) { | ||
| 434 | my ($other) = grep { $_ ne $key->[1] } sort keys %{ $answers{ $key->[0] } } or next; | ||
| 435 | push @found, [ $pair, $other, $answers{ $key->[0] }{$other} ]; | ||
| 436 | last; | ||
| 437 | } | ||
| 438 | } | ||
| 439 | return @found; | ||
| 440 | } | ||
| 441 | |||
| 442 | # Share of pairs reproduced exactly. Pairs with the same parameters are | ||
| 443 | # answered in one pass. | ||
| 444 | sub accuracy ($self, $pairs, %how) { | ||
| 445 | my %same; | ||
| 446 | push @{ $same{ join "\x{1f}", @$_[ 2 .. $#$_ ] } }, $_ for @$pairs; | ||
| 447 | my $card = $how{where_trained} ? $self->_on_card : undef; | ||
| 448 | my $right = 0; | ||
| 449 | for my $group (values %same) { | ||
| 450 | my @values = @{ $group->[0] }[ 2 .. $#{ $group->[0] } ]; | ||
| 451 | my $inputs = [ map { $_->[0] } @$group ]; | ||
| 452 | my @answers = !$card ? map { $_->[0] } @{ $self->inference->answers($inputs, @values) } | ||
| 453 | : $self->{kind} eq 'class' ? map { $_->{answers}{answer} } @{ $card->run($inputs, @values) } | ||
| 454 | : map { $_->{text} } @{ $card->run($inputs, @values) }; | ||
| 455 | $right += grep { $answers[$_] eq $group->[$_][1] } 0 .. $#$group; | ||
| 456 | } | ||
| 457 | return $right / @$pairs; | ||
| 458 | } | ||
| 459 | |||
| 460 | # The training state as plain data: the model's definition and its weights | ||
| 461 | # at full precision. It is what a state file holds, and what carries a model | ||
| 462 | # from one process to another. | ||
| 463 | sub state ($self) { | ||
| 464 | die "the model has not been trained\n" if !$self->{net}; | ||
| 465 | return { %$self{ @CONFIG_KEYS, @LEARNED_KEYS }, format => $STATE_FORMAT, version => $VERSION, | ||
| 466 | net => Peta::NN::freeze_state($self->{net}->state) }; | ||
| 467 | } | ||
| 468 | |||
| 469 | # The model a training state describes. The network inside refuses weights | ||
| 470 | # that do not fit its definition. | ||
| 471 | sub from_state ($class, $state, %arg) { | ||
| 472 | die "this is not a training state\n" | ||
| 473 | if ref $state ne 'HASH' || ($state->{format} // '') ne $STATE_FORMAT || ref $state->{net} ne 'HASH' || ref $state->{layers} ne 'ARRAY'; | ||
| 474 | my $self = $class->new(%$state{@CONFIG_KEYS}, %arg); | ||
| 475 | @$self{@LEARNED_KEYS} = @$state{@LEARNED_KEYS}; | ||
| 476 | $self->{params} //= []; | ||
| 477 | $self->{net} = Peta::NN->from_state(Peta::NN::thaw_state($state->{net}), backend => $self->{backend}); | ||
| 478 | return $self; | ||
| 479 | } | ||
| 480 | |||
| 481 | # The training state as a file (by convention *.state), to be loaded again by | ||
| 482 | # this class. | ||
| 483 | sub save ($self, $file) { | ||
| 484 | Storable::nstore($self->state, $file); | ||
| 485 | return $self; | ||
| 486 | } | ||
| 487 | |||
| 488 | # The file is read as plain data. | ||
| 489 | sub load ($class, $file, %arg) { | ||
| 490 | my $model = eval { $class->from_state(Peta::NN::Inference::read_file($file, $STATE_FORMAT), %arg) }; | ||
| 491 | die $@ =~ s/\Athis is not a training state/$file is a malformed training state/r if !$model; | ||
| 492 | return $model; | ||
| 493 | } | ||
| 494 | |||
| 495 | # Write the model file that ships (by convention *.model): the inference | ||
| 496 | # leg's data with the weights packed, and what the file says of itself. | ||
| 497 | # file where to write it | ||
| 498 | # bits 32 (the default) for 32-bit floats, 8 for signed bytes with | ||
| 499 | # a scale per weight array: a quarter of the size, at some | ||
| 500 | # cost in accuracy that is the caller's to measure | ||
| 501 | # name, description, source, fidelity | ||
| 502 | # free-form: what the model is, what it was trained from, and | ||
| 503 | # the fidelity measured for it ({ name => share }) | ||
| 504 | # Peta::NN::Inference->load reads the file. | ||
| 505 | sub export ($self, %arg) { | ||
| 506 | my $file = $arg{file} // die "file => path is required\n"; | ||
| 507 | my $bits = $arg{bits} // $EXPORT_BITS; | ||
| 508 | die "a model file stores weights with 8 or 32 bits, not '$bits'\n" if $bits != 8 && $bits != 32; | ||
| 509 | my $data = $self->data; | ||
| 510 | for my $layer (@{ $data->{layers} }) { | ||
| 511 | $layer->{weights} = [ map { Peta::NN::Inference::pack_weights($_, $bits) } @{ $layer->{weights} } ]; | ||
| 512 | } | ||
| 513 | Storable::nstore({ | ||
| 514 | %$data, | ||
| 515 | format => $Peta::NN::Inference::FORMAT, | ||
| 516 | layout => $Peta::NN::Inference::LAYOUT, | ||
| 517 | version => $VERSION, | ||
| 518 | bits => $bits, | ||
| 519 | meta => { created => time, map { $_ => $arg{$_} } grep { defined $arg{$_} } @META_KEYS }, | ||
| 520 | }, $file); | ||
| 521 | return $self; | ||
| 522 | } | ||
| 523 | |||
| 524 | 1; | ||
| 525 | |||
| 526 | __END__ |