| 1 | package Peta::NN::Inference; | ||
| 2 | # ABSTRACT: load a trained model and get answers from it | ||
| 3 | |||
| 4 | # The inference leg: load a trained model and get answers from it. Nothing of | ||
| 5 | # the training leg is needed; this module, Peta::NN::Codec and a model file | ||
| 6 | # are all that ships. | ||
| 7 | # | ||
| 8 | # A model is plain data: what kind it is, how it reads a string (window, | ||
| 9 | # side, radius, vocabulary, parameters), its labels, and the layers with | ||
| 10 | # their weights. Model files hold exactly that, in Storable's portable | ||
| 11 | # format, with each weight array packed. | ||
| 12 | # | ||
| 13 | # A model takes a string and, if it was trained with them, parameters: | ||
| 14 | # opaque values it has learned to react to, without knowing what they mean. | ||
| 15 | # "Äpfel" with "dative" gives "Äpfeln" because the training pairs said so. A | ||
| 16 | # model does not judge its input either; given nonsense it answers something. | ||
| 17 | # | ||
| 18 | # The arithmetic runs on PDL when this perl has it and in plain Perl loops | ||
| 19 | # when not; the answers are the same. PETA_NN_ENGINE=plain forces the loops. | ||
| 20 | |||
| 21 | use v5.36; | ||
| 22 | use feature 'refaliasing'; | ||
| 23 | no warnings 'experimental::refaliasing'; | ||
| 24 | |||
| 25 | use Storable (); | ||
| 26 | |||
| 27 | use Peta::NN::Codec qw(window apply_edit apply_edit2); | ||
| 28 | |||
| 29 | our $VERSION = '0.2610090'; | ||
| 30 | |||
| 31 | our $FORMAT = 'Peta::NN model'; | ||
| 32 | |||
| 33 | # The layout of a model file. It goes up when files of the old layout can no | ||
| 34 | # longer be read; such a file is refused by number, not by a puzzling detail. | ||
| 35 | # 1 one optional task, weights packed as named by `pack` | ||
| 36 | # 2 any number of parameters, `bits`, a description of the model | ||
| 37 | our $LAYOUT = 2; | ||
| 38 | |||
| 39 | # PDL exports dozens of functions; they go into a package of their own. | ||
| 40 | my $ENGINE = ($ENV{PETA_NN_ENGINE} // '') ne 'plain' | ||
| 41 | && eval "package Peta::NN::Inference::PDL; use PDL::LiteF; 1" ? 'pdl' : 'plain'; | ||
| 42 | |||
| 43 | my $SATURATED = 20; # beyond this exp() adds nothing in a double | ||
| 44 | my $CHUNK = 256; # windows per forward pass | ||
| 45 | my $FLOOR = 1e-12; # the least probability a pooled label is given, so one zero cannot veto | ||
| 46 | |||
| 47 | sub engine { return $ENGINE } | ||
| 48 | |||
| 49 | # --- how a model reads a string ---------------------------------------------- | ||
| 50 | # Used here to answer and by the training leg to build its training data, so | ||
| 51 | # that both see a string the same way. $config is anything with the keys | ||
| 52 | # side, window, radius and vocab; @tokens are the parameters' tokens. | ||
| 53 | |||
| 54 | # The window of the whole-string kinds: the first characters, the last, or | ||
| 55 | # the first followed by the last. | ||
| 56 | sub end_window ($config, $chars, @tokens) { | ||
| 57 | my ($side, $size, $vocab) = @$config{qw(side window vocab)}; | ||
| 58 | return [ | ||
| 59 | @tokens, | ||
| 60 | ($side ne 'right' ? @{ window($chars, 0, $size, $vocab) } : ()), | ||
| 61 | ($side ne 'left' ? @{ window($chars, @$chars - $size, $size, $vocab) } : ()), | ||
| 62 | ]; | ||
| 63 | } | ||
| 64 | |||
| 65 | # The window around one character, for rewrite. | ||
| 66 | sub char_window ($config, $chars, $pos, @tokens) { | ||
| 67 | my $radius = $config->{radius}; | ||
| 68 | return [ @tokens, @{ window($chars, $pos - $radius, 2 * $radius + 1, $config->{vocab}) } ]; | ||
| 69 | } | ||
| 70 | |||
| 71 | # The tokens of a call's parameters. $params is the model's list of tables, | ||
| 72 | # one per parameter position, each from value to token. The number of | ||
| 73 | # parameters must be the model's; a value the model was never trained with | ||
| 74 | # is refused. | ||
| 75 | sub param_tokens ($params, @values) { | ||
| 76 | my @tables = @{ $params // [] }; | ||
| 77 | die sprintf "this model takes %d parameter%s, not %d\n", scalar @tables, @tables == 1 ? '' : 's', scalar @values | ||
| 78 | if @values != @tables; | ||
| 79 | return map { | ||
| 80 | my $value = $values[$_] // die "parameter @{[ $_ + 1 ]} is undefined\n"; | ||
| 81 | $tables[$_]{$value} // die "parameter @{[ $_ + 1 ]} has no value '$value' in this model (it has: @{[ sort keys %{ $tables[$_] } ]})\n" | ||
| 82 | } 0 .. $#tables; | ||
| 83 | } | ||
| 84 | |||
| 85 | # --- weights as stored --------------------------------------------------------- | ||
| 86 | # A weight array is, in memory, a list of numbers. In a file it is a string | ||
| 87 | # packed as 32- or 64-bit floats, or { int8 => string, scale => number }: | ||
| 88 | # signed bytes that give the weights when multiplied by the scale. | ||
| 89 | |||
| 90 | my %FLOAT_PACK = (32 => 'f<', 64 => 'd<'); | ||
| 91 | |||
| 92 | sub pack_weights ($weights, $bits) { | ||
| 93 | return pack "$FLOAT_PACK{$bits}*", @$weights if $FLOAT_PACK{$bits}; | ||
| 94 | die "weights can be stored with 8, 32 or 64 bits, not '$bits'\n" if $bits != 8; | ||
| 95 | my $largest = 0; | ||
| 96 | for my $w (@$weights) { $largest = abs $w if abs $w > $largest } | ||
| 97 | my $scale = $largest ? $largest / 127 : 1; | ||
| 98 | return { scale => $scale, int8 => pack 'c*', map { my $q = $_ / $scale; int($q + ($q < 0 ? -0.5 : 0.5)) } @$weights }; | ||
| 99 | } | ||
| 100 | |||
| 101 | sub _unpack_weights ($stored, $bits) { | ||
| 102 | return $stored if ref $stored eq 'ARRAY'; | ||
| 103 | return [ map { $_ * $stored->{scale} } unpack 'c*', $stored->{int8} ] if ref $stored eq 'HASH'; | ||
| 104 | return [ unpack "$FLOAT_PACK{$bits}*", $stored ]; | ||
| 105 | } | ||
| 106 | |||
| 107 | # How many numbers a stored weight array holds, or nothing if it is not one. | ||
| 108 | sub _stored_length ($stored, $bits) { | ||
| 109 | return scalar @$stored if ref $stored eq 'ARRAY'; | ||
| 110 | if (ref $stored eq 'HASH') { | ||
| 111 | return if ($bits // 0) != 8 || ref $stored->{int8} || !defined $stored->{int8}; | ||
| 112 | return if !defined $stored->{scale} || ref $stored->{scale} || $stored->{scale} !~ /\A[0-9.eE+-]+\z/ || $stored->{scale} <= 0; | ||
| 113 | return length $stored->{int8}; | ||
| 114 | } | ||
| 115 | my $bytes = !defined $bits ? undef : $bits == 32 ? 4 : $bits == 64 ? 8 : undef; | ||
| 116 | return if ref $stored || !defined $stored || !$bytes || length($stored) % $bytes; | ||
| 117 | return length($stored) / $bytes; | ||
| 118 | } | ||
| 119 | |||
| 120 | # --- reading and checking a model --------------------------------------------- | ||
| 121 | |||
| 122 | my %KIND = map { $_ => 1 } qw(class edit rewrite); | ||
| 123 | my %SIDE = map { $_ => 1 } qw(left right both); | ||
| 124 | my %ACTIVATION = map { $_ => 1 } qw(relu tanh sigmoid); | ||
| 125 | my %BITS = map { $_ => 1 } qw(8 32 64); | ||
| 126 | |||
| 127 | # Read a Storable file as plain data. Flags 0: nothing in the file is blessed | ||
| 128 | # into a class or tied, so reading it runs no code of anyone's choosing. | ||
| 129 | sub read_file ($file, $format) { | ||
| 130 | my $data = eval { Storable::retrieve($file, 0) }; | ||
| 131 | die "$file cannot be read as a $format file: " . ($@ =~ s/ at \S+ line \d+.*//sr || "$!") . "\n" if !defined $data; | ||
| 132 | die "$file is not a $format file\n" if ref $data ne 'HASH' || ($data->{format} // '') ne $format; | ||
| 133 | return $data; | ||
| 134 | } | ||
| 135 | |||
| 136 | sub _is_count ($value) { return defined $value && !ref $value && $value =~ /\A[1-9][0-9]*\z/ } | ||
| 137 | |||
| 138 | # Die unless $data is a model this module can answer from: every field of the | ||
| 139 | # right type, and every layer of the size its neighbours need. A model that | ||
| 140 | # passes cannot index outside its weights or its labels. | ||
| 141 | sub _validate ($data) { | ||
| 142 | my $bad = sub ($what) { die "malformed model: $what\n" }; | ||
| 143 | |||
| 144 | # Data built in this process carries no layout; a file always does. | ||
| 145 | $bad->(sprintf 'it is in layout %s, and this version reads layout %d; export it again', $data->{layout} // 1, $LAYOUT) | ||
| 146 | if defined $data->{format} && ($data->{layout} // 1) ne $LAYOUT; | ||
| 147 | |||
| 148 | $bad->('unknown kind') if !$KIND{ $data->{kind} // '' }; | ||
| 149 | $bad->('unknown side') if !$SIDE{ $data->{side} // '' }; | ||
| 150 | $bad->('window is not a positive whole number') if !_is_count($data->{window}); | ||
| 151 | $bad->('radius is not a positive whole number') if !_is_count($data->{radius}); | ||
| 152 | |||
| 153 | my ($vocab, $labels, $params, $layers) = @$data{qw(vocab labels params layers)}; | ||
| 154 | $bad->('no vocabulary') if ref $vocab ne 'HASH'; | ||
| 155 | $bad->('no labels') if ref $labels ne 'ARRAY' || !@$labels || grep { !defined || ref } @$labels; | ||
| 156 | $bad->('parameters are not a list of tables') | ||
| 157 | if defined $params && (ref $params ne 'ARRAY' || grep { ref ne 'HASH' || !%$_ } @$params); | ||
| 158 | # The parameters' names, if the model has them: one each, no name twice. | ||
| 159 | if (defined(my $given = $data->{given})) { | ||
| 160 | my %seen; | ||
| 161 | $bad->('the names of the parameters are not one plain name for each parameter') | ||
| 162 | if ref $given ne 'ARRAY' || @$given != @{ $params // [] } || grep { !defined || ref || !length || $seen{$_}++ } @$given; | ||
| 163 | } | ||
| 164 | $bad->('no layers') if ref $layers ne 'ARRAY' || !@$layers; | ||
| 165 | $bad->('the description of the model is not a table') if defined $data->{meta} && ref $data->{meta} ne 'HASH'; | ||
| 166 | |||
| 167 | # Token indices: 0 and 1 are reserved; the characters and the parameter | ||
| 168 | # values follow, no index twice and none beyond the table. | ||
| 169 | my @index = (values %$vocab, map { values %$_ } @{ $params // [] }); | ||
| 170 | my %seen; | ||
| 171 | $bad->('a token index is not a whole number of 2 or more, or is used twice') | ||
| 172 | if grep { !_is_count($_) || $_ < 2 || $seen{$_}++ } @index; | ||
| 173 | my $rows = 2 + @index; | ||
| 174 | $bad->('token indices leave gaps') if grep { $_ >= $rows } @index; | ||
| 175 | |||
| 176 | # An edit's label must be an edit: "cut:add", or for both ends "cut:add|cut:add". | ||
| 177 | if ($data->{kind} eq 'edit') { | ||
| 178 | my $form = $data->{side} eq 'both' ? qr/\A[0-9]+:[^|]*\|[0-9]+:.*\z/s : qr/\A[0-9]+:.*\z/s; | ||
| 179 | $bad->('an edit label is not an edit') if grep { $_ !~ $form } @$labels; | ||
| 180 | } | ||
| 181 | |||
| 182 | my $bits = $data->{bits}; | ||
| 183 | $bad->('unknown weight size') if defined $bits && !$BITS{$bits}; | ||
| 184 | |||
| 185 | # Follow the sizes through the layers. $width is what a layer hands on. | ||
| 186 | my $tokens = @{ $params // [] } | ||
| 187 | + ($data->{kind} eq 'rewrite' ? 2 * $data->{radius} + 1 : ($data->{side} eq 'both' ? 2 : 1) * $data->{window}); | ||
| 188 | my $width; | ||
| 189 | for my $n (0 .. $#$layers) { | ||
| 190 | my $layer = $layers->[$n]; | ||
| 191 | $bad->("layer $n is not a layer") if ref $layer ne 'HASH' || ref $layer->{weights} ne 'ARRAY'; | ||
| 192 | my $type = $layer->{type} // ''; | ||
| 193 | my @length = map { _stored_length($_, $bits) // $bad->("layer $n has weights that are not stored as the file says") } | ||
| 194 | @{ $layer->{weights} }; | ||
| 195 | |||
| 196 | if ($type eq 'embed') { | ||
| 197 | $bad->('an embedding that is not the first layer') if $n != 0; | ||
| 198 | $bad->('embedding dimension is not a positive whole number') if !_is_count($layer->{dim}); | ||
| 199 | $bad->('the embedding table does not have one row per token') if @length != 1 || $length[0] != $rows * $layer->{dim}; | ||
| 200 | $width = $tokens * $layer->{dim}; | ||
| 201 | } | ||
| 202 | elsif ($type eq 'dense') { | ||
| 203 | $bad->('a dense layer with nothing before it') if !defined $width; | ||
| 204 | $bad->("dense layer $n does not fit the layer before it") | ||
| 205 | if @length != 2 || !$length[1] || $length[0] != $width * $length[1]; | ||
| 206 | $width = $length[1]; | ||
| 207 | } | ||
| 208 | elsif ($ACTIVATION{$type}) { | ||
| 209 | $bad->("activation $n has weights, or nothing before it") if @length || !defined $width; | ||
| 210 | } | ||
| 211 | else { $bad->("layer $n is of unknown type") } | ||
| 212 | } | ||
| 213 | $bad->('the last layer does not have one output per label') if ($width // -1) != @$labels; | ||
| 214 | return; | ||
| 215 | } | ||
| 216 | |||
| 217 | # --- a model ----------------------------------------------------------------- | ||
| 218 | |||
| 219 | # From model data, which is checked first. Weight arrays may be lists, or | ||
| 220 | # stored as `bits` says. A weight that is not a finite number is refused: it | ||
| 221 | # would poison every answer. | ||
| 222 | sub new ($class, $data) { | ||
| 223 | die "a model is a table of fields\n" if ref $data ne 'HASH'; | ||
| 224 | _validate($data); | ||
| 225 | my $self = bless { %$data }, $class; | ||
| 226 | $self->{as_stored} = $data; | ||
| 227 | $self->{layers} = [ | ||
| 228 | map { | ||
| 229 | my $layer = { %$_ }; | ||
| 230 | $layer->{weights} = [ map { _unpack_weights($_, $data->{bits}) } @{ $layer->{weights} } ]; | ||
| 231 | $layer | ||
| 232 | } @{ $data->{layers} } | ||
| 233 | ]; | ||
| 234 | for my $weights (map { @{ $_->{weights} } } @{ $self->{layers} }) { | ||
| 235 | die "malformed model: a weight is not a finite number\n" if grep { $_ != $_ || abs($_) == 9**9**9 } @$weights; | ||
| 236 | } | ||
| 237 | if ($ENGINE eq 'pdl') { | ||
| 238 | for my $layer (grep { $_->{type} eq 'dense' } @{ $self->{layers} }) { | ||
| 239 | my ($W, $b) = @{ $layer->{weights} }; | ||
| 240 | # (in, out) as stored, transposed once: a batch (in, rows) times it is (out, rows). | ||
| 241 | $layer->{Wt} = PDL->pdl(PDL::double(), $W)->reshape(@$W / @$b, scalar @$b)->transpose->copy; | ||
| 242 | $layer->{bias} = PDL->pdl(PDL::double(), $b); | ||
| 243 | } | ||
| 244 | } | ||
| 245 | return $self; | ||
| 246 | } | ||
| 247 | |||
| 248 | # From a model file (by convention *.model). The file is read as plain data | ||
| 249 | # and checked in full before anything is computed from it. | ||
| 250 | sub load ($class, $file) { | ||
| 251 | my $model = eval { $class->new(read_file($file, $FORMAT)) }; | ||
| 252 | die $@ =~ s/\Amalformed model:/$file is a malformed model:/r if !$model; | ||
| 253 | return $model; | ||
| 254 | } | ||
| 255 | |||
| 256 | sub kind ($self) { return $self->{kind} } | ||
| 257 | |||
| 258 | # The model data this object was made from, weights as they were stored. | ||
| 259 | sub data ($self) { return $self->{as_stored} } | ||
| 260 | sub labels ($self) { return @{ $self->{labels} } } | ||
| 261 | |||
| 262 | # The values the model knows for each of its parameters: one sorted list per | ||
| 263 | # parameter position. A model without parameters has none. | ||
| 264 | sub parameters ($self) { | ||
| 265 | return map { [ sort keys %$_ ] } @{ $self->{params} // [] }; | ||
| 266 | } | ||
| 267 | |||
| 268 | # The names of the parameters, in their order; none for a model without | ||
| 269 | # parameters, or for one whose parameters were not given names. | ||
| 270 | sub given ($self) { return @{ $self->{given} // [] } } | ||
| 271 | |||
| 272 | # What a caller gave after the string, as the parameters' values in their | ||
| 273 | # order. A model whose parameters have names takes them by name | ||
| 274 | # (gender => 'neuter'); one whose parameters have none, in their order. | ||
| 275 | sub values_from ($self, @given) { | ||
| 276 | my $names = $self->{given} or return @given; | ||
| 277 | die sprintf "this model takes %s, by name\n", join ', ', map { "$_ => ..." } @$names if @given != 2 * @$names; | ||
| 278 | my %named = @given; | ||
| 279 | return map { delete $named{$_} // die sprintf "this model takes %s; '%s' is missing\n", join(', ', @$names), $_ } @$names; | ||
| 280 | } | ||
| 281 | |||
| 282 | sub n_params ($self) { | ||
| 283 | my $n = 0; | ||
| 284 | $n += @$_ for map { @{ $_->{weights} } } @{ $self->{layers} }; | ||
| 285 | return $n; | ||
| 286 | } | ||
| 287 | |||
| 288 | # What there is to know about the model, for a human: what the file says of | ||
| 289 | # itself (name, description, source, when it was made, the fidelity it was | ||
| 290 | # measured at) and what follows from its contents. | ||
| 291 | sub info ($self) { | ||
| 292 | my @shape = map { | ||
| 293 | my $type = $_->{type}; | ||
| 294 | $type eq 'embed' ? "embed $_->{dim}" : $type eq 'dense' ? 'dense ' . @{ $_->{weights}[1] } : $type | ||
| 295 | } @{ $self->{layers} }; | ||
| 296 | return { | ||
| 297 | %{ $self->{meta} // {} }, | ||
| 298 | format => $self->{format}, | ||
| 299 | version => $self->{version}, | ||
| 300 | bits => $self->{bits}, | ||
| 301 | kind => $self->{kind}, | ||
| 302 | reads => $self->{kind} eq 'rewrite' ? "$self->{radius} characters either side of each character" | ||
| 303 | : $self->{side} eq 'both' ? "the first and the last $self->{window} characters" | ||
| 304 | : "the " . ($self->{side} eq 'left' ? 'first' : 'last') . " $self->{window} characters", | ||
| 305 | layers => join(', ', @shape), | ||
| 306 | weights => $self->n_params, | ||
| 307 | labels => scalar @{ $self->{labels} }, | ||
| 308 | characters => scalar keys %{ $self->{vocab} }, | ||
| 309 | parameters => [ $self->parameters ], | ||
| 310 | given => [ $self->given ], | ||
| 311 | }; | ||
| 312 | } | ||
| 313 | |||
| 314 | # The table rows of every token of every window, one window after another. | ||
| 315 | sub _embedded ($layer, $windows) { | ||
| 316 | my ($table, $dim) = ($layer->{weights}[0], $layer->{dim}); | ||
| 317 | return [ map { my $at = $_ * $dim; @$table[ $at .. $at + $dim - 1 ] } map { @$_ } @$windows ]; | ||
| 318 | } | ||
| 319 | |||
| 320 | # Y = X W^T + b for rows of $n_in numbers. The arrays are aliased to lexical | ||
| 321 | # ones so that the loops are plain array code, which is what a JIT compiles. | ||
| 322 | sub _affine ($X, $W, $b, $n_in) { | ||
| 323 | \my @x = $X; | ||
| 324 | \my @W = $W; | ||
| 325 | \my @b = $b; | ||
| 326 | my $n_out = @b; | ||
| 327 | my $rows = @x / $n_in; | ||
| 328 | my @y = (0.0) x ($rows * $n_out); | ||
| 329 | for my $s (0 .. $rows - 1) { | ||
| 330 | my $x_at = $s * $n_in; | ||
| 331 | my $y_at = $s * $n_out; | ||
| 332 | for my $j (0 .. $n_out - 1) { | ||
| 333 | my $sum = $b[$j]; | ||
| 334 | my $w_at = $j * $n_in; | ||
| 335 | for my $i (0 .. $n_in - 1) { | ||
| 336 | $sum += $W[ $w_at + $i ] * $x[ $x_at + $i ]; | ||
| 337 | } | ||
| 338 | $y[ $y_at + $j ] = $sum; | ||
| 339 | } | ||
| 340 | } | ||
| 341 | return \@y; | ||
| 342 | } | ||
| 343 | |||
| 344 | # In place. | ||
| 345 | sub _activate ($type, $X) { | ||
| 346 | \my @x = $X; | ||
| 347 | if ($type eq 'relu') { | ||
| 348 | for my $i (0 .. $#x) { $x[$i] = 0.0 if $x[$i] < 0 } | ||
| 349 | } | ||
| 350 | elsif ($type eq 'tanh') { | ||
| 351 | for my $i (0 .. $#x) { | ||
| 352 | my $v = $x[$i]; | ||
| 353 | if ($v > $SATURATED) { $x[$i] = 1.0 } | ||
| 354 | elsif ($v < -$SATURATED) { $x[$i] = -1.0 } | ||
| 355 | else { my $e = exp(2 * $v); $x[$i] = ($e - 1) / ($e + 1) } | ||
| 356 | } | ||
| 357 | } | ||
| 358 | else { | ||
| 359 | for my $i (0 .. $#x) { | ||
| 360 | my $v = $x[$i]; | ||
| 361 | $x[$i] = $v > $SATURATED ? 1.0 : $v < -$SATURATED ? 0.0 : 1 / (1 + exp(-$v)); | ||
| 362 | } | ||
| 363 | } | ||
| 364 | return; | ||
| 365 | } | ||
| 366 | |||
| 367 | # The network's outputs for a list of windows, as one flat list of rows. | ||
| 368 | sub _outputs_plain ($self, $windows) { | ||
| 369 | my ($x, $cols); | ||
| 370 | for my $layer (@{ $self->{layers} }) { | ||
| 371 | my $type = $layer->{type}; | ||
| 372 | if ($type eq 'embed') { | ||
| 373 | $x = _embedded($layer, $windows); | ||
| 374 | $cols = @$x / @$windows; | ||
| 375 | } | ||
| 376 | elsif ($type eq 'dense') { | ||
| 377 | $x = _affine($x, @{ $layer->{weights} }, $cols); | ||
| 378 | $cols = @{ $layer->{weights}[1] }; | ||
| 379 | } | ||
| 380 | else { _activate($type, $x) } | ||
| 381 | } | ||
| 382 | return $x; | ||
| 383 | } | ||
| 384 | |||
| 385 | sub _outputs_pdl ($self, $windows) { | ||
| 386 | my $x; | ||
| 387 | for my $layer (@{ $self->{layers} }) { | ||
| 388 | my $type = $layer->{type}; | ||
| 389 | if ($type eq 'embed') { | ||
| 390 | my $flat = _embedded($layer, $windows); | ||
| 391 | $x = PDL->pdl(PDL::double(), $flat)->reshape(@$flat / @$windows, scalar @$windows); | ||
| 392 | } | ||
| 393 | elsif ($type eq 'dense') { $x = ($x x $layer->{Wt}) + $layer->{bias} } | ||
| 394 | elsif ($type eq 'relu') { $x = $x * ($x > 0) } | ||
| 395 | elsif ($type eq 'tanh') { $x = PDL::tanh($x) } | ||
| 396 | else { $x = 1 / (1 + exp(-$x)) } | ||
| 397 | } | ||
| 398 | return [ $x->list ]; | ||
| 399 | } | ||
| 400 | |||
| 401 | # For each window the probability of every label, in the labels' order. | ||
| 402 | sub _probabilities ($self, $windows) { | ||
| 403 | my $cols = @{ $self->{labels} }; | ||
| 404 | my @rows; | ||
| 405 | for (my $start = 0; $start < @$windows; $start += $CHUNK) { | ||
| 406 | my $end = $start + $CHUNK - 1; | ||
| 407 | $end = $#$windows if $end > $#$windows; | ||
| 408 | my @chunk = @$windows[ $start .. $end ]; | ||
| 409 | my $out = $ENGINE eq 'pdl' ? $self->_outputs_pdl(\@chunk) : $self->_outputs_plain(\@chunk); | ||
| 410 | for my $row (0 .. $#chunk) { | ||
| 411 | my @z = @$out[ $row * $cols .. ($row + 1) * $cols - 1 ]; | ||
| 412 | my $top = $z[0]; | ||
| 413 | for my $v (@z) { $top = $v if $v > $top } | ||
| 414 | my $sum = 0; | ||
| 415 | $sum += ($_ = exp($_ - $top)) for @z; # the largest first becomes 1: nothing overflows | ||
| 416 | $_ /= $sum for @z; | ||
| 417 | push @rows, \@z; | ||
| 418 | } | ||
| 419 | } | ||
| 420 | return \@rows; | ||
| 421 | } | ||
| 422 | |||
| 423 | sub _best ($probabilities) { | ||
| 424 | my $best = 0; | ||
| 425 | for my $i (1 .. $#$probabilities) { $best = $i if $probabilities->[$i] > $probabilities->[$best] } | ||
| 426 | return $best; | ||
| 427 | } | ||
| 428 | |||
| 429 | # A decided edit, carried out. An edit that cannot apply to this string | ||
| 430 | # leaves it as it is. | ||
| 431 | sub _edited ($self, $text, $label) { | ||
| 432 | my $side = $self->{side}; | ||
| 433 | return apply_edit2($text, $label) // $text if $side eq 'both'; | ||
| 434 | return apply_edit($text, $label) // $text if $side eq 'right'; | ||
| 435 | my $result = apply_edit(scalar reverse($text), $label); # at the front: the same on the reversed string | ||
| 436 | return defined $result ? scalar reverse $result : $text; | ||
| 437 | } | ||
| 438 | |||
| 439 | # The windows of many strings and, for rewrite, how many belong to each. | ||
| 440 | sub _windows ($self, $texts, @values) { | ||
| 441 | my @tokens = param_tokens($self->{params}, @values); | ||
| 442 | my @chars = map { [ split // ] } @$texts; | ||
| 443 | return [ map { end_window($self, $_, @tokens) } @chars ] if $self->{kind} ne 'rewrite'; | ||
| 444 | return ([ map { my $c = $_; map { char_window($self, $c, $_, @tokens) } 0 .. $#$c } @chars ], [ map { scalar @$_ } @chars ]); | ||
| 445 | } | ||
| 446 | |||
| 447 | # The probability of every label, in the labels' order, for each of many | ||
| 448 | # windows. | ||
| 449 | sub probabilities ($self, $windows) { return $self->_probabilities($windows) } | ||
| 450 | |||
| 451 | # [index of the best label, its probability] for each of many windows: the | ||
| 452 | # model's decisions for input that is already tokens, which is what a fused | ||
| 453 | # model hands its parts. | ||
| 454 | sub decisions ($self, $windows) { | ||
| 455 | return [ map { my $best = _best($_); [ $best, $_->[$best] ] } @{ $self->_probabilities($windows) } ]; | ||
| 456 | } | ||
| 457 | |||
| 458 | # [answer, confidence] for each of many strings, all their decisions in one | ||
| 459 | # pass. For a rewrite the confidence is that of its least certain character. | ||
| 460 | sub answers ($self, $texts, @values) { | ||
| 461 | my ($windows, $lengths) = $self->_windows($texts, @values); | ||
| 462 | my $rows = $self->_probabilities($windows); | ||
| 463 | my $labels = $self->{labels}; | ||
| 464 | |||
| 465 | if ($self->{kind} eq 'rewrite') { | ||
| 466 | my @answers; | ||
| 467 | for my $length (@$lengths) { | ||
| 468 | my ($result, $confidence) = ('', 1); | ||
| 469 | for my $p (splice @$rows, 0, $length) { | ||
| 470 | my $best = _best($p); | ||
| 471 | $result .= $labels->[$best]; | ||
| 472 | $confidence = $p->[$best] if $p->[$best] < $confidence; | ||
| 473 | } | ||
| 474 | push @answers, [ $result, $confidence ]; | ||
| 475 | } | ||
| 476 | return \@answers; | ||
| 477 | } | ||
| 478 | |||
| 479 | my $edit = $self->{kind} eq 'edit'; | ||
| 480 | return [ | ||
| 481 | map { | ||
| 482 | my $best = _best($rows->[$_]); | ||
| 483 | [ $edit ? $self->_edited($texts->[$_], $labels->[$best]) : $labels->[$best], $rows->[$_][$best] ] | ||
| 484 | } 0 .. $#$texts | ||
| 485 | ]; | ||
| 486 | } | ||
| 487 | |||
| 488 | # The model's answer for a string, with its parameters if the model has any; | ||
| 489 | # in list context also its confidence. | ||
| 490 | sub predict ($self, $text, @given) { | ||
| 491 | my ($answer) = @{ $self->answers([$text], $self->values_from(@given)) }; | ||
| 492 | return wantarray ? @$answer : $answer->[0]; | ||
| 493 | } | ||
| 494 | |||
| 495 | # The answers for many strings at once, in their order, all with the same | ||
| 496 | # parameters. | ||
| 497 | sub predict_all ($self, $texts, @given) { | ||
| 498 | return map { $_->[0] } @{ $self->answers($texts, $self->values_from(@given)) }; | ||
| 499 | } | ||
| 500 | |||
| 501 | # Everything the model considers for a string, not only its best answer: a | ||
| 502 | # list of [answer, probability], most probable first, the probabilities | ||
| 503 | # adding up to 1. A model that is not sure shows it here. For an edit model | ||
| 504 | # the answers are the resulting strings (two edits giving the same string | ||
| 505 | # are one answer). For a rewrite it is one such list per character. | ||
| 506 | sub distribution ($self, $text, @given) { | ||
| 507 | my ($windows) = $self->_windows([$text], $self->values_from(@given)); | ||
| 508 | my $rows = $self->_probabilities($windows); | ||
| 509 | my $labels = $self->{labels}; | ||
| 510 | my $sorted = sub ($p) { | ||
| 511 | my %share; | ||
| 512 | for my $i (0 .. $#$labels) { | ||
| 513 | my $answer = $self->{kind} eq 'edit' ? $self->_edited($text, $labels->[$i]) : $labels->[$i]; | ||
| 514 | $share{$answer} += $p->[$i]; | ||
| 515 | } | ||
| 516 | return [ map { [ $_, $share{$_} ] } sort { $share{$b} <=> $share{$a} || $a cmp $b } keys %share ]; | ||
| 517 | }; | ||
| 518 | return $self->{kind} eq 'rewrite' ? [ map { $sorted->($_) } @$rows ] : $sorted->($rows->[0]); | ||
| 519 | } | ||
| 520 | |||
| 521 | # One distribution for several strings together, for a class model: what | ||
| 522 | # label fits ALL of them, taking each string as independent evidence (the | ||
| 523 | # product of the probabilities, renormalised). A long text sharpens it; one | ||
| 524 | # short ambiguous word leaves it flat. Returned as distribution() does. | ||
| 525 | sub pooled ($self, $texts, @given) { return $self->pooled_for($texts, $self->values_from(@given)) } | ||
| 526 | |||
| 527 | # The same with the parameters' values in their order, as answers() takes them. | ||
| 528 | sub pooled_for ($self, $texts, @values) { | ||
| 529 | die "pooling is for class models\n" if $self->{kind} ne 'class'; | ||
| 530 | my ($windows) = $self->_windows($texts, @values); | ||
| 531 | return $self->pool($self->_probabilities($windows)); | ||
| 532 | } | ||
| 533 | |||
| 534 | # The same from the probabilities themselves: one list per string, as | ||
| 535 | # probabilities() gives them. | ||
| 536 | sub pool ($self, $rows) { | ||
| 537 | my $labels = $self->{labels}; | ||
| 538 | my @log = (0) x @$labels; | ||
| 539 | for my $p (@$rows) { | ||
| 540 | $log[$_] += log($p->[$_] > $FLOOR ? $p->[$_] : $FLOOR) for 0 .. $#log; | ||
| 541 | } | ||
| 542 | my $top = $log[0]; | ||
| 543 | for my $v (@log) { $top = $v if $v > $top } | ||
| 544 | my @share = map { exp($_ - $top) } @log; | ||
| 545 | my $sum = 0; | ||
| 546 | $sum += $_ for @share; | ||
| 547 | return [ sort { $b->[1] <=> $a->[1] || $a->[0] cmp $b->[0] } map { [ $labels->[$_], $share[$_] / $sum ] } 0 .. $#share ]; | ||
| 548 | } | ||
| 549 | |||
| 550 | 1; | ||
| 551 | |||
| 552 | __END__ |