| 1 | package Peta::NN::Layer::Embed; | ||
| 2 | # ABSTRACT: learned vectors for token indices | ||
| 3 | |||
| 4 | # Embedding layer: the input is a fixed number of token indices per sample | ||
| 5 | # (the characters of a window, say), each looked up in a learned table and | ||
| 6 | # the rows concatenated. It can only be the first layer, so it never | ||
| 7 | # produces an input gradient. | ||
| 8 | |||
| 9 | use v5.36; | ||
| 10 | |||
| 11 | our $VERSION = '0.2610090'; | ||
| 12 | |||
| 13 | my $INIT_SCALE = 0.1; | ||
| 14 | |||
| 15 | sub new ($class, %arg) { | ||
| 16 | return bless { dim => $arg{dim} }, $class; | ||
| 17 | } | ||
| 18 | |||
| 19 | sub type ($self) { return 'embed' } | ||
| 20 | |||
| 21 | # $in is { tokens => count per sample, vocab => table rows }. | ||
| 22 | sub init ($self, $in, $rng, $backend) { | ||
| 23 | die "embed must be the first layer, on a token input\n" if ref $in ne 'HASH'; | ||
| 24 | my $dim = $self->{dim}; | ||
| 25 | $self->{backend} = $backend; | ||
| 26 | $self->{E} = $backend->tensor([ map { $rng->normal * $INIT_SCALE } 1 .. $in->{vocab} * $dim ], $dim); | ||
| 27 | return $in->{tokens} * $dim; | ||
| 28 | } | ||
| 29 | |||
| 30 | sub param_names ($self) { return qw(E) } | ||
| 31 | |||
| 32 | # Numbers per row of the table: one row per token. | ||
| 33 | sub param_cols ($self, $name) { return $self->{dim} } | ||
| 34 | |||
| 35 | sub forward ($self, $tokens) { | ||
| 36 | $self->{tokens} = $tokens; | ||
| 37 | return $self->{backend}->embed($self->{E}, $tokens); | ||
| 38 | } | ||
| 39 | |||
| 40 | sub backward ($self, $dY) { | ||
| 41 | $self->{gE} = $self->{backend}->embed_grad($self->{E}, $self->{tokens}, $dY); | ||
| 42 | return; | ||
| 43 | } | ||
| 44 | |||
| 45 | sub spec ($self) { return [ embed => $self->{dim} ] } | ||
| 46 | |||
| 47 | 1; | ||
| 48 | |||
| 49 | __END__ |