blib/lib/Peta/NN/Layer/Embed.pm untested

filter: Statementstmt 0.0% 0 / 23 Branchbran 0.0% 0 / 2 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 9 Podpod 0.0% 0 / 8 totaltotal 0.0% 0 / 42 CC 2 SCAR 10.3
CC
2
Cov
0%
init
6.0
new
2.0
type
2.0
CRAP 2.8
1 package Peta::NN::Layer::Embed;
2 # ABSTRACT: learned vectors for token indices
3  
4 # Embedding layer: the input is a fixed number of token indices per sample
5 # (the characters of a window, say), each looked up in a learned table and
6 # the rows concatenated. It can only be the first layer, so it never
7 # produces an input gradient.
8  
9 use v5.36;
10  
11 our $VERSION = '0.2610090';
12  
13 my $INIT_SCALE = 0.1;
14  
15 sub new ($class, %arg) {
16 return bless { dim => $arg{dim} }, $class;
17 }
18  
19 sub type ($self) { return 'embed' }
20  
21 # $in is { tokens => count per sample, vocab => table rows }.
22 sub init ($self, $in, $rng, $backend) {
23 die "embed must be the first layer, on a token input\n" if ref $in ne 'HASH';
24 my $dim = $self->{dim};
25 $self->{backend} = $backend;
26 $self->{E} = $backend->tensor([ map { $rng->normal * $INIT_SCALE } 1 .. $in->{vocab} * $dim ], $dim);
27 return $in->{tokens} * $dim;
28 }
29  
30 sub param_names ($self) { return qw(E) }
31  
32 # Numbers per row of the table: one row per token.
33 sub param_cols ($self, $name) { return $self->{dim} }
34  
35 sub forward ($self, $tokens) {
36 $self->{tokens} = $tokens;
37 return $self->{backend}->embed($self->{E}, $tokens);
38 }
39  
40 sub backward ($self, $dY) {
41 $self->{gE} = $self->{backend}->embed_grad($self->{E}, $self->{tokens}, $dY);
42 return;
43 }
44  
45 sub spec ($self) { return [ embed => $self->{dim} ] }
46  
47 1;
48  
49 __END__