| 1 | package Peta::NN::Layer::Dense; | ||
| 2 | # ABSTRACT: fully connected layer | ||
| 3 | |||
| 4 | # Fully connected layer: Y = X W^T + b, for a whole batch of rows X. The | ||
| 5 | # weights are a backend tensor of one row per output. | ||
| 6 | |||
| 7 | use v5.36; | ||
| 8 | |||
| 9 | our $VERSION = '0.2610090'; | ||
| 10 | |||
| 11 | sub new ($class, %arg) { | ||
| 12 | return bless { n_out => $arg{out}, need_dx => 1 }, $class; | ||
| 13 | } | ||
| 14 | |||
| 15 | sub type ($self) { return 'dense' } | ||
| 16 | |||
| 17 | # Glorot uniform: keeps the signal's variance about constant through the layer | ||
| 18 | # for tanh and sigmoid, and is close enough to He's for the small ReLU nets here. | ||
| 19 | sub init ($self, $n_in, $rng, $backend) { | ||
| 20 | die "dense cannot read tokens; put an embed layer first\n" if ref $n_in; | ||
| 21 | my $n_out = $self->{n_out}; | ||
| 22 | my $limit = sqrt(6 / ($n_in + $n_out)); | ||
| 23 | $self->{backend} = $backend; | ||
| 24 | $self->{n_in} = $n_in; | ||
| 25 | $self->{W} = $backend->tensor([ map { (2 * $rng->uniform - 1) * $limit } 1 .. $n_in * $n_out ], $n_in); | ||
| 26 | $self->{b} = $backend->tensor([ (0.0) x $n_out ], $n_out); | ||
| 27 | return $n_out; | ||
| 28 | } | ||
| 29 | |||
| 30 | sub param_names ($self) { return qw(W b) } | ||
| 31 | |||
| 32 | # Numbers per row of a parameter tensor: W has one row per output. | ||
| 33 | sub param_cols ($self, $name) { return $name eq 'W' ? $self->{n_in} : $self->{n_out} } | ||
| 34 | |||
| 35 | sub forward ($self, $X) { | ||
| 36 | $self->{X} = $X; | ||
| 37 | return $self->{backend}->affine($X, $self->{W}, $self->{b}); | ||
| 38 | } | ||
| 39 | |||
| 40 | # Leaves the batch's gradient in gW and gb and returns the gradient with | ||
| 41 | # respect to the input. The first layer of a net has nobody to hand that to, | ||
| 42 | # so the net clears need_dx there and the backend skips it. | ||
| 43 | sub backward ($self, $dY) { | ||
| 44 | my @grad = $self->{backend}->affine_grad($self->{X}, $self->{W}, $dY, $self->{need_dx}); | ||
| 45 | @$self{qw(gW gb)} = @grad[ 0, 1 ]; | ||
| 46 | return $grad[2]; | ||
| 47 | } | ||
| 48 | |||
| 49 | sub spec ($self) { return [ dense => $self->{n_out} ] } | ||
| 50 | |||
| 51 | 1; | ||
| 52 | |||
| 53 | __END__ |