| 1 | package Peta::NN::Layer::Dense; | ||||||||||
| 2 | # ABSTRACT: fully connected layer | ||||||||||
| 3 | |||||||||||
| 4 | # Fully connected layer: Y = X W^T + b, for a whole batch of rows X. The | ||||||||||
| 5 | # weights are a backend tensor of one row per output. | ||||||||||
| 6 | |||||||||||
| 7 | 27 | ▶ | use v5.36; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 8 | |||||||||||
| 9 | 27 | our $VERSION = '0.2610090'; | |||||||||
| 10 | |||||||||||
| 11 | 302 | ▶ | sub new ($class, %arg) { | ||||||||
|
Subroutine
new: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 12 | 302 | return bless { n_out => $arg{out}, need_dx => 1 }, $class; | |||||||||
| 13 | } | ||||||||||
| 14 | |||||||||||
| 15 | 3195 | ▶ | sub type ($self) { return 'dense' } | ||||||||
|
Subroutine
type: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 16 | |||||||||||
| 17 | # Glorot uniform: keeps the signal's variance about constant through the layer | ||||||||||
| 18 | # for tanh and sigmoid, and is close enough to He's for the small ReLU nets here. | ||||||||||
| 19 | 301 | ▶ | sub init ($self, $n_in, $rng, $backend) { | ||||||||
|
Subroutine
init: called CC=2 SCAR=6.9
Pod
documented
| |||||||||||
| 20 | 301 | ▶ | die "dense cannot read tokens; put an embed layer first\n" if ref $n_in; | ||||||||
|
Branch: 1 branch
| |||||||||||
| 21 | 300 | my $n_out = $self->{n_out}; | |||||||||
| 22 | 300 | my $limit = sqrt(6 / ($n_in + $n_out)); | |||||||||
| 23 | 300 | $self->{backend} = $backend; | |||||||||
| 24 | 300 | $self->{n_in} = $n_in; | |||||||||
| 25 | 300 | $self->{W} = $backend->tensor([ map { (2 * $rng->uniform - 1) * $limit } 1 .. $n_in * $n_out ], $n_in); | |||||||||
| 26 | 300 | $self->{b} = $backend->tensor([ (0.0) x $n_out ], $n_out); | |||||||||
| 27 | 300 | return $n_out; | |||||||||
| 28 | } | ||||||||||
| 29 | |||||||||||
| 30 | 34949 | ▶ | sub param_names ($self) { return qw(W b) } | ||||||||
|
Subroutine
param_names: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 31 | |||||||||||
| 32 | # Numbers per row of a parameter tensor: W has one row per output. | ||||||||||
| 33 | 8008 | ▶ | sub param_cols ($self, $name) { return $name eq 'W' ? $self->{n_in} : $self->{n_out} } | ||||||||
|
Subroutine
param_cols: called CC=2 SCAR=6.9
Pod
documented
Branch: 1 branch
| |||||||||||
| 34 | |||||||||||
| 35 | 191094 | ▶ | sub forward ($self, $X) { | ||||||||
|
Subroutine
forward: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 36 | 191094 | $self->{X} = $X; | |||||||||
| 37 | 191094 | return $self->{backend}->affine($X, $self->{W}, $self->{b}); | |||||||||
| 38 | } | ||||||||||
| 39 | |||||||||||
| 40 | # Leaves the batch's gradient in gW and gb and returns the gradient with | ||||||||||
| 41 | # respect to the input. The first layer of a net has nobody to hand that to, | ||||||||||
| 42 | # so the net clears need_dx there and the backend skips it. | ||||||||||
| 43 | 186200 | ▶ | sub backward ($self, $dY) { | ||||||||
|
Subroutine
backward: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 44 | 186200 | my @grad = $self->{backend}->affine_grad($self->{X}, $self->{W}, $dY, $self->{need_dx}); | |||||||||
| 45 | 186200 | @$self{qw(gW gb)} = @grad[ 0, 1 ]; | |||||||||
| 46 | 186200 | return $grad[2]; | |||||||||
| 47 | } | ||||||||||
| 48 | |||||||||||
| 49 | 20 | ▶ | sub spec ($self) { return [ dense => $self->{n_out} ] } | ||||||||
|
Subroutine
spec: called CC=1 SCAR=0.0
Pod
documented
| |||||||||||
| 50 | |||||||||||
| 51 | 27 | 1; | |||||||||
| 52 | |||||||||||
| 53 | __END__ | ||||||||||