blib/lib/Peta/NN/Optimizer.pm untested

filter: Statementstmt 0.0% 0 / 36 Branchbran 0.0% 0 / 10 Conditioncond 0.0% 0 / 9 MC/DCmcdc 0.0% 0 / 5 Subroutinesub 0.0% 0 / 3 Podpod 0.0% 0 / 2 totaltotal 0.0% 0 / 65 CC 6 SCAR 30.2
CC
6
Cov
0%
step
30.0
new
6.0
BEGIN
2.0
CRAP 20.5
1 package Peta::NN::Optimizer;
2 # ABSTRACT: SGD with momentum, and Adam
3  
4 # Turns a batch's gradients into parameter updates. It keeps the per-parameter
5 # state (momentum, or Adam's two moments) and leaves the arithmetic to the
6 # backend that owns the tensors.
7  
8 use v5.36;
9  
10 our $VERSION = '0.2610090';
11  
12 my %DEFAULT = (
13 sgd => { lr => 0.1, momentum => 0.9 },
14 adam => { lr => 0.005, beta1 => 0.9, beta2 => 0.999, epsilon => 1e-8 },
15 # both also take weight_decay => rate; default none
16 );
17  
18 # params is a list of [holder, name]: the parameter tensor is $holder->{name}
19 # and its gradient, renewed by every backward pass, is $holder->{"g$name"}.
20 sub new ($class, %arg) {
21 my $kind = $arg{kind} // 'adam';
22 my $default = $DEFAULT{$kind} or die "unknown optimizer '$kind'\n";
23 my $self = bless { %$default, %arg, kind => $kind, steps => 0 }, $class;
24 my $backend = $self->{backend} // die "backend => is required\n";
25 my @tensors = map { $_->[0]{ $_->[1] } } @{ $self->{params} };
26 $self->{m} = [ map { $backend->zeros_like($_) } @tensors ];
27 $self->{v} = [ map { $backend->zeros_like($_) } @tensors ] if $kind eq 'adam';
28 return $self;
29 }
30  
31 # One update. $scale multiplies every gradient: 1/batch-size turns a batch's
32 # summed gradient into its mean.
33 sub step ($self, $scale = 1) {
34 my $backend = $self->{backend};
35 my $t = ++$self->{steps};
36 my ($lr, $b1, $b2) = @$self{qw(lr beta1 beta2)};
37 # Weight decay, decoupled from the gradient: every weight shrinks a little
38 # at each step. Biases are left alone; shrinking them buys nothing.
39 my $shrink = $self->{weight_decay} ? 1 - $lr * $self->{weight_decay} : undef;
40 # Adam's bias correction for the zero-initialised moments, folded into the step size.
41 my $rate = $self->{kind} eq 'adam' ? $lr * sqrt(1 - $b2**$t) / (1 - $b1**$t) : undef;
42  
43 for my $p (0 .. $#{ $self->{params} }) {
44 my ($holder, $name) = @{ $self->{params}[$p] };
45 my ($value, $grad) = ($holder->{$name}, $holder->{"g$name"});
46 $backend->decay($value, $shrink) if defined $shrink && $name ne 'b';
47 if (defined $rate) {
48 $backend->adam_update($value, $grad, $self->{m}[$p], $self->{v}[$p],
49 $rate, $b1, $b2, $self->{epsilon}, $scale);
50 }
51 else {
52 $backend->sgd_update($value, $grad, $self->{m}[$p], $lr, $self->{momentum}, $scale);
53 }
54 }
55 return;
56 }
57  
58 1;
59  
60 __END__