| 1 | package Peta::NN::Optimizer; | ||
| 2 | # ABSTRACT: SGD with momentum, and Adam | ||
| 3 | |||
| 4 | # Turns a batch's gradients into parameter updates. It keeps the per-parameter | ||
| 5 | # state (momentum, or Adam's two moments) and leaves the arithmetic to the | ||
| 6 | # backend that owns the tensors. | ||
| 7 | |||
| 8 | use v5.36; | ||
| 9 | |||
| 10 | our $VERSION = '0.2610090'; | ||
| 11 | |||
| 12 | my %DEFAULT = ( | ||
| 13 | sgd => { lr => 0.1, momentum => 0.9 }, | ||
| 14 | adam => { lr => 0.005, beta1 => 0.9, beta2 => 0.999, epsilon => 1e-8 }, | ||
| 15 | # both also take weight_decay => rate; default none | ||
| 16 | ); | ||
| 17 | |||
| 18 | # params is a list of [holder, name]: the parameter tensor is $holder->{name} | ||
| 19 | # and its gradient, renewed by every backward pass, is $holder->{"g$name"}. | ||
| 20 | sub new ($class, %arg) { | ||
| 21 | my $kind = $arg{kind} // 'adam'; | ||
| 22 | my $default = $DEFAULT{$kind} or die "unknown optimizer '$kind'\n"; | ||
| 23 | my $self = bless { %$default, %arg, kind => $kind, steps => 0 }, $class; | ||
| 24 | my $backend = $self->{backend} // die "backend => is required\n"; | ||
| 25 | my @tensors = map { $_->[0]{ $_->[1] } } @{ $self->{params} }; | ||
| 26 | $self->{m} = [ map { $backend->zeros_like($_) } @tensors ]; | ||
| 27 | $self->{v} = [ map { $backend->zeros_like($_) } @tensors ] if $kind eq 'adam'; | ||
| 28 | return $self; | ||
| 29 | } | ||
| 30 | |||
| 31 | # One update. $scale multiplies every gradient: 1/batch-size turns a batch's | ||
| 32 | # summed gradient into its mean. | ||
| 33 | sub step ($self, $scale = 1) { | ||
| 34 | my $backend = $self->{backend}; | ||
| 35 | my $t = ++$self->{steps}; | ||
| 36 | my ($lr, $b1, $b2) = @$self{qw(lr beta1 beta2)}; | ||
| 37 | # Weight decay, decoupled from the gradient: every weight shrinks a little | ||
| 38 | # at each step. Biases are left alone; shrinking them buys nothing. | ||
| 39 | my $shrink = $self->{weight_decay} ? 1 - $lr * $self->{weight_decay} : undef; | ||
| 40 | # Adam's bias correction for the zero-initialised moments, folded into the step size. | ||
| 41 | my $rate = $self->{kind} eq 'adam' ? $lr * sqrt(1 - $b2**$t) / (1 - $b1**$t) : undef; | ||
| 42 | |||
| 43 | for my $p (0 .. $#{ $self->{params} }) { | ||
| 44 | my ($holder, $name) = @{ $self->{params}[$p] }; | ||
| 45 | my ($value, $grad) = ($holder->{$name}, $holder->{"g$name"}); | ||
| 46 | $backend->decay($value, $shrink) if defined $shrink && $name ne 'b'; | ||
| 47 | if (defined $rate) { | ||
| 48 | $backend->adam_update($value, $grad, $self->{m}[$p], $self->{v}[$p], | ||
| 49 | $rate, $b1, $b2, $self->{epsilon}, $scale); | ||
| 50 | } | ||
| 51 | else { | ||
| 52 | $backend->sgd_update($value, $grad, $self->{m}[$p], $lr, $self->{momentum}, $scale); | ||
| 53 | } | ||
| 54 | } | ||
| 55 | return; | ||
| 56 | } | ||
| 57 | |||
| 58 | 1; | ||
| 59 | |||
| 60 | __END__ |