| 1 | package Peta::NN::Backend::PDL; | ||
| 2 | # ABSTRACT: Peta::NN on PDL ndarrays | ||
| 3 | |||
| 4 | # The CPU backend on PDL: a tensor is an ndarray of dims (cols, rows), which | ||
| 5 | # is PDL's own order for "rows of cols numbers", so a flat row-after-row list | ||
| 6 | # maps onto it without rearranging. Every operation is a handful of whole-array | ||
| 7 | # expressions; the batch never passes through a Perl loop. | ||
| 8 | |||
| 9 | use v5.36; | ||
| 10 | |||
| 11 | # PDL exports dozens of functions, flat() among them, which is a method here. | ||
| 12 | # They are imported into a package of their own; this one calls PDL by name. | ||
| 13 | { | ||
| 14 | package Peta::NN::Backend::PDL::Imports; | ||
| 15 | use PDL::LiteF; | ||
| 16 | } | ||
| 17 | |||
| 18 | use parent -norequire, 'Peta::NN::Backend'; | ||
| 19 | use Peta::NN::Backend; | ||
| 20 | |||
| 21 | our $VERSION = '0.2610090'; | ||
| 22 | |||
| 23 | my $TINY = 1e-300; # keeps log() defined when a probability underflows | ||
| 24 | |||
| 25 | sub new ($class) { return bless {}, $class } | ||
| 26 | |||
| 27 | sub name ($self) { return 'pdl' } | ||
| 28 | |||
| 29 | sub tensor ($self, $flat, $cols) { | ||
| 30 | return PDL->pdl(PDL::double(), $flat)->reshape($cols, @$flat / $cols); | ||
| 31 | } | ||
| 32 | |||
| 33 | # Tokens stay a Perl list: all a backend does with them is build an index. | ||
| 34 | sub tokens ($self, $flat, $per_row) { return [ [@$flat], $per_row ] } | ||
| 35 | |||
| 36 | sub flat ($self, $tensor) { return [ $tensor->list ] } | ||
| 37 | |||
| 38 | sub zeros_like ($self, $tensor) { return PDL->zeroes(PDL::double(), $tensor->dims) } | ||
| 39 | |||
| 40 | # X is (in, batch), W is (in, out): X x W^T is (out, batch), and b of dims | ||
| 41 | # (out) broadcasts over the rows. | ||
| 42 | sub affine ($self, $X, $W, $b) { | ||
| 43 | return ($X x $W->transpose) + $b; | ||
| 44 | } | ||
| 45 | |||
| 46 | sub affine_grad ($self, $X, $W, $dY, $need_dx) { | ||
| 47 | my $gW = $dY->transpose x $X; # (in, out) | ||
| 48 | my $gb = $dY->mv(1, 0)->sumover; # summed over the batch: (out) | ||
| 49 | return ($gW, $gb) if !$need_dx; | ||
| 50 | return ($gW, $gb, $dY x $W); # (in, batch) | ||
| 51 | } | ||
| 52 | |||
| 53 | my %ACTIVATE = ( | ||
| 54 | relu => sub ($x) { $x * ($x > 0) }, | ||
| 55 | tanh => sub ($x) { PDL::tanh($x) }, | ||
| 56 | sigmoid => sub ($x) { 1 / (1 + exp(-$x)) }, | ||
| 57 | ); | ||
| 58 | |||
| 59 | my %ACTIVATE_GRAD = ( | ||
| 60 | relu => sub ($y, $dy) { $dy * ($y > 0) }, | ||
| 61 | tanh => sub ($y, $dy) { $dy * (1 - $y * $y) }, | ||
| 62 | sigmoid => sub ($y, $dy) { $dy * $y * (1 - $y) }, | ||
| 63 | ); | ||
| 64 | |||
| 65 | sub activate ($self, $kind, $X) { | ||
| 66 | my $f = $ACTIVATE{$kind} // die "unknown activation '$kind'\n"; | ||
| 67 | return $f->($X); | ||
| 68 | } | ||
| 69 | |||
| 70 | sub activate_grad ($self, $kind, $Y, $dY) { | ||
| 71 | my $f = $ACTIVATE_GRAD{$kind} // die "unknown activation '$kind'\n"; | ||
| 72 | return $f->($Y, $dY); | ||
| 73 | } | ||
| 74 | |||
| 75 | # One row per token, a single 1 in the column of its index: (vocab, tokens). | ||
| 76 | # Lookup and its gradient are then both one matrix product. | ||
| 77 | sub _one_hot ($tokens, $vocab) { | ||
| 78 | my $index = PDL->pdl(PDL::long(), $tokens->[0]); | ||
| 79 | my $hot = PDL->zeroes(PDL::double(), $vocab, $index->nelem); | ||
| 80 | (my $ones = $hot->index($index)) .= 1; | ||
| 81 | return $hot; | ||
| 82 | } | ||
| 83 | |||
| 84 | sub embed ($self, $E, $tokens) { | ||
| 85 | my ($dim, $vocab) = $E->dims; | ||
| 86 | my $rows = _one_hot($tokens, $vocab) x $E; # (dim, tokens) | ||
| 87 | my $per_row = $tokens->[1]; | ||
| 88 | return $rows->reshape($dim * $per_row, @{ $tokens->[0] } / $per_row); | ||
| 89 | } | ||
| 90 | |||
| 91 | sub embed_grad ($self, $E, $tokens, $dX) { | ||
| 92 | my ($dim, $vocab) = $E->dims; | ||
| 93 | my $count = @{ $tokens->[0] }; | ||
| 94 | return _one_hot($tokens, $vocab)->transpose x $dX->copy->reshape($dim, $count); | ||
| 95 | } | ||
| 96 | |||
| 97 | sub softmax_ce ($self, $logits, $classes, $weights = undef) { | ||
| 98 | my $class = PDL->pdl(PDL::long(), $classes); | ||
| 99 | my $p = exp($logits - $logits->maximum->dummy(0)); | ||
| 100 | $p /= $p->sumover->dummy(0); | ||
| 101 | my $row_loss = -log($p->index($class) + $TINY); | ||
| 102 | (my $hit = $p->index($class)) -= 1; | ||
| 103 | return ($row_loss->sum, $p) if !$weights; | ||
| 104 | my $weight = PDL->pdl(PDL::double(), $weights); # one per row | ||
| 105 | return (($row_loss * $weight)->sum, $p * $weight->dummy(0)); | ||
| 106 | } | ||
| 107 | |||
| 108 | sub mse ($self, $out, $targets, $weights = undef) { | ||
| 109 | my ($cols) = $out->dims; | ||
| 110 | my $diff = $out - $self->tensor($targets, $cols); | ||
| 111 | return (($diff * $diff)->sum / $cols, 2 * $diff / $cols) if !$weights; | ||
| 112 | my $weight = PDL->pdl(PDL::double(), $weights); | ||
| 113 | return ((($diff * $diff)->sumover * $weight)->sum / $cols, 2 * $diff * $weight->dummy(0) / $cols); | ||
| 114 | } | ||
| 115 | |||
| 116 | # Multiply every value by $factor, in place: weight decay. | ||
| 117 | sub decay ($self, $P, $factor) { | ||
| 118 | $P *= $factor; | ||
| 119 | return; | ||
| 120 | } | ||
| 121 | |||
| 122 | sub sgd_update ($self, $P, $G, $V, $lr, $momentum, $scale) { | ||
| 123 | $V *= $momentum; | ||
| 124 | $V -= ($lr * $scale) * $G; | ||
| 125 | $P += $V; | ||
| 126 | return; | ||
| 127 | } | ||
| 128 | |||
| 129 | sub adam_update ($self, $P, $G, $M, $V, $rate, $b1, $b2, $eps, $scale) { | ||
| 130 | my $grad = $scale * $G; | ||
| 131 | $M *= $b1; | ||
| 132 | $M += (1 - $b1) * $grad; | ||
| 133 | $V *= $b2; | ||
| 134 | $V += (1 - $b2) * $grad * $grad; | ||
| 135 | $P -= $rate * $M / (sqrt($V) + $eps); | ||
| 136 | return; | ||
| 137 | } | ||
| 138 | |||
| 139 | 1; | ||
| 140 | |||
| 141 | __END__ |