| 1 | package Peta::NN::Backend::Plain; | ||
| 2 | # ABSTRACT: Peta::NN on plain Perl arrays | ||
| 3 | |||
| 4 | # The reference backend: a tensor is [ \@numbers, $cols ], row after row, and | ||
| 5 | # every operation is a plain Perl loop. | ||
| 6 | # | ||
| 7 | # Each kernel first aliases the tensors' arrays to lexical arrays (\my @W = | ||
| 8 | # ...). The loops then read as ordinary array code, which is also the form | ||
| 9 | # pperl's JIT compiles; through the references themselves it does not. | ||
| 10 | |||
| 11 | use v5.36; | ||
| 12 | use feature 'refaliasing'; | ||
| 13 | no warnings 'experimental::refaliasing'; | ||
| 14 | |||
| 15 | use parent -norequire, 'Peta::NN::Backend'; | ||
| 16 | use Peta::NN::Backend; | ||
| 17 | |||
| 18 | our $VERSION = '0.2610090'; | ||
| 19 | |||
| 20 | my $SATURATED = 20; # beyond this exp() adds nothing in a double | ||
| 21 | my $TINY = 1e-300; # keeps log() defined when a probability underflows | ||
| 22 | |||
| 23 | sub new ($class) { return bless {}, $class } | ||
| 24 | |||
| 25 | sub name ($self) { return 'plain' } | ||
| 26 | |||
| 27 | sub tensor ($self, $flat, $cols) { return [ [@$flat], $cols ] } | ||
| 28 | sub tokens ($self, $flat, $per_row) { return [ [@$flat], $per_row ] } | ||
| 29 | sub flat ($self, $tensor) { return [ @{ $tensor->[0] } ] } | ||
| 30 | sub zeros_like ($self, $tensor) { return [ [ (0.0) x @{ $tensor->[0] } ], $tensor->[1] ] } | ||
| 31 | |||
| 32 | sub affine ($self, $X, $W, $b) { | ||
| 33 | \my @x = $X->[0]; | ||
| 34 | \my @W = $W->[0]; | ||
| 35 | \my @b = $b->[0]; | ||
| 36 | my $n_in = $X->[1]; | ||
| 37 | my $n_out = @b; | ||
| 38 | my $batch = @x / $n_in; | ||
| 39 | my @y = (0.0) x ($batch * $n_out); | ||
| 40 | for my $s (0 .. $batch - 1) { | ||
| 41 | my $x_at = $s * $n_in; | ||
| 42 | my $y_at = $s * $n_out; | ||
| 43 | for my $j (0 .. $n_out - 1) { | ||
| 44 | my $sum = $b[$j]; | ||
| 45 | my $w_at = $j * $n_in; | ||
| 46 | for my $i (0 .. $n_in - 1) { | ||
| 47 | $sum += $W[ $w_at + $i ] * $x[ $x_at + $i ]; | ||
| 48 | } | ||
| 49 | $y[ $y_at + $j ] = $sum; | ||
| 50 | } | ||
| 51 | } | ||
| 52 | return [ \@y, $n_out ]; | ||
| 53 | } | ||
| 54 | |||
| 55 | sub affine_grad ($self, $X, $W, $dY, $need_dx) { | ||
| 56 | \my @x = $X->[0]; | ||
| 57 | \my @W = $W->[0]; | ||
| 58 | \my @dy = $dY->[0]; | ||
| 59 | my $n_in = $X->[1]; | ||
| 60 | my $n_out = $dY->[1]; | ||
| 61 | my $batch = @x / $n_in; | ||
| 62 | my @gW = (0.0) x @W; | ||
| 63 | my @gb = (0.0) x $n_out; | ||
| 64 | for my $s (0 .. $batch - 1) { | ||
| 65 | my $x_at = $s * $n_in; | ||
| 66 | my $y_at = $s * $n_out; | ||
| 67 | for my $j (0 .. $n_out - 1) { | ||
| 68 | my $d = $dy[ $y_at + $j ]; | ||
| 69 | my $w_at = $j * $n_in; | ||
| 70 | $gb[$j] += $d; | ||
| 71 | for my $i (0 .. $n_in - 1) { | ||
| 72 | $gW[ $w_at + $i ] += $d * $x[ $x_at + $i ]; | ||
| 73 | } | ||
| 74 | } | ||
| 75 | } | ||
| 76 | return ([ \@gW, $n_in ], [ \@gb, $n_out ]) if !$need_dx; | ||
| 77 | |||
| 78 | my @dx = (0.0) x @x; | ||
| 79 | for my $s (0 .. $batch - 1) { | ||
| 80 | my $x_at = $s * $n_in; | ||
| 81 | my $y_at = $s * $n_out; | ||
| 82 | for my $j (0 .. $n_out - 1) { | ||
| 83 | my $d = $dy[ $y_at + $j ]; | ||
| 84 | my $w_at = $j * $n_in; | ||
| 85 | for my $i (0 .. $n_in - 1) { | ||
| 86 | $dx[ $x_at + $i ] += $W[ $w_at + $i ] * $d; | ||
| 87 | } | ||
| 88 | } | ||
| 89 | } | ||
| 90 | return ([ \@gW, $n_in ], [ \@gb, $n_out ], [ \@dx, $n_in ]); | ||
| 91 | } | ||
| 92 | |||
| 93 | sub activate ($self, $kind, $X) { | ||
| 94 | \my @x = $X->[0]; | ||
| 95 | my @y = (0.0) x @x; | ||
| 96 | if ($kind eq 'relu') { | ||
| 97 | for my $i (0 .. $#x) { $y[$i] = $x[$i] > 0 ? $x[$i] : 0.0 } | ||
| 98 | } | ||
| 99 | elsif ($kind eq 'tanh') { | ||
| 100 | for my $i (0 .. $#x) { | ||
| 101 | my $v = $x[$i]; | ||
| 102 | if ($v > $SATURATED) { $y[$i] = 1.0 } | ||
| 103 | elsif ($v < -$SATURATED) { $y[$i] = -1.0 } | ||
| 104 | else { my $e = exp(2 * $v); $y[$i] = ($e - 1) / ($e + 1) } | ||
| 105 | } | ||
| 106 | } | ||
| 107 | elsif ($kind eq 'sigmoid') { | ||
| 108 | for my $i (0 .. $#x) { | ||
| 109 | my $v = $x[$i]; | ||
| 110 | $y[$i] = $v > $SATURATED ? 1.0 : $v < -$SATURATED ? 0.0 : 1 / (1 + exp(-$v)); | ||
| 111 | } | ||
| 112 | } | ||
| 113 | else { die "unknown activation '$kind'\n" } | ||
| 114 | return [ \@y, $X->[1] ]; | ||
| 115 | } | ||
| 116 | |||
| 117 | sub activate_grad ($self, $kind, $Y, $dY) { | ||
| 118 | \my @y = $Y->[0]; | ||
| 119 | \my @dy = $dY->[0]; | ||
| 120 | my @dx = (0.0) x @y; | ||
| 121 | if ($kind eq 'relu') { | ||
| 122 | for my $i (0 .. $#y) { $dx[$i] = $y[$i] > 0 ? $dy[$i] : 0.0 } | ||
| 123 | } | ||
| 124 | elsif ($kind eq 'tanh') { | ||
| 125 | for my $i (0 .. $#y) { $dx[$i] = $dy[$i] * (1 - $y[$i] * $y[$i]) } | ||
| 126 | } | ||
| 127 | elsif ($kind eq 'sigmoid') { | ||
| 128 | for my $i (0 .. $#y) { $dx[$i] = $dy[$i] * $y[$i] * (1 - $y[$i]) } | ||
| 129 | } | ||
| 130 | else { die "unknown activation '$kind'\n" } | ||
| 131 | return [ \@dx, $Y->[1] ]; | ||
| 132 | } | ||
| 133 | |||
| 134 | sub embed ($self, $E, $tokens) { | ||
| 135 | my ($table, $dim) = @$E; | ||
| 136 | my @y; | ||
| 137 | for my $token (@{ $tokens->[0] }) { | ||
| 138 | my $at = $token * $dim; | ||
| 139 | push @y, @$table[ $at .. $at + $dim - 1 ]; | ||
| 140 | } | ||
| 141 | return [ \@y, $tokens->[1] * $dim ]; | ||
| 142 | } | ||
| 143 | |||
| 144 | sub embed_grad ($self, $E, $tokens, $dX) { | ||
| 145 | \my @dx = $dX->[0]; | ||
| 146 | my $dim = $E->[1]; | ||
| 147 | my @gE = (0.0) x @{ $E->[0] }; | ||
| 148 | my $pos = 0; | ||
| 149 | for my $token (@{ $tokens->[0] }) { | ||
| 150 | my $at = $token * $dim; | ||
| 151 | for my $k (0 .. $dim - 1) { | ||
| 152 | $gE[ $at + $k ] += $dx[ $pos++ ]; | ||
| 153 | } | ||
| 154 | } | ||
| 155 | return [ \@gE, $dim ]; | ||
| 156 | } | ||
| 157 | |||
| 158 | # Cross entropy on softmax, per row; with both taken together the gradient | ||
| 159 | # is p - onehot. The row maximum is subtracted first so exp() stays small. | ||
| 160 | sub softmax_ce ($self, $logits, $classes, $weights = undef) { | ||
| 161 | \my @z = $logits->[0]; | ||
| 162 | my $cols = $logits->[1]; | ||
| 163 | my @d = (0.0) x @z; | ||
| 164 | my $loss = 0; | ||
| 165 | for my $s (0 .. $#$classes) { | ||
| 166 | my $at = $s * $cols; | ||
| 167 | my $max = $z[$at]; | ||
| 168 | for my $i (1 .. $cols - 1) { $max = $z[ $at + $i ] if $z[ $at + $i ] > $max } | ||
| 169 | my $sum = 0; | ||
| 170 | for my $i (0 .. $cols - 1) { | ||
| 171 | my $e = exp($z[ $at + $i ] - $max); | ||
| 172 | $d[ $at + $i ] = $e; | ||
| 173 | $sum += $e; | ||
| 174 | } | ||
| 175 | for my $i (0 .. $cols - 1) { $d[ $at + $i ] /= $sum } | ||
| 176 | my $target = $at + $classes->[$s]; | ||
| 177 | my $weight = $weights ? $weights->[$s] : 1; | ||
| 178 | $loss -= $weight * log($d[$target] + $TINY); | ||
| 179 | $d[$target] -= 1; | ||
| 180 | next if $weight == 1; | ||
| 181 | for my $i (0 .. $cols - 1) { $d[ $at + $i ] *= $weight } | ||
| 182 | } | ||
| 183 | return ($loss, [ \@d, $cols ]); | ||
| 184 | } | ||
| 185 | |||
| 186 | # Mean squared error per row, summed over the rows. | ||
| 187 | sub mse ($self, $out, $targets, $weights = undef) { | ||
| 188 | \my @o = $out->[0]; | ||
| 189 | my $cols = $out->[1]; | ||
| 190 | my @d = (0.0) x @o; | ||
| 191 | my $loss = 0; | ||
| 192 | for my $i (0 .. $#o) { | ||
| 193 | my $diff = $o[$i] - $targets->[$i]; | ||
| 194 | my $weight = $weights ? $weights->[ int($i / $cols) ] : 1; | ||
| 195 | $loss += $weight * $diff * $diff; | ||
| 196 | $d[$i] = $weight * 2 * $diff / $cols; | ||
| 197 | } | ||
| 198 | return ($loss / $cols, [ \@d, $cols ]); | ||
| 199 | } | ||
| 200 | |||
| 201 | # Multiply every value by $factor, in place: weight decay. | ||
| 202 | sub decay ($self, $P, $factor) { | ||
| 203 | \my @p = $P->[0]; | ||
| 204 | for my $i (0 .. $#p) { $p[$i] *= $factor } | ||
| 205 | return; | ||
| 206 | } | ||
| 207 | |||
| 208 | sub sgd_update ($self, $P, $G, $V, $lr, $momentum, $scale) { | ||
| 209 | \my @p = $P->[0]; | ||
| 210 | \my @g = $G->[0]; | ||
| 211 | \my @v = $V->[0]; | ||
| 212 | for my $i (0 .. $#p) { | ||
| 213 | $v[$i] = $momentum * $v[$i] - $lr * $scale * $g[$i]; | ||
| 214 | $p[$i] += $v[$i]; | ||
| 215 | } | ||
| 216 | return; | ||
| 217 | } | ||
| 218 | |||
| 219 | sub adam_update ($self, $P, $G, $M, $V, $rate, $b1, $b2, $eps, $scale) { | ||
| 220 | \my @p = $P->[0]; | ||
| 221 | \my @g = $G->[0]; | ||
| 222 | \my @m = $M->[0]; | ||
| 223 | \my @v = $V->[0]; | ||
| 224 | for my $i (0 .. $#p) { | ||
| 225 | my $grad = $scale * $g[$i]; | ||
| 226 | $m[$i] = $b1 * $m[$i] + (1 - $b1) * $grad; | ||
| 227 | $v[$i] = $b2 * $v[$i] + (1 - $b2) * $grad * $grad; | ||
| 228 | $p[$i] -= $rate * $m[$i] / (sqrt($v[$i]) + $eps); | ||
| 229 | } | ||
| 230 | return; | ||
| 231 | } | ||
| 232 | |||
| 233 | 1; | ||
| 234 | |||
| 235 | __END__ |