blib/lib/Peta/NN/Backend/Plain.pm untested

filter: Statementstmt 0.0% 0 / 195 Branchbran 0.0% 0 / 34 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 22 Podpod 0.0% 0 / 17 totaltotal 0.0% 0 / 268 CC 18 SCAR 33.4
CC
18
Cov
0%
activate
90.0
activate_grad
30.0
softmax_ce
20.0
CRAP 28.2
1 package Peta::NN::Backend::Plain;
2 # ABSTRACT: Peta::NN on plain Perl arrays
3  
4 # The reference backend: a tensor is [ \@numbers, $cols ], row after row, and
5 # every operation is a plain Perl loop.
6 #
7 # Each kernel first aliases the tensors' arrays to lexical arrays (\my @W =
8 # ...). The loops then read as ordinary array code, which is also the form
9 # pperl's JIT compiles; through the references themselves it does not.
10  
11 use v5.36;
12 use feature 'refaliasing';
13 no warnings 'experimental::refaliasing';
14  
15 use parent -norequire, 'Peta::NN::Backend';
16 use Peta::NN::Backend;
17  
18 our $VERSION = '0.2610090';
19  
20 my $SATURATED = 20; # beyond this exp() adds nothing in a double
21 my $TINY = 1e-300; # keeps log() defined when a probability underflows
22  
23 sub new ($class) { return bless {}, $class }
24  
25 sub name ($self) { return 'plain' }
26  
27 sub tensor ($self, $flat, $cols) { return [ [@$flat], $cols ] }
28 sub tokens ($self, $flat, $per_row) { return [ [@$flat], $per_row ] }
29 sub flat ($self, $tensor) { return [ @{ $tensor->[0] } ] }
30 sub zeros_like ($self, $tensor) { return [ [ (0.0) x @{ $tensor->[0] } ], $tensor->[1] ] }
31  
32 sub affine ($self, $X, $W, $b) {
33 \my @x = $X->[0];
34 \my @W = $W->[0];
35 \my @b = $b->[0];
36 my $n_in = $X->[1];
37 my $n_out = @b;
38 my $batch = @x / $n_in;
39 my @y = (0.0) x ($batch * $n_out);
40 for my $s (0 .. $batch - 1) {
41 my $x_at = $s * $n_in;
42 my $y_at = $s * $n_out;
43 for my $j (0 .. $n_out - 1) {
44 my $sum = $b[$j];
45 my $w_at = $j * $n_in;
46 for my $i (0 .. $n_in - 1) {
47 $sum += $W[ $w_at + $i ] * $x[ $x_at + $i ];
48 }
49 $y[ $y_at + $j ] = $sum;
50 }
51 }
52 return [ \@y, $n_out ];
53 }
54  
55 sub affine_grad ($self, $X, $W, $dY, $need_dx) {
56 \my @x = $X->[0];
57 \my @W = $W->[0];
58 \my @dy = $dY->[0];
59 my $n_in = $X->[1];
60 my $n_out = $dY->[1];
61 my $batch = @x / $n_in;
62 my @gW = (0.0) x @W;
63 my @gb = (0.0) x $n_out;
64 for my $s (0 .. $batch - 1) {
65 my $x_at = $s * $n_in;
66 my $y_at = $s * $n_out;
67 for my $j (0 .. $n_out - 1) {
68 my $d = $dy[ $y_at + $j ];
69 my $w_at = $j * $n_in;
70 $gb[$j] += $d;
71 for my $i (0 .. $n_in - 1) {
72 $gW[ $w_at + $i ] += $d * $x[ $x_at + $i ];
73 }
74 }
75 }
76 return ([ \@gW, $n_in ], [ \@gb, $n_out ]) if !$need_dx;
77  
78 my @dx = (0.0) x @x;
79 for my $s (0 .. $batch - 1) {
80 my $x_at = $s * $n_in;
81 my $y_at = $s * $n_out;
82 for my $j (0 .. $n_out - 1) {
83 my $d = $dy[ $y_at + $j ];
84 my $w_at = $j * $n_in;
85 for my $i (0 .. $n_in - 1) {
86 $dx[ $x_at + $i ] += $W[ $w_at + $i ] * $d;
87 }
88 }
89 }
90 return ([ \@gW, $n_in ], [ \@gb, $n_out ], [ \@dx, $n_in ]);
91 }
92  
93 sub activate ($self, $kind, $X) {
94 \my @x = $X->[0];
95 my @y = (0.0) x @x;
96 if ($kind eq 'relu') {
97 for my $i (0 .. $#x) { $y[$i] = $x[$i] > 0 ? $x[$i] : 0.0 }
98 }
99 elsif ($kind eq 'tanh') {
100 for my $i (0 .. $#x) {
101 my $v = $x[$i];
102 if ($v > $SATURATED) { $y[$i] = 1.0 }
103 elsif ($v < -$SATURATED) { $y[$i] = -1.0 }
104 else { my $e = exp(2 * $v); $y[$i] = ($e - 1) / ($e + 1) }
105 }
106 }
107 elsif ($kind eq 'sigmoid') {
108 for my $i (0 .. $#x) {
109 my $v = $x[$i];
110 $y[$i] = $v > $SATURATED ? 1.0 : $v < -$SATURATED ? 0.0 : 1 / (1 + exp(-$v));
111 }
112 }
113 else { die "unknown activation '$kind'\n" }
114 return [ \@y, $X->[1] ];
115 }
116  
117 sub activate_grad ($self, $kind, $Y, $dY) {
118 \my @y = $Y->[0];
119 \my @dy = $dY->[0];
120 my @dx = (0.0) x @y;
121 if ($kind eq 'relu') {
122 for my $i (0 .. $#y) { $dx[$i] = $y[$i] > 0 ? $dy[$i] : 0.0 }
123 }
124 elsif ($kind eq 'tanh') {
125 for my $i (0 .. $#y) { $dx[$i] = $dy[$i] * (1 - $y[$i] * $y[$i]) }
126 }
127 elsif ($kind eq 'sigmoid') {
128 for my $i (0 .. $#y) { $dx[$i] = $dy[$i] * $y[$i] * (1 - $y[$i]) }
129 }
130 else { die "unknown activation '$kind'\n" }
131 return [ \@dx, $Y->[1] ];
132 }
133  
134 sub embed ($self, $E, $tokens) {
135 my ($table, $dim) = @$E;
136 my @y;
137 for my $token (@{ $tokens->[0] }) {
138 my $at = $token * $dim;
139 push @y, @$table[ $at .. $at + $dim - 1 ];
140 }
141 return [ \@y, $tokens->[1] * $dim ];
142 }
143  
144 sub embed_grad ($self, $E, $tokens, $dX) {
145 \my @dx = $dX->[0];
146 my $dim = $E->[1];
147 my @gE = (0.0) x @{ $E->[0] };
148 my $pos = 0;
149 for my $token (@{ $tokens->[0] }) {
150 my $at = $token * $dim;
151 for my $k (0 .. $dim - 1) {
152 $gE[ $at + $k ] += $dx[ $pos++ ];
153 }
154 }
155 return [ \@gE, $dim ];
156 }
157  
158 # Cross entropy on softmax, per row; with both taken together the gradient
159 # is p - onehot. The row maximum is subtracted first so exp() stays small.
160 sub softmax_ce ($self, $logits, $classes, $weights = undef) {
161 \my @z = $logits->[0];
162 my $cols = $logits->[1];
163 my @d = (0.0) x @z;
164 my $loss = 0;
165 for my $s (0 .. $#$classes) {
166 my $at = $s * $cols;
167 my $max = $z[$at];
168 for my $i (1 .. $cols - 1) { $max = $z[ $at + $i ] if $z[ $at + $i ] > $max }
169 my $sum = 0;
170 for my $i (0 .. $cols - 1) {
171 my $e = exp($z[ $at + $i ] - $max);
172 $d[ $at + $i ] = $e;
173 $sum += $e;
174 }
175 for my $i (0 .. $cols - 1) { $d[ $at + $i ] /= $sum }
176 my $target = $at + $classes->[$s];
177 my $weight = $weights ? $weights->[$s] : 1;
178 $loss -= $weight * log($d[$target] + $TINY);
179 $d[$target] -= 1;
180 next if $weight == 1;
181 for my $i (0 .. $cols - 1) { $d[ $at + $i ] *= $weight }
182 }
183 return ($loss, [ \@d, $cols ]);
184 }
185  
186 # Mean squared error per row, summed over the rows.
187 sub mse ($self, $out, $targets, $weights = undef) {
188 \my @o = $out->[0];
189 my $cols = $out->[1];
190 my @d = (0.0) x @o;
191 my $loss = 0;
192 for my $i (0 .. $#o) {
193 my $diff = $o[$i] - $targets->[$i];
194 my $weight = $weights ? $weights->[ int($i / $cols) ] : 1;
195 $loss += $weight * $diff * $diff;
196 $d[$i] = $weight * 2 * $diff / $cols;
197 }
198 return ($loss / $cols, [ \@d, $cols ]);
199 }
200  
201 # Multiply every value by $factor, in place: weight decay.
202 sub decay ($self, $P, $factor) {
203 \my @p = $P->[0];
204 for my $i (0 .. $#p) { $p[$i] *= $factor }
205 return;
206 }
207  
208 sub sgd_update ($self, $P, $G, $V, $lr, $momentum, $scale) {
209 \my @p = $P->[0];
210 \my @g = $G->[0];
211 \my @v = $V->[0];
212 for my $i (0 .. $#p) {
213 $v[$i] = $momentum * $v[$i] - $lr * $scale * $g[$i];
214 $p[$i] += $v[$i];
215 }
216 return;
217 }
218  
219 sub adam_update ($self, $P, $G, $M, $V, $rate, $b1, $b2, $eps, $scale) {
220 \my @p = $P->[0];
221 \my @g = $G->[0];
222 \my @m = $M->[0];
223 \my @v = $V->[0];
224 for my $i (0 .. $#p) {
225 my $grad = $scale * $g[$i];
226 $m[$i] = $b1 * $m[$i] + (1 - $b1) * $grad;
227 $v[$i] = $b2 * $v[$i] + (1 - $b2) * $grad * $grad;
228 $p[$i] -= $rate * $m[$i] / (sqrt($v[$i]) + $eps);
229 }
230 return;
231 }
232  
233 1;
234  
235 __END__