Nerl/Network.pm

filter: Statementstmt 61.1% 96 / 157 Branchbran 30.0% 9 / 30 Conditioncond 21.4% 3 / 14 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 63.1% 12 / 19 Podpod 27.2% 3 / 11 totaltotal 51.4% 123 / 239 CC 25 SCAR 24.2
CC
25
Cov
54%
USE_G2D
30.0
show_neuron
30.0
append_l2
12.0
CRAP 11.2
« Nerl.pm ↑ Summary
1 package AI::Nerl::Network;
2 2 ▶use Moose 'has', inner => { -as => 'moose_inner' };
Subroutine BEGIN: called CC=1 SCAR=0.0
3 2 ▶use PDL;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 2 ▶use PDL::NiceSlice;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 2 ▶use PDL::Constants 'E';
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 # ABSTRACT: 3-layer Neural network on PDL with backpropagation
8 #
9 2 my $DEBUG=0;
10  
11 # http://ufldl.stanford.edu/wiki/index.php/Backpropagation_Algorithm
12 # http://www.stanford.edu/class/cs294a/sparseAutoencoder_2011new.pdf
13  
14  
15 =head1 NAME
16  
17 AI::Nerl::Network - 3-layer neural network with backpropagation
18  
19 =head1 SYNOPSIS
20  
21 use AI::Nerl::Network;
22 use PDL;
23
24 my $x = pdl([0,0,1,1],
25 [0,1,0,1],
26 [1,0,0,1]);
27 my $y = pdl([1,1,0,1]);
28 my $nn = AI::Nerl::Network->new(
29 l1 => 3, # 3 inputs
30 l2 => 18, # 18 hidden neurons
31 l3 => 1, # 1 output
32 alpha => .3, # learning rate
33 lambda => .01, # 'squashing' parameter
34 );
35 $nn->train($x,$y, passes=>45);
36 my ($cost,$num_correct) = $nn->cost($x,$y);
37 #$nn wasn't programmed with this input. could be anything:
38 print $nn->run(pdl([0,0,0]));
39  
40 =head1 DESCRIPTION
41  
42 =head1 METHODS
43  
44 =head2 train($x,$y, %params)
45  
46 Train with backpropagation using $x as input & $y as target.
47 $x and $y are both pdls. If there are multiple cases, each one will
48 occupy a column (dimension 2) of the pdl. If your dimensions are off,
49 you will experience an pdl error of some sort.
50  
51 =head3 %params
52  
53 =head4 passes
54  
55 number of passes.
56  
57 =head2 run($x)
58  
59 $output = $nn->run($x);
60  
61 =head2 cost($x,$y)
62  
63 ($cost,$num_correct) = $nn->cost($x,$y);
64  
65 Calculate the 'cost' of the network. This is basically the difference between the
66 actual output ($nn->run($x)) and the the target output($y), added to the sum of
67 the neural weights if you're penalizing weights with lambda. The cost should
68 B<Always> decrease after being trained with ($x,$y).
69  
70 This function returns both the cost, and the number of "correct" responses
71 if using output neurons for classification.
72  
73 =head1 SEE ALSO
74  
75 L<http://en.wikipedia.org/wiki/Feedforward_neural_network#Multi-layer_perceptron>
76  
77 L<http://en.wikipedia.org/wiki/Backpropagation>
78  
79 =head1 AUTHOR
80  
81 Zach Morgan C<< <zpmorgan@gmail.com> >>
82  
83  
84 =head1 COPYRIGHT
85  
86 Copyright 2012 by Zach Morgan
87  
88 This package is free software; you can redistribute it and/or modify it under the
89 same terms as Perl itself.
90  
91 =cut
92  
93 # Simple nn with 1 hidden layer
94 # train with $nn->train(data,labels);
95  
96 2 has scale_input => (
97 is => 'ro',
98 required => 0,
99 isa => 'Num',
100 default => 0,
101 );
102  
103 # number of input,hidden,output neurons
104 2 has [qw/ l1 l2 l3 /] => (
105 is => 'ro',
106 isa => 'Int',
107 );
108  
109 2 has theta1 => (
110 is => 'ro',
111 isa => 'PDL',
112 lazy => 1,
113 builder => '_mk_theta1',
114 );
115 2 has theta2 => (
116 is => 'ro',
117 isa => 'PDL',
118 lazy => 1,
119 builder => '_mk_theta2',
120 );
121  
122 2 has b1 => (
123 is => 'ro',
124 isa => 'PDL',
125 lazy => 1,
126 builder => '_mk_b1',
127 );
128 2 has b2 => (
129 is => 'ro',
130 isa => 'PDL',
131 lazy => 1,
132 builder => '_mk_b2',
133 );
134 2 has alpha => ( #learning rate
135 isa => 'Num',
136 is => 'rw',
137 default => .6,
138 );
139 2 has lambda => (
140 isa => 'Num',
141 is => 'rw',
142 default => .01,
143 );
144  
145 6 ▶sub _mk_theta1{
Subroutine _mk_theta1: called CC=1 SCAR=0.0
146 6 my $self = shift;
147 6 return grandom($self->l1, $self->l2) * .01;
148 }
149 6 ▶sub _mk_theta2{
Subroutine _mk_theta2: called CC=1 SCAR=0.0
150 6 my $self = shift;
151 6 return grandom($self->l2, $self->l3) * .01;
152 }
153 6 ▶sub _mk_b1{
Subroutine _mk_b1: called CC=1 SCAR=0.0
154 6 my $self = shift;
155 6 return grandom($self->l2) * .01;
156 }
157 6 ▶sub _mk_b2{
Subroutine _mk_b2: called CC=1 SCAR=0.0
158 6 my $self = shift;
159 6 return grandom($self->l3) * .01;
160 }
161  
162  
163 705 ▶sub train{
Subroutine train: called CC=9 SCAR=23.4
Pod documented
164 705 my ($self,$x,$y, %params) = @_;
165 705 $x->sever();
166 705 ▶ my $passes = $params{passes} // 10;
Condition50%1/2
$params{"passes"} // 10
l!l
7050
MC/DC0%0/1
$params{"passes"} // 10
$params{"passes"}
Truth table50%1/2
A
A= $params{"passes"}
Aresult
11
00
167  
168 705 ▶ if ($self->scale_input){
Branch: 1 branch
BranchTrueFalseTotal
if ($self->scale_input) 0 705 705
169 0 $x *= $self->scale_input;
170 }
171 705 my $num_examples = $x->dim(0);
172  
173 705 for my $pass (1..$passes){
174 # warn 'blah:'. $self->theta1->slice(':,2')->flat->sum;
175 1497 ▶ show784($self->theta1->slice(':,0')) if $pass%30==29 and $DEBUG;
Branch: 1 branch
BranchTrueFalseTotal
if $pass % 30 == 29 and $DEBUG 0 1497 1497
Condition66%2/3
$pass % 30 == 29 and $DEBUG
!ll&&!rl&&r
1473240
MC/DC0%0/2
$pass % 30 == 29 and $DEBUG
$pass % 30 == 29 $DEBUG
Truth table66%2/3
A and B
A= $pass % 30 == 29B= $DEBUG
ABresult
0X0
100
111
176 1497 my $delta1 = $self->theta1->copy * 0;
177 1497 my $delta2 = $self->theta2->copy * 0;
178 1497 my $deltab1 = $self->b1->copy * 0;
179 1497 my $deltab2 = $self->b2->copy * 0;
180  
181 #iterate over examples :(
182 1497 for my $i (0..$num_examples-1){
183 4558 my $a1 = $x(($i));
184 4558 my $z2 = ($self->theta1 x $a1->transpose)->squeeze;
185 4558 $z2 += $self->b1; #add bias.
186 4558 my $a2 = $z2->tanh;
187 4558 my $z3 = ($self->theta2 x $a2->transpose)->squeeze;
188 4558 $z3 += $self->b2; #add bias.
189 4558 my $a3 = $z3->tanh;
190 # warn $y(($i)) - $a3;
191 4558 my $d3= -($y(($i)) - $a3) * tanhxderivative($a3);
192 #warn $d3;
193 4558 $delta2 += $d3->transpose x $a2;
194 4558 my $d2 = ($self->theta2->transpose x $d3->transpose)->squeeze * tanhxderivative($a2);
195 4558 $delta1 += $d2->transpose x $a1;
196 #warn $delta2(4);
197 4558 $deltab1 += $d2;
198 4558 $deltab2 += $d3;
199  
200 4558 ▶ if($DEBUG==1){
Branch: 1 branch
BranchTrueFalseTotal
if ($DEBUG == 1) 0 4558 4558
201 0 warn "z2: $z2\n$z3: $z3\n";
202 0 warn "d3:$d3\n";
203 }
204 }
205 #warn $deltab1;
206 1497 ▶ if($DEBUG==1){
Branch: 1 branch
BranchTrueFalseTotal
if ($DEBUG == 1) 0 1497 1497
207 0 warn "theta1: ". $self->theta1;#/ $num_examples;
208 0 warn "theta2: ". $self->theta2;
209 0 warn "delta1: $delta1\n";
210 0 warn "delta2: $delta2\n";
211 }
212 1497 $self->{theta2} -= $self->alpha * ($delta2 / $num_examples + $self->theta2 * $self->lambda);
213 1497 $self->{theta1} -= $self->alpha * ($delta1 / $num_examples + $self->theta1 * $self->lambda);
214 1497 $self->{b1} -= $self->alpha * $deltab1 / $num_examples;
215 1497 $self->{b2} -= $self->alpha * $deltab2 / $num_examples;
216 }
217 }
218  
219 3 ▶sub run{
Subroutine run: called CC=3 SCAR=11.2
Pod documented
220 3 my ($self,$x) = @_;
221 3 $x->sever();
222 3 ▶ if ($self->scale_input){
Branch: 1 branch
BranchTrueFalseTotal
if ($self->scale_input) 0 3 3
223 0 $x *= $self->scale_input;
224 }
225  
226 3 ▶ $x = $x->transpose if $self->l1 != $x->dim(1);
Branch: 1 branch
BranchTrueFalseTotal
if $self->l1 != $x->dim(1) 0 3 3
227 3 my $y = $self->theta1 x $x;
228 3 $y += $self->b1->transpose;
229 3 $y->inplace()->tanh;# = tanhx($y);
230  
231 3 $y = $self->theta2 x $y;
232 3 $y += $self->b2->transpose;
233 3 $y->inplace()->tanh();# = tanhx($y);
234 3 return $y;
235 }
236  
237 0 ▶sub append_l2{
Subroutine append_l2: not called CC=3 SCAR=24.8
Pod undocumented
238 0 my ($self,$x) = @_;
239 0 $x->sever();
240 0 ▶ if ($self->scale_input){
Branch: 1 branch
BranchTrueFalseTotal
if ($self->scale_input) 0 0 0
241 0 $x *= $self->scale_input;
242 }
243 0 ▶ $x = $x->transpose if $self->l1 != $x->dim(1);
Branch: 1 branch
BranchTrueFalseTotal
if $self->l1 != $x->dim(1) 0 0 0
244 0 my $l2 = $self->theta1 x $x;
245 0 $l2 += $self->b1->transpose;
246 0 $l2->inplace()->tanh;
247 # warn join ',',$x->dims;
248 # warn join ',',$l2->dims;
249 0 return $x->glue(1,$l2);
250 }
251  
252 703 ▶sub cost{
Subroutine cost: called CC=4 SCAR=13.9
Pod documented
253 703 my ($self,$x,$y) = @_;
254 703 $x->sever();# = $x->copy();
255 703 my $n = $x->dim(0);
256 703 ▶ if ($self->scale_input){
Branch: 1 branch
BranchTrueFalseTotal
if ($self->scale_input) 0 703 703
257 0 $x *= $self->scale_input;
258 }
259 703 my $num_correct = 0;
260 #die join(',',$x->dims) .',,,'. join(',',$y->dims);
261 703 my $total_cost = 0;
262 703 for my $i (0..$n-1){
263 1412 my $a1 = $x(($i));
264 1412 my $z2 = ($self->theta1 x $a1->transpose)->squeeze;
265 1412 $z2 += $self->b1;
266 1412 my $a2 = $z2->tanh();
267 1412 my $z3 = ($self->theta2 x $a2->transpose)->squeeze;
268 1412 $z3 += $self->b2;
269 1412 my $a3 = $z3->tanh;
270 1412 $total_cost += ($y(($i))-$a3)->abs()->power(2,0)->sum()/2;
271 #warn $a3->maximum_ind . ' ' . $y(($i))->maximum_ind;;
272 1412 ▶ $num_correct++ if $a3->maximum_ind == $y(($i))->maximum_ind;
Branch: 1 branch
BranchTrueFalseTotal
if $a3->maximum_ind == $y->slice([$i, 0, 0])->maximum_ind 1410 2 1412
273 }
274 703 $total_cost /= $n;
275 703 $total_cost += $self->theta1->flat->power(2,0)->sum * $self->lambda;
276 703 $total_cost += $self->theta2->flat->power(2,0)->sum * $self->lambda;
277 703 return ($total_cost, $num_correct);
278 }
279  
280 0 ▶sub tanhx{ #don't use this. pdl has $pdl->tanh which can be used in place.
Subroutine tanhx: not called CC=1 SCAR=6.9
Pod undocumented
281 0 my $foo = shift;
282 0 my $p = E**$foo;
283 0 my $n = E**-$foo;
284 0 return (($p-$n)/($p+$n));
285 }
286 9116 ▶sub tanhxderivative{ #use: tanhxderivative($pdl->tanh()). save time by finding tanh first.
Subroutine tanhxderivative: called CC=1 SCAR=0.0
Pod undocumented
287 9116 my $tanhx = shift;
288 9116 return (1 - $tanhx**2);
289 }
290  
291 0 ▶sub sigmoid{
Subroutine sigmoid: not called CC=1 SCAR=6.9
Pod undocumented
292 0 my $foo = shift;
293 0 return 1/(1+E**-$foo);
294 }
295  
296 0 ▶sub logistic{
Subroutine logistic: not called CC=1 SCAR=6.9
Pod undocumented
297 #find sigmoid before calling this.
298 #grad=logistic(sigmoid(foo))
299 0 my $foo = shift;
300 0 return $foo * (1-$foo);
301 }
302  
303 2 my $g2d_tried = 0;
304 2 my $g2d_failed = '';
305 0 ▶sub USE_G2D{
Subroutine USE_G2D: not called CC=5 SCAR=34.0
Pod undocumented
306 0 ▶ return 0 if $g2d_tried and $g2d_failed;
Branch: 1 branch
BranchTrueFalseTotal
if $g2d_tried and $g2d_failed 0 0 0
Condition0%0/3
$g2d_tried and $g2d_failed
!ll&&!rl&&r
000
MC/DC0%0/2
$g2d_tried and $g2d_failed
$g2d_tried $g2d_failed
Truth table0%0/3
A and B
A= $g2d_triedB= $g2d_failed
ABresult
0X0
100
111
307 0 ▶ return 1 if $g2d_tried;
Branch: 1 branch
BranchTrueFalseTotal
if $g2d_tried 0 0 0
308 eval{
309 0 require PDL::Graphics2D;
310 0 PDL::Graphics2d->import('imag2d');
311 0 1;
312 0 ▶ } or do {
Branch: 1 branch
BranchTrueFalseTotal
eval { do { require PDL::Graphics2D; "PDL::Graphics2d"->import("imag2d"); 1 } } or do { $g2d_failed = $@; warn "PDL::Graphics2d failed to load. perhaps $g2d_failed" } 0 0 0
313 0 $g2d_failed = $@;
314 0 warn "PDL::Graphics2d failed to load. perhaps $g2d_failed";
315 };
316 0 $g2d_tried = 1;
317 0 return USE_G2D();
318 }
319 #display 28x28 grayscale pdl.
320 0 ▶sub show784{
Subroutine show784: not called CC=2 SCAR=17.9
Pod undocumented
321 0 ▶ return unless USE_G2D();
Branch: 1 branch
BranchTrueFalseTotal
unless USE_G2D() 0 0 0
322 0 my $w = shift;
323 0 $w = $w->copy;
324 #warn join',', $w->dims;
325 0 $w = $w->squeeze;
326 0 my $min = $w->minimum;
327 0 $w -= $min;
328 0 my $max = $w->maximum;
329 0 $w /= $max;
330 0 $w = $w->reshape(28,28);
331 0 imag2d $w;
332 }
333  
334 0 ▶sub show_neuron{
Subroutine show_neuron: not called CC=5 SCAR=34.0
Pod undocumented
335 0 ▶ unless (USE_G2D()){
Branch: 1 branch
BranchTrueFalseTotal
unless (USE_G2D()) 0 0 0
336 0 warn 'Can\'t display neuron. Get OpenGL?';
337 0 return;
338 }
339 0 my $self = shift;
340 0 ▶ my $n = shift // 0;
Condition0%0/2
shift() // 0
l!l
00
MC/DC0%0/1
shift() // 0
shift()
Truth table0%0/2
A
A= shift()
Aresult
11
00
341 0 ▶ my $x = shift || 28;
Condition0%0/2
shift() || 28
l!l
00
MC/DC0%0/1
shift() || 28
shift()
Truth table0%0/2
A
A= shift()
Aresult
11
00
342 0 ▶ my $y = shift || 28;
Condition0%0/2
shift() || 28
l!l
00
MC/DC0%0/1
shift() || 28
shift()
Truth table0%0/2
A
A= shift()
Aresult
11
00
343 0 my $w = $self->theta1->slice(":,$n")->copy;
344 0 $w = $w->squeeze;
345 0 my $min = $w->minimum;
346 0 $w -= $min;
347 0 my $max = $w->maximum;
348 0 $w /= $max;
349 0 $w = $w->reshape($x,$y);
350 0 PDL::Graphics2D::imag2d $w;
351 }
352  
353 2 '$nn->train($sovietRussian)';
354  
« Nerl.pm ↑ Summary