lib/Peta/NN/Pipeline.pm

filter: Statementstmt 100% 121 / 121 Branchbran 92.3% 48 / 52 Conditioncond 87.5% 28 / 32 MC/DCmcdc 70.5% 12 / 17 Subroutinesub 100% 11 / 11 Podpod 100% 8 / 8 totaltotal 94.6% 228 / 241 CC 53 SCAR 31.4
CC
53
Cov
96%
new
32.4
_run
20.0
run_texts
2.0
CRAP 23.0
« blib/lib/Peta/NN/Backend/Plain.pm ↑ Summary blib/lib/Peta/NN/Optimizer.pm »
1 package Peta::NN::Pipeline;
2 # ABSTRACT: models in series, and routed by a classifier
3  
4 # Models put together: in series, where each step's answer is the next step's
5 # input (Apfel -> Äpfel -> Äpfeln), and with routing, where a classifying
6 # step's answer decides which model a later step uses.
7 #
8 # The models stay what they are: each does its one limited transformation and
9 # knows nothing of the others, or of what its parameters mean. What a chain
10 # means is written here, in the pipeline's steps, by whoever builds it.
11 #
12 # This is part of the inference leg: it needs Peta::NN::Inference and the
13 # model files, nothing of the training leg.
14  
15 5 ▶use v5.36;
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 5 ▶use Peta::NN::Inference;
Subroutine BEGIN: called CC=1 SCAR=0.0
18  
19 5 our $VERSION = '0.2610090';
20  
21 5 my $ANY = '*'; # in a routing table: the model for every answer not listed
22  
23 # models { name => model }: a model file's path, or an object that answers
24 # (Peta::NN::Inference, or the training leg's Peta::NN::Model)
25 # steps the steps, in order; each is a table with
26 # model the name of a model, or a routing table
27 # { step => { answer => model name, ... } }: the model is chosen by
28 # the answer of the earlier step called `step`; '*' stands for any
29 # answer not listed
30 # params the step's parameters, each one of
31 # 'text' that value, always
32 # \N the Nth argument of the call (0 is the first)
33 # { answer => step } the answer of the earlier step called `step`
34 # name a name for the step, so that later steps can refer to its answer
35 # classify true if the step only looks: its answer is recorded under its
36 # name, and the string goes on to the next step unchanged
37 # pool true if a classifying step answers once for all strings of a
38 # text together (run_texts): every word is evidence, and the
39 # text gets the one answer they point to
40 60 ▶sub new ($class, %arg) {
Subroutine new: called CC=32 SCAR=34.8
Pod documented
41 60 ▶ my $models = $arg{models} // die "models => { name => model, ... } is required\n";
Condition50%1/2
$arg{"models"} // die("models => { name => model, ... } is required\n")
l!l
600
MC/DC0%0/1
$arg{"models"} // die("models => { name => model, ... } is required\n")
$arg{"models"}
Truth table50%1/2
A
A= $arg{"models"}
Aresult
11
00
42 60 ▶ my $steps = $arg{steps} // die "steps => [ ... ] is required\n";
Condition50%1/2
$arg{"steps"} // die("steps => [ ... ] is required\n")
l!l
600
MC/DC0%0/1
$arg{"steps"} // die("steps => [ ... ] is required\n")
$arg{"steps"}
Truth table50%1/2
A
A= $arg{"steps"}
Aresult
11
00
43 60 ▶ die "a pipeline needs at least one step\n" if !@$steps;
Branch: 1 branch
BranchTrueFalseTotal
unless @$steps 1 59 60
44  
45 my %model = map {
46 59 my $model = $models->{$_};
47 270 ▶ $_ => !ref $model ? Peta::NN::Inference->load($model)
Branch: 2 branches
BranchTrueFalseTotal
!ref($model) ? : 4 266 270
$model->can('inference') ? : 221 45 266
48 : $model->can('inference') ? $model->inference
49 : $model
50 } keys %$models;
51  
52 59 my (%named, $arguments);
53 59 my $self = bless { model => \%model, steps => [] }, $class;
54 59 for my $n (0 .. $#$steps) {
55 119 my %step = %{ $steps->[$n] };
56 119 my $where = 'step ' . ($n + 1);
57 119 ▶ my $spec = $step{model} // die "$where names no model\n";
Condition100%2/2
$step{"model"} // die("$where names no model\n")
l!l
1181
MC/DC100%1/1
$step{"model"} // die("$where names no model\n")
$step{"model"}
Truth table100%2/2
A
A= $step{"model"}
Aresult
11
00
58  
59 # Every model a step may use must exist, and a routing step must
60 # route by a step that came before it.
61 118 my @used = $spec;
62 118 ▶ if (ref $spec eq 'HASH') {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $spec eq "HASH") 19 99 118
63 19 my ($by, @more) = keys %$spec;
64 19 ▶ die "$where routes by more than one step\n" if @more;
Branch: 1 branch
BranchTrueFalseTotal
if @more 0 19 19
65 19 ▶ die "$where routes by '$by', which is not an earlier step\n" if !$named{$by};
Branch: 1 branch
BranchTrueFalseTotal
unless $named{$by} 1 18 19
66 18 @used = values %{ $spec->{$by} };
67 }
68 117 ▶ defined $model{$_} or die "$where uses the model '$_', which the pipeline does not have\n" for @used;
Branch: 1 branch
BranchTrueFalseTotal
defined $model{$_} or die "$where uses the model '${_}', which the pipeline does not have\n" 133 2 135
69  
70 115 ▶ for my $param (@{ $step{params} // [] }) {
Condition100%2/2
$step{"params"} // []
l!l
5857
MC/DC100%1/1
$step{"params"} // []
$step{"params"}
Truth table100%2/2
A
A= $step{"params"}
Aresult
11
00
71 34 ▶ if (ref $param eq 'HASH') {
Branch: 3 branches
BranchTrueFalseTotal
if (ref $param eq 'HASH') { } 4 30 34
elsif (ref $param eq 'SCALAR') { } 23 7 30
elsif (ref $param) { } 1 6 7
72 4 ▶ my $from = $param->{answer} // die "$where has a parameter that is a table without `answer`\n";
Condition50%1/2
$param->{"answer"} // die("$where has a parameter that is a table without `answer`\n")
l!l
40
MC/DC0%0/1
$param->{"answer"} // die("$where has a parameter that is a table without `answer`\n")
$param->{"answer"}
Truth table50%1/2
A
A= $param->{"answer"}
Aresult
11
00
73 4 ▶ die "$where takes a parameter from '$from', which is not an earlier step\n" if !$named{$from};
Branch: 1 branch
BranchTrueFalseTotal
unless $named{$from} 1 3 4
74 }
75 elsif (ref $param eq 'SCALAR') {
76 23 ▶ die "$where has an argument number that is not a whole number\n" if $$param !~ /\A[0-9]+\z/;
Branch: 1 branch
BranchTrueFalseTotal
unless $$param =~ /\A[0-9]+\z/u 0 23 23
77 23 ▶ $arguments = $$param + 1 if $$param + 1 > ($arguments // 0);
Branch: 1 branch
BranchTrueFalseTotal
if $$param + 1 > ($arguments // 0) 21 2 23
Condition100%2/2
$arguments // 0
l!l
419
MC/DC100%1/1
$arguments // 0
$arguments
Truth table100%2/2
A
A= $arguments
Aresult
11
00
78 }
79 1 elsif (ref $param) { die "$where has a parameter that is neither text, \\N nor { answer => step }\n" }
80 }
81 113 ▶ die "$where only classifies, so it needs a name for its answer\n" if $step{classify} && !defined $step{name};
Branch: 1 branch
BranchTrueFalseTotal
if $step{"classify"} and not defined $step{"name"} 1 112 113
Condition100%3/3
$step{"classify"} and not defined $step{"name"}
!ll&&!rl&&r
77351
MC/DC100%2/2
$step{"classify"} and not defined $step{"name"}
$step{"classify"} not defined $step{"name"}
Truth table100%3/3
A and not(B)
A= $step{"classify"}B= not defined $step{"name"}
ABresult
0X0
100
111
82 112 ▶ die "$where pools, which is for a step that classifies\n" if $step{pool} && !$step{classify};
Branch: 1 branch
BranchTrueFalseTotal
if $step{"pool"} and not $step{"classify"} 1 111 112
Condition100%3/3
$step{"pool"} and not $step{"classify"}
!ll&&!rl&&r
99121
MC/DC100%2/2
$step{"pool"} and not $step{"classify"}
$step{"pool"} not $step{"classify"}
Truth table100%3/3
A and not(B)
A= $step{"pool"}B= not $step{"classify"}
ABresult
0X0
100
111
83 111 for my $model (grep { $_->can('kind') } @model{@used}) {
84 129 ▶ die "$where pools, which is for class models\n" if $step{pool} && $model->kind ne 'class';
Branch: 1 branch
BranchTrueFalseTotal
if $step{"pool"} and $model->kind ne "class" 0 129 129
Condition66%2/3
$step{"pool"} and $model->kind ne "class"
!ll&&!rl&&r
114150
MC/DC0%0/2
$step{"pool"} and $model->kind ne "class"
$step{"pool"} $model->kind ne "class"
Truth table66%2/3
A and B
A= $step{"pool"}B= $model->kind ne "class"
ABresult
0X0
100
111
85 }
86 111 ▶ $named{ $step{name} } = 1 if defined $step{name};
Branch: 1 branch
BranchTrueFalseTotal
if defined $step{"name"} 35 76 111
87 111 push @{ $self->{steps} }, \%step;
88 }
89 51 ▶ $self->{arguments} = $arguments // 0;
Condition100%2/2
$arguments // 0
l!l
1932
MC/DC100%1/1
$arguments // 0
$arguments
Truth table100%2/2
A
A= $arguments
Aresult
11
00
90 51 return $self;
91 }
92  
93 # How many arguments a call takes after the string.
94 3 ▶sub arguments ($self) { return $self->{arguments} }
Subroutine arguments: called CC=1 SCAR=0.0
Pod documented
95  
96 # Run many strings through the steps. Returns one record per string:
97 # { text, confidence, answers => { step name => answer }, trace => [ ... ] }
98 # with a trace entry [model name, parameters, input, output, confidence] per
99 # step. Strings that reach a step with the same model and parameters go
100 # through it in one pass.
101 262 ▶sub run ($self, $texts, @arguments) { return $self->_run($texts, [ 0 .. $#$texts ], @arguments) }
Subroutine run: called CC=1 SCAR=0.0
Pod documented
102  
103 # Run many texts, each a list of strings (its words, say). Returns for each
104 # text the records of its strings. What differs from run() is a pooling
105 # step: it answers once per text.
106 14 ▶sub run_texts ($self, $texts, @arguments) {
Subroutine run_texts: called CC=2 SCAR=6.9
Pod documented
107 14 my @of = map { my $n = $_; ($n) x @{ $texts->[$n] } } 0 .. $#$texts;
108 14 my $states = $self->_run([ map { @$_ } @$texts ], \@of, @arguments);
109 14 my @result = map { [] } @$texts;
110 14 push @{ $result[ $of[$_] ] }, $states->[$_] for 0 .. $#of;
111 14 return \@result;
112 }
113  
114 # $of says for each string which text it belongs to.
115 276 ▶sub _run ($self, $texts, $of, @arguments) {
Subroutine _run: called CC=20 SCAR=30.0
116 die sprintf "this pipeline takes %d argument%s after the string, not %d\n",
117 276 ▶ $self->{arguments}, $self->{arguments} == 1 ? '' : 's', scalar @arguments if @arguments != $self->{arguments};
Branch: 2 branches
BranchTrueFalseTotal
if @arguments != $self->{"arguments"} 1 275 276
$self->{'arguments'} == 1 ? : 1 0 1
118  
119 275 my @state = map { { text => $texts->[$_], of => $of->[$_], confidence => 1, answers => {}, trace => [] } } 0 .. $#$texts;
120 275 for my $step (@{ $self->{steps} }) {
121 557 my %batch; # "model, parameters" => the states that take this step that way
122 557 for my $state (@state) {
123 18007 my $spec = $step->{model};
124 18007 my $name = $spec;
125 18007 ▶ if (ref $spec) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $spec) 3334 14673 18007
126 3334 my ($by) = keys %$spec;
127 3334 my $answer = $state->{answers}{$by};
128 3334 ▶ $name = $spec->{$by}{$answer} // $spec->{$by}{$ANY}
Condition100%5/5
($spec->{$by}{$answer} // $spec->{$by}{$ANY}) // die("no model for the answer '${answer}' of step '${by}'\n")
l!l
33331
$spec->{$by}{$answer} // $spec->{$by}{$ANY}
l!l&&r!l&&!r
198613471
MC/DC100%2/2
($spec->{$by}{$answer} // $spec->{$by}{$ANY}) // die("no model for the answer '${answer}' of step '${by}'\n")
$spec->{$by}{$answer} $spec->{$by}{$ANY}
Truth table100%3/3
A // B
A= $spec->{$by}{$answer}B= $spec->{$by}{$ANY}
ABresult
1X1
011
000
129 // die "no model for the answer '$answer' of step '$by'\n";
130 }
131 4350 ▶ my @values = map { ref eq 'HASH' ? $state->{answers}{ $_->{answer} } : ref eq 'SCALAR' ? $arguments[$$_] : $_ }
Branch: 2 branches
BranchTrueFalseTotal
ref $_ eq 'HASH' ? : 213 4137 4350
ref $_ eq 'SCALAR' ? : 3507 630 4137
132 18006 ▶ @{ $step->{params} // [] };
Condition100%2/2
$step->{"params"} // []
l!l
717010836
MC/DC100%1/1
$step->{"params"} // []
$step->{"params"}
Truth table100%2/2
A
A= $step->{"params"}
Aresult
11
00
133 18006 push @{ $batch{ join "\x{1f}", $name, @values }{states} }, $state;
134 18006 ▶ $batch{ join "\x{1f}", $name, @values }{call} //= [ $name, @values ];
Condition100%2/2
$batch{join "\37", $name, @values}{"call"} //= [$name, @values]
l!l
17435571
MC/DC100%1/1
$batch{join "\37", $name, @values}{"call"} //= [$name, @values]
$batch{join "\37", $name, @values}{"call"}
Truth table100%2/2
A
A= $batch{join "\37", $name, @values}{"call"}
Aresult
11
00
135 }
136 556 for my $group (values %batch) {
137 571 my ($name, @values) = @{ $group->{call} };
138 571 my @states = @{ $group->{states} };
139 571 ▶ if ($step->{pool}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($step->{"pool"}) 18 553 571
140 18 my %text;
141 18 push @{ $text{ $_->{of} } }, $_ for @states;
142 18 for my $members (values %text) {
143 774 my ($answer, $share) = @{ $self->{model}{$name}->pooled_for([ map { $_->{text} } @$members ], @values)->[0] };
144 774 for my $state (@$members) {
145 2521 push @{ $state->{trace} }, [ $name, \@values, $state->{text}, $answer, $share ];
146 2521 $state->{confidence} *= $share;
147 2521 $state->{answers}{ $step->{name} } = $answer;
148 }
149 }
150 18 next;
151 }
152 553 my $answers = $self->{model}{$name}->answers([ map { $_->{text} } @states ], @values);
153 552 for my $n (0 .. $#states) {
154 15484 my ($state, $answer, $confidence) = ($states[$n], @{ $answers->[$n] });
155 15484 push @{ $state->{trace} }, [ $name, \@values, $state->{text}, $answer, $confidence ];
156 15484 $state->{confidence} *= $confidence;
157 15484 ▶ $state->{answers}{ $step->{name} } = $answer if defined $step->{name};
Branch: 1 branch
BranchTrueFalseTotal
if defined $step->{"name"} 3880 11604 15484
158 15484 ▶ $state->{text} = $answer if !$step->{classify};
Branch: 1 branch
BranchTrueFalseTotal
unless $step->{"classify"} 11604 3880 15484
159 }
160 }
161 }
162 273 return \@state;
163 }
164  
165 # The pipeline's answer for a string; in list context also its confidence,
166 # the product of the steps' confidences: a chain is as sure as all its links
167 # together.
168 212 ▶sub predict ($self, $text, @arguments) {
Subroutine predict: called CC=2 SCAR=6.9
Pod documented
169 212 my ($state) = @{ $self->run([$text], @arguments) };
170 209 ▶ return wantarray ? @$state{qw(text confidence)} : $state->{text};
Branch: 1 branch
BranchTrueFalseTotal
wantarray ? : 3 206 209
171 }
172  
173 # The answers for many strings at once, in their order.
174 7 ▶sub predict_all ($self, $texts, @arguments) {
Subroutine predict_all: called CC=1 SCAR=0.0
Pod documented
175 7 return map { $_->{text} } @{ $self->run($texts, @arguments) };
176 }
177  
178 # The same steps as one fused model (Peta::NN::Fused): the models as they
179 # are, and what happens here between two steps inside it. Options as for
180 # Peta::NN::Fused->new (engine, meta). Only steps in series fuse.
181 1 ▶sub fuse ($self, %arg) {
Subroutine fuse: called CC=1 SCAR=0.0
Pod documented
182 1 require Peta::NN::Fused;
183 1 return Peta::NN::Fused->new(models => $self->{model}, steps => $self->{steps}, %arg);
184 }
185  
186 # What each step did with one string: a list of
187 # [model name, parameters, input, output, confidence].
188 2 ▶sub trace ($self, $text, @arguments) {
Subroutine trace: called CC=1 SCAR=0.0
Pod documented
189 2 my ($state) = @{ $self->run([$text], @arguments) };
190 2 return @{ $state->{trace} };
191 }
192  
193 5 1;
194  
195 __END__
« blib/lib/Peta/NN/Backend/Plain.pm ↑ Summary blib/lib/Peta/NN/Optimizer.pm »