blib/lib/Peta/NN/Fused.pm untested

filter: Statementstmt 0.0% 0 / 546 Branchbran 0.0% 0 / 158 Conditioncond 0.0% 0 / 109 MC/DCmcdc 0.0% 0 / 62 Subroutinesub 0.0% 0 / 35 Podpod 0.0% 0 / 29 totaltotal 0.0% 0 / 939 CC 80 SCAR 55.7
CC
80
Cov
0%
new
870.0
_batch_gpu
156.0
_on_gpu
110.0
CRAP 262.5
1 package Peta::NN::Fused;
2 # ABSTRACT: micro models fused into one, as they are, the seams inside
3  
4 # Micro models fused into one: the models of a pipeline exactly as they are,
5 # weight for weight, and between them, inside the model, what a pipeline does
6 # in Perl between two steps.
7 #
8 # in a pipeline N1 -> [Perl: the answer becomes the input] -> N2
9 # fused N3 = N1 -> N2, the seam inside
10 # fused, consolidated N4, one model trained on what the chain does
11 #
12 # This module makes the second. Nothing is trained and no weight changes, so
13 # a fused model answers exactly what its pipeline answers, and has as many
14 # weights as its parts together. (The third is a new, smaller model; it is
15 # trained, and it is not the same function.)
16 #
17 # The seams. A pipeline does one of three things with a step's outputs, and
18 # each is an operation without weights in here:
19 #
20 # route The step is an edit model: its best label is an edit, which is
21 # carried out, and the next model reads the new string. Every
22 # label is one fixed edit, so what the next model will read is
23 # known per label in advance: some characters the label adds, and
24 # behind them characters of the string as it came in, shifted.
25 # class The step classifies: its best label is kept as the step's
26 # answer and the string passes on as it is.
27 # pool The step classifies a whole text: all strings of the text are
28 # evidence, and each gets the one answer they point to together.
29 #
30 # A kept answer does two things later on. It chooses which of several models
31 # a step runs for a string (all of them are computed side by side, and each
32 # only takes effect on the strings that are its own), and it can be a
33 # parameter of a later model.
34 #
35 # What is kept per string is its two ends as numbers: its last `reach`
36 # characters, which is as far back as any part can come to read once the
37 # parts before it have cut as deep as they can, and its first `front`, as
38 # far as any part reads from the front. A string of any length goes in as
39 # those and its length, passes all parts without coming back to Perl, and
40 # comes out as one edit (how much of the string that came in to cut, which
41 # characters to add) and its answers.
42 #
43 # The parts stay separate in the fused model and in its file. One of them can
44 # be replaced by a better one without touching the others; the seams are
45 # derived again from the parts' labels, which costs nothing.
46 #
47 # Two engines run a fused model. `cpu` computes each part as the inference
48 # leg does (PDL or plain Perl), so it agrees with the pipeline to the last
49 # bit. `gpu` keeps the whole batch on the card from the first part to the
50 # last, seams included, in 32-bit floats.
51 #
52 # What fuses: class models, and edit models that rewrite the end of a string.
53  
54 use v5.36;
55  
56 use Storable ();
57  
58 use Peta::NN::Codec qw(PAD UNKNOWN);
59 use Peta::NN::Inference;
60  
61 our $VERSION = '0.2610090';
62  
63 our $FORMAT = 'Peta::NN fused model';
64  
65 # The layout of a fused model's file.
66 # 1 the parts' model data under their names, and the steps
67 our $LAYOUT = 1;
68  
69 my $ANY = '*'; # in a routing table: the model for every answer not listed
70 my $FIRST_CHAR = 2; # 0 and 1 are PAD and UNKNOWN, as in every model
71 my $STORE_BITS = 64; # for a part that came as a trained model, not from a file
72  
73 my $GPU_REACH = 64; # the longest tail the route shader holds
74 my $GPU_ROWS = 16384; # strings per pass on the card, at most
75 my $GPU_1D_LIMIT = 4_000_000; # invocations of a one-dimensional dispatch (65535 groups of 64)
76  
77 my %ENGINE = map { $_ => 1 } qw(cpu gpu);
78  
79 # A string in flight is a list with these fields.
80 use constant {
81 TAIL => 0, # the numbers of its last characters, at least as many as any part will read
82 LENGTH => 1, # how long it is now
83 CUT => 2, # how many characters of the string that came in are cut from its end
84 ADDED => 3, # how many of the tail's last characters are added ones
85 CONFIDENCE => 4, # the product of the parts' confidences so far
86 HEAD => 5, # the numbers of its first characters
87 ANSWERS => 6, # per step that classifies: the number of its answer
88 TEXT => 7, # which text it belongs to
89 };
90  
91 # --- the shaders of the gpu engine --------------------------------------------
92 # Per string s: its tail tl[s * reach + r], r counting from the string's end;
93 # its head hd[s * front + i]; st[s * 3 ..] its length, cut and added; cf[s]
94 # its confidence; an[s * slots + k] its answer to classifying step k. A model
95 # that is one of several a step chooses from is given the slot to choose by
96 # (as slot + 1; 0 for a model that is the only one) and flags that say for
97 # which answers there it is the one.
98  
99 my $IS_MINE = <<'WGSL';
100 fn is_mine(s: u32, slots: u32, by: u32) -> bool {
101 if (by == 0u) { return true; }
102 return flags[an[s * slots + by - 1u]] != 0u;
103 }
104 WGSL
105  
106 # name => [ bindings, dimensions, body, functions ], as the backend's define() takes them.
107 my %SHADER = (
108  
109 # A part's input: for every string the table rows of its parameters'
110 # tokens and of its window's characters, read from head and tail.
111 # n0 dim, n1 parameters, n2 characters from the front, n3 from the end,
112 # n4 reach, n5 front
113 fused_read => [ [qw(r:e u:hd u:tl u:ptok u:cmap w:y)], 1, <<'WGSL' ],
114 let i = id.x;
115 if (i >= arrayLength(&y)) { return; }
116 let per = (p.n1 + p.n2 + p.n3) * p.n0;
117 let s = i / per; let t = (i % per) / p.n0;
118 var tok = 0u;
119 if (t < p.n1) { tok = ptok[s * p.n1 + t]; }
120 else if (t < p.n1 + p.n2) { tok = cmap[hd[s * p.n5 + t - p.n1]]; }
121 else { tok = cmap[tl[s * p.n4 + (p.n3 - 1u - (t - p.n1 - p.n2))]]; }
122 y[i] = e[tok * p.n0 + i % p.n0];
123 WGSL
124  
125 # A parameter that is an earlier answer: the token for it, per string.
126 # n0 parameters, n1 which of them, n2 slots, n3 the slot
127 fused_param => [ [qw(u:an u:table x:ptok)], 1, <<'WGSL' ],
128 let s = id.x;
129 if (s * p.n0 >= arrayLength(&ptok)) { return; }
130 ptok[s * p.n0 + p.n1] = table[an[s * p.n2 + p.n3]];
131 WGSL
132  
133 # Route: the best label of an edit model's outputs, and its edit carried
134 # out on the string's two ends. ed holds per label: how many characters
135 # to cut, how many to add, and those, the last one first. An edit that
136 # cuts more than the string has leaves it as it is.
137 # n0 labels, n1 reach, n2 the longest addition, n3 front, n4 slots,
138 # n5 the slot to choose by
139 fused_route => [ [qw(r:z u:ed x:tl x:st w:cf x:hd u:an u:flags)], 1, <<'WGSL', $IS_MINE ],
140 let s = id.x;
141 if (s >= arrayLength(&cf) || !is_mine(s, p.n4, p.n5)) { return; }
142 let at = s * p.n0;
143 var best = 0u;
144 var top = z[at];
145 for (var i = 1u; i < p.n0; i++) { if (z[at + i] > top) { top = z[at + i]; best = i; } }
146 var sum = 0.0;
147 for (var i = 0u; i < p.n0; i++) { sum += exp(z[at + i] - top); }
148 cf[s] = cf[s] / sum;
149  
150 let row = best * (2u + p.n2);
151 let cut = ed[row]; let grow = ed[row + 1u];
152 let len = st[s * 3u];
153 if (cut > len) { return; }
154 var was: array<u32, 64>;
155 for (var r = 0u; r < p.n1; r++) { was[r] = tl[s * p.n1 + r]; }
156 for (var r = 0u; r < p.n1; r++) {
157 var c = 0u;
158 if (r < grow) { c = ed[row + 2u + r]; }
159 else {
160 let source = r - grow + cut;
161 if (source < p.n1) { c = was[source]; }
162 }
163 tl[s * p.n1 + r] = c;
164 }
165 let kept = len - cut;
166 for (var i = kept; i < p.n3; i++) {
167 var c = 0u;
168 if (i - kept < grow) { c = ed[row + 2u + (grow - 1u - (i - kept))]; }
169 hd[s * p.n3 + i] = c;
170 }
171 let added = st[s * 3u + 2u];
172 if (cut <= added) { st[s * 3u + 2u] = added - cut + grow; }
173 else {
174 st[s * 3u + 1u] = st[s * 3u + 1u] + cut - added;
175 st[s * 3u + 2u] = grow;
176 }
177 st[s * 3u] = kept + grow;
178 WGSL
179  
180 # Class: the best label of a class model's outputs, kept as the step's
181 # answer. names holds for each label its number among the step's answers.
182 # n0 labels, n1 slots, n2 the step's slot, n3 the slot to choose by
183 fused_class => [ [qw(r:z w:cf x:an u:flags u:names)], 1, <<'WGSL', $IS_MINE ],
184 let s = id.x;
185 if (s >= arrayLength(&cf) || !is_mine(s, p.n1, p.n3)) { return; }
186 let at = s * p.n0;
187 var best = 0u;
188 var top = z[at];
189 for (var i = 1u; i < p.n0; i++) { if (z[at + i] > top) { top = z[at + i]; best = i; } }
190 var sum = 0.0;
191 for (var i = 0u; i < p.n0; i++) { sum += exp(z[at + i] - top); }
192 cf[s] = cf[s] / sum;
193 an[s * p.n1 + p.n2] = names[best];
194 WGSL
195  
196 # Pool, first half: the logarithm of every label's probability, per
197 # string; no probability counts as less than 1e-12, as in the inference leg.
198 # n0 labels
199 fused_logp => [ [qw(r:z w:lp)], 1, <<'WGSL' ],
200 let s = id.x;
201 if (s * p.n0 >= arrayLength(&lp)) { return; }
202 let at = s * p.n0;
203 var top = z[at];
204 for (var i = 1u; i < p.n0; i++) { top = max(top, z[at + i]); }
205 var sum = 0.0;
206 for (var i = 0u; i < p.n0; i++) { sum += exp(z[at + i] - top); }
207 for (var i = 0u; i < p.n0; i++) { lp[at + i] = log(max(exp(z[at + i] - top) / sum, 1e-12)); }
208 WGSL
209  
210 # Pool, second half, per text: the label its strings point to together,
211 # kept as the answer of each of them. tx holds where each text's strings
212 # start, and one more.
213 # n0 labels, n1 slots, n2 the step's slot, n3 the slot to choose by
214 fused_pool => [ [qw(r:lp u:tx w:cf x:an u:flags u:names)], 1, <<'WGSL', $IS_MINE ],
215 let g = id.x;
216 if (g + 1u >= arrayLength(&tx)) { return; }
217 var best = 0u;
218 var top = 0.0;
219 var some = false;
220 for (var l = 0u; l < p.n0; l++) {
221 var sum = 0.0;
222 for (var s = tx[g]; s < tx[g + 1u]; s++) { if (is_mine(s, p.n1, p.n3)) { sum += lp[s * p.n0 + l]; some = true; } }
223 if (l == 0u || sum > top) { top = sum; best = l; }
224 }
225 if (!some) { return; }
226 var all = 0.0;
227 for (var l = 0u; l < p.n0; l++) {
228 var sum = 0.0;
229 for (var s = tx[g]; s < tx[g + 1u]; s++) { if (is_mine(s, p.n1, p.n3)) { sum += lp[s * p.n0 + l]; } }
230 all += exp(sum - top);
231 }
232 for (var s = tx[g]; s < tx[g + 1u]; s++) {
233 if (is_mine(s, p.n1, p.n3)) {
234 cf[s] = cf[s] / all;
235 an[s * p.n1 + p.n2] = names[best];
236 }
237 }
238 WGSL
239 );
240  
241 sub _max (@n) { my $max = 0; for (@n) { $max = $_ if $_ > $max } return $max }
242  
243 # --- fusing ---------------------------------------------------------------------
244  
245 # A model as given to new(), as a part.
246 sub _part ($name, $model) {
247 my $data = !ref $model ? Peta::NN::Inference::read_file($model, $Peta::NN::Inference::FORMAT)
248 : ref $model eq 'HASH' ? $model
249 : $model->data;
250 my $read = eval { Peta::NN::Inference->new($data) } or die "the model '$name': $@";
251 die "the model '$name' is a $data->{kind} model that does not fuse; class models do, and edit models that rewrite the end of a string\n"
252 if $data->{kind} eq 'rewrite' || $data->{kind} eq 'edit' && $data->{side} ne 'right';
253 return {
254 name => $name, data => $data, model => $read, kind => $data->{kind}, params => $data->{params} // [],
255 front => $data->{side} ne 'right' ? $data->{window} : 0,
256 end => $data->{side} ne 'left' ? $data->{window} : 0,
257 labels => $data->{labels},
258 };
259 }
260  
261 # models { name => model }: a model file's path, model data, or an object
262 # that has it (Peta::NN::Inference, or the training leg's
263 # Peta::NN::Model)
264 # steps the steps, in order, as a pipeline takes them (Peta::NN::Pipeline)
265 # engine 'cpu' or 'gpu'; cpu unless PETA_NN_ENGINE says gpu
266 # meta a table that describes the fused model (name, description, ...)
267 sub new ($class, %arg) {
268 my $models = $arg{models} // die "models => { name => model, ... } is required\n";
269 my $steps = $arg{steps} // die "steps => [ ... ] is required\n";
270 die "a fused model needs at least one step\n" if !@$steps;
271 my $engine = $arg{engine} // (($ENV{PETA_NN_ENGINE} // '') eq 'gpu' ? 'gpu' : 'cpu');
272 die "a fused model runs on cpu or gpu, not on '$engine'\n" if !$ENGINE{$engine};
273 die "the description of the fused model is not a table\n" if defined $arg{meta} && ref $arg{meta} ne 'HASH';
274  
275 my $self = bless { engine => $engine, given => [], steps => [], part => {}, slot => {}, slots => [], meta => $arg{meta} }, $class;
276  
277 # The steps, as a pipeline reads them. A step becomes one `use` of a part
278 # per model it may run.
279 my $arguments = 0;
280 for my $n (0 .. $#$steps) {
281 my $where = 'step ' . ($n + 1);
282 die "$where is not a table\n" if ref $steps->[$n] ne 'HASH';
283 my %step = %{ $steps->[$n] };
284 my $spec = $step{model} // die "$where names no model\n";
285 push @{ $self->{given} }, { map { $_ => $step{$_} } grep { defined $step{$_} } qw(name model params classify pool) };
286  
287 # Which model, for which answers of which earlier step.
288 my (%by_model, $by);
289 if (ref $spec eq 'HASH') {
290 my ($from, @more) = keys %$spec;
291 die "$where routes by more than one step\n" if @more;
292 $by = $self->{slot}{$from} // die "$where routes by '$from', which is not an earlier step that classifies\n";
293 for my $answer (@{ $by->{labels} }) {
294 my $name = $spec->{$from}{$answer} // $spec->{$from}{$ANY}
295 // die "$where has no model for the answer '$answer' of step '$from'\n";
296 push @{ $by_model{$name} }, $answer;
297 }
298 }
299 elsif (ref $spec) { die "$where names its model by something that is neither a name nor a routing table\n" }
300 else { $by_model{$spec} = undef }
301  
302 my @params = @{ $step{params} // [] };
303 my @uses;
304 for my $name (sort keys %by_model) {
305 my $of = $self->{part}{$name}
306 //= _part($name, $models->{$name} // die "$where uses the model '$name', which is not among the models\n");
307 die "$where: the model '$name' is a class model, so the step has to classify (classify => 1, and a name)\n"
308 if $of->{kind} eq 'class' && !$step{classify};
309 die "$where classifies with the model '$name', which is an edit model\n" if $of->{kind} ne 'class' && $step{classify};
310 die sprintf "$where gives the model '$name' %d parameter%s; it takes %d\n",
311 scalar @params, @params == 1 ? '' : 's', scalar @{ $of->{params} } if @params != @{ $of->{params} };
312  
313 my @take;
314 for my $at (0 .. $#params) {
315 my ($param, $known) = ($params[$at], $of->{params}[$at]);
316 if (ref $param eq 'SCALAR') {
317 die "$where has an argument number that is not a whole number\n" if $$param !~ /\A[0-9]+\z/;
318 $arguments = $$param + 1 if $$param + 1 > $arguments;
319 push @take, { argument => $$param };
320 }
321 elsif (ref $param eq 'HASH') {
322 my $from = $param->{answer} // die "$where has a parameter that is a table without `answer`\n";
323 my $slot = $self->{slot}{$from} // die "$where takes a parameter from '$from', which is not an earlier step that classifies\n";
324 die "$where pools and takes a parameter from an answer; that does not fuse\n" if $step{pool};
325 # Only the answers that can reach this model need to be values it knows.
326 my %reach = map { $_ => 1 } $by && $by == $slot ? @{ $by_model{$name} } : @{ $slot->{labels} };
327 push @take, { slot => $slot->{at}, tokens => [
328 map { !$reach{$_} ? 0 : $known->{$_} // die "$where: the model '$name' has no value '$_' for its parameter @{[ $at + 1 ]}, which step '$from' can answer\n" }
329 @{ $slot->{labels} }
330 ] };
331 }
332 elsif (ref $param) { die "$where has a parameter that is neither text, \\N nor { answer => step }\n" }
333 else { push @take, { token => Peta::NN::Inference::param_tokens([$known], $param) } } # dies if unknown
334 }
335 my %mine = map { $_ => 1 } @{ $by_model{$name} // [] };
336 push @uses, {
337 part => $of,
338 params => \@take,
339 ($by ? (by => $by->{at}, flags => [ map { $mine{$_} ? 1 : 0 } @{ $by->{labels} } ]) : ()),
340 };
341 }
342  
343 # A step that classifies has a slot for its answer: the labels of all
344 # the models it may run, numbered together.
345 my %made = (uses => \@uses, pool => $step{pool} ? 1 : 0);
346 if ($step{classify}) {
347 die "$where only classifies, so it needs a name for its answer\n" if !defined $step{name};
348 my %seen = map { $_ => 1 } map { @{ $_->{part}{labels} } } @uses;
349 my @names = sort keys %seen;
350 my %at = map { $names[$_] => $_ } 0 .. $#names;
351 my $slot = { at => scalar @{ $self->{slots} }, name => $step{name}, labels => \@names };
352 $_->{names} = [ map { $at{$_} } @{ $_->{part}{labels} } ] for @uses;
353 $made{slot} = $slot->{at};
354 push @{ $self->{slots} }, $slot;
355 $self->{slot}{ $step{name} } = $slot; # from here on later steps can refer to it
356 }
357 elsif ($step{pool}) { die "$where pools, which is for a step that classifies\n" }
358 push @{ $self->{steps} }, \%made;
359 }
360 $self->{arguments} = $arguments;
361  
362 # One numbering of characters for the whole fused model: everything a part
363 # can read and everything a part can add.
364 my @parts = values %{ $self->{part} };
365 my @edit = grep { $_->{kind} eq 'edit' } @parts;
366 $_->{edits} = [ map { my ($cut, $add) = split /:/, $_, 2; [ $cut, $add ] } @{ $_->{labels} } ] for @edit;
367 my %seen = map { $_ => 1 } (map { keys %{ $_->{data}{vocab} } } @parts), map { map { split //, $_->[1] } @{ $_->{edits} } } @edit;
368 my @chars = sort keys %seen;
369 my $next = $FIRST_CHAR;
370 $self->{id} = { map { $_ => $next++ } @chars };
371 $self->{char} = [ '', '', @chars ];
372  
373 # The seams, from each part's labels: its edits in the common numbering,
374 # and for its window the token it reads each character as.
375 for my $of (@parts) {
376 my $vocab = $of->{data}{vocab};
377 $of->{cmap} = [ PAD, UNKNOWN, map { $vocab->{$_} // UNKNOWN } @chars ];
378 next if $of->{kind} ne 'edit';
379 $of->{edits} = [ map { [ $_->[0], [ map { $self->{id}{$_} } split //, $_->[1] ] ] } @{ $of->{edits} } ];
380 $of->{deepest} = _max(map { $_->[0] } @{ $of->{edits} });
381 $of->{longest} = _max(map { scalar @{ $_->[1] } } @{ $of->{edits} });
382 }
383  
384 # How much of a string's end must be kept: a step reads its window after
385 # the steps before it have cut as deep as they can; and everything the
386 # steps add together has to fit. And of its front: the widest window there.
387 my ($depth, $grown, $reach) = (0, 0, 1);
388 for my $step (@{ $self->{steps} }) {
389 my @of = map { $_->{part} } @{ $step->{uses} };
390 $reach = _max($reach, map { $_->{end} ? $_->{end} + $depth : 0 } @of);
391 $depth += _max(map { $_->{deepest} // 0 } @of);
392 $grown += _max(map { $_->{longest} // 0 } @of);
393 }
394 $self->{reach} = _max($reach, $grown);
395 $self->{front} = _max(map { $_->{front} } @parts);
396 return $self;
397 }
398  
399 # From a fused model's file (by convention *.fused). `engine` as for new().
400 sub load ($class, $file, %arg) {
401 my $data = Peta::NN::Inference::read_file($file, $FORMAT);
402 die sprintf "$file is in layout %s, and this version reads layout %d; fuse it again\n", $data->{layout} // '?', $LAYOUT
403 if ($data->{layout} // '') ne $LAYOUT;
404 die "$file is a malformed fused model: no models or no steps\n" if ref $data->{models} ne 'HASH' || ref $data->{steps} ne 'ARRAY';
405 my $self = eval { $class->new(models => $data->{models}, steps => $data->{steps}, meta => $data->{meta}, %arg) }
406 or die "$file is a malformed fused model: $@";
407 return $self;
408 }
409  
410 # A part's data as it goes into a file: as it came from its own file, or, for
411 # a part that came as a trained model, with its weights packed in full.
412 sub _stored ($data) {
413 return $data if !grep { ref eq 'ARRAY' } map { @{ $_->{weights} } } @{ $data->{layers} };
414 return {
415 %$data,
416 format => $Peta::NN::Inference::FORMAT,
417 layout => $Peta::NN::Inference::LAYOUT,
418 bits => $STORE_BITS,
419 layers => [
420 map { { %$_, weights => [ map { ref eq 'ARRAY' ? Peta::NN::Inference::pack_weights($_, $STORE_BITS) : $_ } @{ $_->{weights} } ] } }
421 @{ $data->{layers} }
422 ],
423 };
424 }
425  
426 # Write the fused model to a file: its parts, each as it is, and the steps.
427 sub save ($self, $file) {
428 Storable::nstore({
429 format => $FORMAT,
430 layout => $LAYOUT,
431 version => $VERSION,
432 models => { map { $_ => _stored($self->{part}{$_}{data}) } keys %{ $self->{part} } },
433 steps => $self->{given},
434 meta => { created => time, %{ $self->{meta} // {} } },
435 }, $file);
436 return $self;
437 }
438  
439 # The same fused model with some parts replaced: replace(name => model, ...),
440 # a model being what new() takes. The others are not touched.
441 sub replace ($self, %models) {
442 defined $self->{part}{$_} or die "this fused model has no part '$_'\n" for keys %models;
443 return ref($self)->new(
444 models => { (map { $_ => $self->{part}{$_}{data} } keys %{ $self->{part} }), %models },
445 steps => $self->{given},
446 engine => $self->{engine},
447 meta => $self->{meta},
448 );
449 }
450  
451 # --- what it is -----------------------------------------------------------------
452  
453 sub engine ($self) { return $self->{engine} }
454 sub arguments ($self) { return $self->{arguments} }
455 sub reach ($self) { return $self->{reach} }
456 sub front ($self) { return $self->{front} }
457  
458 # The names of the parts, in the order the steps first use them.
459 sub parts ($self) {
460 my %seen;
461 return grep { !$seen{$_}++ } map { map { $_->{part}{name} } @{ $_->{uses} } } @{ $self->{steps} };
462 }
463  
464 # A part, as the inference leg's model.
465 sub part ($self, $name) { return ($self->{part}{$name} // die "this fused model has no part '$name'\n")->{model} }
466  
467 # The names of the steps that classify, in order: what a record's answers are.
468 sub names ($self) { return map { $_->{name} } @{ $self->{slots} } }
469  
470 sub n_params ($self) {
471 my $n = 0;
472 $n += $self->part($_)->n_params for $self->parts;
473 return $n;
474 }
475  
476 # What there is to know about the fused model, for a human. A step is the
477 # name of its model, or of its models with | between them.
478 sub info ($self) {
479 return {
480 %{ $self->{meta} // {} },
481 format => $FORMAT,
482 version => $VERSION,
483 kind => 'fused',
484 reads => join(' and ', ($self->{front} ? "the first $self->{front}" : ()), "the last $self->{reach}") . ' characters',
485 steps => [ map { join '|', map { $_->{part}{name} } @{ $_->{uses} } } @{ $self->{steps} } ],
486 answers => [ $self->names ],
487 weights => $self->n_params,
488 arguments => $self->{arguments},
489 parts => { map { $_ => $self->part($_)->info } $self->parts },
490 };
491 }
492  
493 # --- answering --------------------------------------------------------------------
494  
495 # One edit, carried out on a string in flight.
496 sub _route ($row, $edit, $front) {
497 my ($cut, $add) = @$edit;
498 return if $cut > $row->[LENGTH];
499 my $tail = $row->[TAIL];
500 splice @$tail, -$cut if $cut;
501 push @$tail, @$add;
502 my $kept = $row->[LENGTH] - $cut;
503 if ($kept < $front) {
504 my $head = $row->[HEAD];
505 $head->[$_] = $add->[ $_ - $kept ] // PAD for $kept .. $front - 1;
506 }
507 if ($cut <= $row->[ADDED]) { $row->[ADDED] += @$add - $cut }
508 else { $row->[CUT] += $cut - $row->[ADDED]; $row->[ADDED] = @$add }
509 $row->[LENGTH] = $kept + @$add;
510 return;
511 }
512  
513 # The parts are computed as the inference leg computes them; the seams are here.
514 sub _run_cpu ($self, $rows) {
515 my $front = $self->{front};
516 for my $step (@{ $self->{steps} }) {
517 for my $use (@{ $step->{uses} }) {
518 my $of = $use->{part};
519 my @mine = defined $use->{by} ? grep { $use->{flags}[ $_->[ANSWERS][ $use->{by} ] ] } @$rows : @$rows;
520 next if !@mine;
521 my ($from_front, $from_end, $cmap) = @$of{qw(front end cmap)};
522 my @windows = map {
523 my $row = $_;
524 [ (map { $_->{token} // $_->{tokens}[ $row->[ANSWERS][ $_->{slot} ] ] } @{ $use->{tokens} }),
525 ($from_front ? @$cmap[ @{ $row->[HEAD] }[ 0 .. $from_front - 1 ] ] : ()),
526 ($from_end ? @$cmap[ @{ $row->[TAIL] }[ -$from_end .. -1 ] ] : ()),
527 ]
528 } @mine;
529  
530 if ($of->{kind} eq 'edit') {
531 my $decided = $of->{model}->decisions(\@windows);
532 for my $i (0 .. $#mine) {
533 $mine[$i][CONFIDENCE] *= $decided->[$i][1];
534 _route($mine[$i], $of->{edits}[ $decided->[$i][0] ], $front);
535 }
536 }
537 elsif ($step->{pool}) {
538 my $probabilities = $of->{model}->probabilities(\@windows);
539 my (%text, @order);
540 for my $i (0 .. $#mine) {
541 push @order, $mine[$i][TEXT] if !$text{ $mine[$i][TEXT] };
542 push @{ $text{ $mine[$i][TEXT] } }, $i;
543 }
544 my %at = map { $of->{labels}[$_] => $use->{names}[$_] } 0 .. $#{ $of->{labels} };
545 for my $members (@text{@order}) {
546 my ($answer, $share) = @{ $of->{model}->pool([ @$probabilities[@$members] ])->[0] };
547 for my $row (@mine[@$members]) {
548 $row->[CONFIDENCE] *= $share;
549 $row->[ANSWERS][ $step->{slot} ] = $at{$answer};
550 }
551 }
552 }
553 else {
554 my $decided = $of->{model}->decisions(\@windows);
555 for my $i (0 .. $#mine) {
556 $mine[$i][CONFIDENCE] *= $decided->[$i][1];
557 $mine[$i][ANSWERS][ $step->{slot} ] = $use->{names}[ $decided->[$i][0] ];
558 }
559 }
560 }
561 }
562 return;
563 }
564  
565 # The backend, and each part on the card: its weights, its window's tokens
566 # and its edits. A forked child puts them on its own device.
567 sub _on_gpu ($self) {
568 delete $self->{gpu} if $self->{gpu} && $self->{gpu}{owner} != $$;
569 return $self->{gpu} //= do {
570 require Peta::NN::Backend;
571 my $gpu = Peta::NN::Backend::try('gpu') // die "a fused model cannot run on the gpu here: $@";
572 die "this fused model keeps $self->{reach} characters per string; the gpu engine holds $GPU_REACH\n" if $self->{reach} > $GPU_REACH;
573 $gpu->define($_, @{ $SHADER{$_} }) for keys %SHADER;
574  
575 my ($widest, %on) = (1);
576 for my $name ($self->parts) {
577 my $of = $self->{part}{$name};
578 my ($embed, @rest) = @{ $of->{model}{layers} };
579 die "the model '$name' does not start with an embedding\n" if $embed->{type} ne 'embed';
580 my $width = $embed->{dim} * ($of->{front} + $of->{end} + @{ $of->{params} });
581 my @layers = map {
582 my $layer = $_;
583 $widest = _max($widest, $width);
584 if ($layer->{type} eq 'dense') {
585 my ($W, $b) = @{ $layer->{weights} };
586 my $n_in = $width;
587 $width = @$b;
588 [ $gpu->tensor($W, $n_in), $gpu->tensor($b, scalar @$b) ]
589 }
590 else { $layer->{type} }
591 } @rest;
592 $widest = _max($widest, $width);
593 my $longest = $of->{longest} // 0;
594 $on{$name} = {
595 dim => $embed->{dim},
596 table => $gpu->tensor($embed->{weights}[0], $embed->{dim}),
597 layers => \@layers,
598 cmap => $gpu->buffer(pack 'L*', @{ $of->{cmap} }),
599 ($of->{kind} eq 'edit'
600 ? (edits => $gpu->buffer(pack 'L*', map { my @add = reverse @{ $_->[1] }; ($_->[0], scalar @add, @add, (0) x ($longest - @add)) } @{ $of->{edits} }))
601 : ()),
602 };
603 }
604 # What each use of a part adds to it: for which answers it is the
605 # one, the numbers of its labels, the tokens of answers as parameters.
606 my $none = $gpu->buffer(pack 'L', 0);
607 my %used;
608 for my $use (map { @{ $_->{uses} } } @{ $self->{steps} }) {
609 $used{$use} = {
610 flags => $use->{flags} ? $gpu->buffer(pack 'L*', @{ $use->{flags} }) : $none,
611 names => $use->{names} ? $gpu->buffer(pack 'L*', @{ $use->{names} }) : $none,
612 tokens => [ map { $_->{tokens} ? $gpu->buffer(pack 'L*', @{ $_->{tokens} }) : undef } @{ $use->{params} } ],
613 };
614 }
615 my $rows = int($GPU_1D_LIMIT / $widest);
616 { owner => $$, backend => $gpu, on => \%on, used => \%used, none => $none, spare => $gpu->buffer(pack 'L', 0), rows => $rows < $GPU_ROWS ? $rows : $GPU_ROWS };
617 };
618 }
619  
620 # One batch, everything on the card: the strings' ends go up once, every part
621 # and every seam runs there, and what became of them comes back once.
622 sub _batch_gpu ($self, $rows) {
623 my $card = $self->_on_gpu;
624 my $gpu = $card->{backend};
625 my ($reach, $front) = @$self{qw(reach front)};
626 my $slots = @{ $self->{slots} };
627 my $count = @$rows;
628  
629 my $tail = $gpu->buffer(pack 'L*', map { reverse @{ $_->[TAIL] } } @$rows);
630 my $head = $front ? $gpu->buffer(pack 'L*', map { @{ $_->[HEAD] } } @$rows) : $card->{none};
631 my $state = $gpu->buffer(pack 'L*', map { @$_[ LENGTH, CUT, ADDED ] } @$rows);
632 my $heads = $front ? $head : $card->{spare}; # where route writes them: never the buffer it also only reads
633 my $conf = $gpu->buffer(pack 'f*', map { $_->[CONFIDENCE] } @$rows);
634 my $answers = $slots ? $gpu->buffer(pack 'L*', (0) x ($count * $slots)) : $card->{none};
635  
636 # Where each text's strings start, for the steps that pool.
637 my @starts = (0);
638 for my $i (1 .. $count - 1) { push @starts, $i if $rows->[$i][TEXT] != $rows->[ $i - 1 ][TEXT] }
639 my $texts = $gpu->buffer(pack 'L*', @starts, $count);
640  
641 for my $step (@{ $self->{steps} }) {
642 for my $use (@{ $step->{uses} }) {
643 my $of = $use->{part};
644 my $on = $card->{on}{ $of->{name} };
645 my $used = $card->{used}{$use};
646 my @params = @{ $use->{tokens} };
647 my $ptok = @params ? $gpu->buffer(pack 'L*', (map { $_->{token} // 0 } @params) x $count) : $card->{none};
648 for my $at (grep { defined $params[$_]{slot} } 0 .. $#params) {
649 $gpu->dispatch('fused_param', [ $answers, $used->{tokens}[$at], $ptok ], [ scalar @params, $at, $slots, $params[$at]{slot} ], $count);
650 }
651 my $cols = $on->{dim} * (@params + $of->{front} + $of->{end});
652 my $x = [ $gpu->buffer(4 * $count * $cols), $count, $cols ];
653 $gpu->dispatch('fused_read', [ $on->{table}[0], $head, $tail, $ptok, $on->{cmap}, $x->[0] ],
654 [ $on->{dim}, scalar @params, $of->{front}, $of->{end}, $reach, $front ], $count * $cols);
655 for my $layer (@{ $on->{layers} }) {
656 $x = ref $layer ? $gpu->affine($x, @$layer) : $gpu->activate($layer, $x);
657 }
658  
659 my $labels = @{ $of->{labels} };
660 my $by = defined $use->{by} ? $use->{by} + 1 : 0;
661 if ($of->{kind} eq 'edit') {
662 $gpu->dispatch('fused_route', [ $x->[0], $on->{edits}, $tail, $state, $conf, $heads, $answers, $used->{flags} ],
663 [ $labels, $reach, $of->{longest}, $front, $slots, $by ], $count);
664 }
665 elsif ($step->{pool}) {
666 my $logp = $gpu->buffer(4 * $count * $labels);
667 $gpu->dispatch('fused_logp', [ $x->[0], $logp ], [$labels], $count);
668 $gpu->dispatch('fused_pool', [ $logp, $texts, $conf, $answers, $used->{flags}, $used->{names} ],
669 [ $labels, $slots, $step->{slot}, $by ], scalar @starts);
670 }
671 else {
672 $gpu->dispatch('fused_class', [ $x->[0], $conf, $answers, $used->{flags}, $used->{names} ],
673 [ $labels, $slots, $step->{slot}, $by ], $count);
674 }
675 }
676 }
677  
678 my @tail = unpack 'L*', $tail->read;
679 my @state = unpack 'L*', $state->read;
680 my @conf = unpack 'f*', $conf->read;
681 my @answers = $slots ? unpack 'L*', $answers->read : ();
682 for my $i (0 .. $count - 1) {
683 my $row = $rows->[$i];
684 $row->[TAIL] = [ reverse @tail[ $i * $reach .. ($i + 1) * $reach - 1 ] ];
685 @$row[ LENGTH, CUT, ADDED ] = @state[ 3 * $i .. 3 * $i + 2 ];
686 $row->[CONFIDENCE] = $conf[$i];
687 $row->[ANSWERS] = [ @answers[ $i * $slots .. ($i + 1) * $slots - 1 ] ] if $slots;
688 }
689 return;
690 }
691  
692 # In batches the card takes, a text never split over two.
693 sub _run_gpu ($self, $rows) {
694 my $most = $self->_on_gpu->{rows};
695 my $from = 0;
696 while ($from < @$rows) {
697 my $to = $from + $most - 1;
698 if ($to >= $#$rows) { $to = $#$rows }
699 else {
700 $to-- while $to > $from && $rows->[ $to + 1 ][TEXT] == $rows->[$to][TEXT];
701 # One text that is longer than a batch goes in whole.
702 $to++ while $to < $#$rows && $rows->[ $to + 1 ][TEXT] == $rows->[$to][TEXT];
703 }
704 $self->_batch_gpu([ @$rows[ $from .. $to ] ]);
705 $from = $to + 1;
706 }
707 return;
708 }
709  
710 # The strings in flight after all parts: one row per string. $of says for
711 # each string which text it belongs to; the strings of a text follow one
712 # another.
713 sub _rows ($self, $texts, $of, @arguments) {
714 die sprintf "this fused model takes %d argument%s after the string, not %d\n",
715 $self->{arguments}, $self->{arguments} == 1 ? '' : 's', scalar @arguments if @arguments != $self->{arguments};
716 # The parameters that are arguments, as tokens: the same for every string.
717 for my $use (map { @{ $_->{uses} } } @{ $self->{steps} }) {
718 my $known = $use->{part}{params};
719 $use->{tokens} = [
720 map {
721 my $param = $use->{params}[$_];
722 defined $param->{argument} ? { token => Peta::NN::Inference::param_tokens([ $known->[$_] ], $arguments[ $param->{argument} ]) } : $param
723 } 0 .. $#{ $use->{params} }
724 ];
725 }
726  
727 my ($reach, $front, $id) = @$self{qw(reach front id)};
728 my $slots = @{ $self->{slots} };
729 # A model that reads no front keeps none, and one without parts that
730 # classify no answers: every string then has the same empty list for it,
731 # which nothing writes to.
732 my $nothing = [];
733 my @rows = map {
734 my $text = $texts->[$_];
735 my $length = length $text;
736 my @tail = map { $id->{$_} // UNKNOWN } split //, $length > $reach ? substr($text, -$reach) : $text;
737 my $head = $nothing;
738 if ($front) {
739 my @head = map { $id->{$_} // UNKNOWN } split //, substr($text, 0, $front);
740 $head = [ @head, (PAD) x ($front - @head) ];
741 }
742 [ [ (PAD) x ($reach - @tail), @tail ], $length, 0, 0, 1, $head, ($slots ? [ (0) x $slots ] : $nothing), $of->[$_] ]
743 } 0 .. $#$texts;
744  
745 if ($self->{engine} eq 'gpu') { $self->_run_gpu(\@rows) }
746 else { $self->_run_cpu(\@rows) }
747  
748 return \@rows;
749 }
750  
751 # What became of a string: it as it came in, less what was cut from its end,
752 # and what was added.
753 sub _text ($self, $text, $row) {
754 my $tail = $row->[TAIL];
755 return substr($text, 0, length($text) - $row->[CUT]) . join '', @{ $self->{char} }[ @$tail[ @$tail - $row->[ADDED] .. $#$tail ] ];
756 }
757  
758 # One record per string: { text, confidence, answers => { step name => answer } }.
759 sub _run ($self, $texts, $of, @arguments) {
760 my $rows = $self->_rows($texts, $of, @arguments);
761 my $named = $self->{slots};
762 return [
763 map {
764 my $row = $rows->[$_];
765 { text => $self->_text($texts->[$_], $row),
766 confidence => $row->[CONFIDENCE],
767 answers => { map { $_->{name} => $_->{labels}[ $row->[ANSWERS][ $_->{at} ] ] } @$named },
768 }
769 } 0 .. $#$texts
770 ];
771 }
772  
773 # Run many strings through the fused model. Returns one record per string:
774 # { text, confidence, answers => { step name => answer } }.
775 sub run ($self, $texts, @arguments) { return $self->_run($texts, [ 0 .. $#$texts ], @arguments) }
776  
777 # Run many texts, each a list of strings (its words, say). Returns for each
778 # text the records of its strings. What differs from run() is a pooling
779 # step: it answers once per text.
780 sub run_texts ($self, $texts, @arguments) {
781 my @of = map { my $n = $_; ($n) x @{ $texts->[$n] } } 0 .. $#$texts;
782 my $records = $self->_run([ map { @$_ } @$texts ], \@of, @arguments);
783 my @result = map { [] } @$texts;
784 push @{ $result[ $of[$_] ] }, $records->[$_] for 0 .. $#of;
785 return \@result;
786 }
787  
788 # [answer, confidence] for each of many strings, all with the same arguments.
789 sub answers ($self, $texts, @arguments) {
790 my $rows = $self->_rows($texts, [ 0 .. $#$texts ], @arguments);
791 return [ map { [ $self->_text($texts->[$_], $rows->[$_]), $rows->[$_][CONFIDENCE] ] } 0 .. $#$texts ];
792 }
793  
794 # The fused model's answer for a string; in list context also its confidence,
795 # the product of its parts' confidences.
796 sub predict ($self, $text, @arguments) {
797 my ($record) = @{ $self->run([$text], @arguments) };
798 return wantarray ? @$record{qw(text confidence)} : $record->{text};
799 }
800  
801 # The answers for many strings at once, in their order.
802 sub predict_all ($self, $texts, @arguments) {
803 return map { $_->[0] } @{ $self->answers($texts, @arguments) };
804 }
805  
806 1;
807  
808 __END__