lib/Peta/NN/Codec.pm

filter: Statementstmt 100% 46 / 46 Branchbran 100% 14 / 14 Conditioncond 100% 8 / 8 MC/DCmcdc 100% 5 / 5 Subroutinesub 100% 10 / 10 Podpod 100% 6 / 6 totaltotal 100% 89 / 89 CC 15 SCAR 11.8
CC
15
Cov
100%
_common
5.0
window
4.0
edit_label
4.0
CRAP 3.2
« blib/lib/Peta/NN/Layer/Dense.pm ↑ Summary blib/lib/Peta/NN/RNG.pm »
1 package Peta::NN::Codec;
2 # ABSTRACT: characters to token indices, string pairs to edit labels
3  
4 # Between strings and what a network eats and emits: characters become token
5 # indices through a vocabulary, and a pair of strings becomes an edit label.
6  
7 29 ▶use v5.36;
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 29 ▶use Exporter 'import';
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 29 our $VERSION = '0.2610090';
12 29 our @EXPORT_OK = qw(build_vocab window edit_label apply_edit edit2_label apply_edit2 PAD UNKNOWN);
13  
14 # Two indices are reserved: a window reaching past either end of the string
15 # reads PAD there, and a character never seen in training reads UNKNOWN.
16 29 ▶use constant { PAD => 0, UNKNOWN => 1 };
Subroutine BEGIN: called CC=1 SCAR=0.0
17  
18 29 my $FIRST_CHAR_INDEX = 2;
19  
20 # { character => index } over every character of the given strings.
21 79 ▶sub build_vocab ($strings) {
Subroutine build_vocab: called CC=3 SCAR=11.0
Pod documented
22 79 my %seen;
23 79 for my $string (@$strings) {
24 68930 $seen{$_} = 1 for split //, $string;
25 }
26 79 my $index = $FIRST_CHAR_INDEX;
27 79 return { map { $_ => $index++ } sort keys %seen };
28 }
29  
30 # The token indices of $count consecutive positions of @$chars starting at
31 # $from, which may lie before the start or run past the end. Every window in
32 # this library is this one shape: the last n characters start at length - n,
33 # the first n at 0, and n either side of position p at p - n.
34 426339 ▶sub window ($chars, $from, $count, $vocab) {
Subroutine window: called CC=4 SCAR=13.9
Pod documented
35 return [
36 426339 ▶ map { $_ < 0 || $_ > $#$chars ? PAD : $vocab->{ $chars->[$_] } // UNKNOWN }
Branch: 1 branch
BranchTrueFalseTotal
$_ < 0 || $_ > $chars->$#* ? : 17936 1458910 1476846
Condition100%5/5
$_ < 0 or $_ > $chars->$#*
l!l&&r!l&&!r
1205158851458910
$$vocab{$chars->[$_]} // 1
l!l
14576581252
MC/DC100%2/2
$_ < 0 or $_ > $chars->$#*
$_ < 0 $_ > $chars->$#*
MC/DC100%1/1
$$vocab{$chars->[$_]} // 1
$$vocab{$chars->[$_]}
Truth table100%3/3
A or B
A= $_ < 0B= $_ > $chars->$#*
ABresult
1X1
011
000
Truth table100%2/2
A
A= $$vocab{$chars->[$_]}
Aresult
11
00
37 $from .. $from + $count - 1
38 ];
39 }
40  
41 # How to turn $in into $out by rewriting its end: "cut:add" says drop that
42 # many characters and append that text. Jelínek -> Jelínku is "2:ku". Many
43 # words share one label, which is what makes it learnable as a class.
44 186741 ▶sub edit_label ($in, $out) {
Subroutine edit_label: called CC=4 SCAR=13.9
Pod documented
45 186741 my $common = 0;
46 186741 ▶ my $limit = length $in < length $out ? length $in : length $out;
Branch: 1 branch
BranchTrueFalseTotal
length $in < length $out ? : 131215 55526 186741
47 186741 ▶ $common++ while $common < $limit && substr($in, $common, 1) eq substr($out, $common, 1);
Branch: 1 branch
BranchTrueFalseTotal
$common < $limit and substr($in, $common, 1) eq substr($out, $common, 1) and do { ++$common } 1358005 186741 1544746
Condition100%3/3
$common < $limit and substr($in, $common, 1) eq substr($out, $common, 1)
!ll&&!rl&&r
154409323321358005
MC/DC100%2/2
$common < $limit and substr($in, $common, 1) eq substr($out, $common, 1)
$common < $limit substr($in, $common, 1) eq substr($out, $common, 1)
Truth table100%3/3
A and B
A= $common < $limitB= substr($in, $common, 1) eq substr($out, $common, 1)
ABresult
0X0
100
111
48 186741 return (length($in) - $common) . ':' . substr($out, $common);
49 }
50  
51 # Undefined when the label asks to cut more than the string has.
52 138131 ▶sub apply_edit ($in, $label) {
Subroutine apply_edit: called CC=2 SCAR=6.9
Pod documented
53 138131 my ($cut, $add) = split /:/, $label, 2;
54 138131 ▶ return if $cut > length $in;
Branch: 1 branch
BranchTrueFalseTotal
if $cut > length $in 613 137518 138131
55 137518 return substr($in, 0, length($in) - $cut) . $add;
56 }
57  
58 # The longest stretch of characters two strings share: where it starts in
59 # each, and how long it is. Of several equally long, the first in $in.
60 6007 ▶sub _common ($in, $out) {
Subroutine _common: called CC=5 SCAR=16.1
61 6007 my @a = split //, $in;
62 6007 my @b = split //, $out;
63 6007 my ($best, $at_in, $at_out) = (0, 0, 0);
64 6007 my @run = (0) x (@b + 1); # $run[j]: length of the stretch ending at a[i-1], b[j-1]
65 6007 for my $i (1 .. @a) {
66 48887 my @next = (0) x (@b + 1);
67 48887 for my $j (1 .. @b) {
68 479831 ▶ next if $a[ $i - 1 ] ne $b[ $j - 1 ];
Branch: 1 branch
BranchTrueFalseTotal
if $a[$i - 1] ne $b[$j - 1] 417209 62622 479831
69 62622 my $length = $next[$j] = $run[ $j - 1 ] + 1;
70 62622 ▶ ($best, $at_in, $at_out) = ($length, $i - $length, $j - $length) if $length > $best;
Branch: 1 branch
BranchTrueFalseTotal
if $length > $best 46468 16154 62622
71 }
72 48887 @run = @next;
73 }
74 6007 return ($at_in, $at_out, $best);
75 }
76  
77 # How to turn $in into $out by rewriting BOTH ends around what they share:
78 # "cut:add|cut:add", first for the front, then for the end. chytrý ->
79 # nejchytřejší keeps "chyt" and is "0:nej|2:řejší".
80 6007 ▶sub edit2_label ($in, $out) {
Subroutine edit2_label: called CC=1 SCAR=0.0
Pod documented
81 6007 my ($at_in, $at_out, $length) = _common($in, $out);
82 6007 return "$at_in:" . substr($out, 0, $at_out) . '|'
83 . (length($in) - $at_in - $length) . ':' . substr($out, $at_out + $length);
84 }
85  
86 # Undefined when the label asks to cut more than the string has.
87 815 ▶sub apply_edit2 ($in, $label) {
Subroutine apply_edit2: called CC=2 SCAR=6.9
Pod documented
88 815 my ($front, $back) = split /\|/, $label, 2;
89 815 my ($cut_front, $add_front) = split /:/, $front, 2;
90 815 my ($cut_back, $add_back) = split /:/, $back, 2;
91 815 ▶ return if $cut_front + $cut_back > length $in;
Branch: 1 branch
BranchTrueFalseTotal
if $cut_front + $cut_back > length $in 1 814 815
92 814 return $add_front . substr($in, $cut_front, length($in) - $cut_front - $cut_back) . $add_back;
93 }
94  
95 29 1;
96  
97 __END__
« blib/lib/Peta/NN/Layer/Dense.pm ↑ Summary blib/lib/Peta/NN/RNG.pm »