lib/Peta/NN/Data.pm

filter: Statementstmt 100% 109 / 109 Branchbran 96.6% 29 / 30 Conditioncond 75.0% 21 / 28 MC/DCmcdc 46.6% 7 / 15 Subroutinesub 100% 24 / 24 Podpod 100% 19 / 19 totaltotal 92.8% 209 / 225 CC 35 SCAR 15.3
CC
35
Cov
95%
hold_out
11.0
read
7.0
new
4.1
CRAP 4.6
« blib/lib/Peta/NN/Codec.pm ↑ Summary blib/lib/Peta/NN/Backend/WebGPU.pm »
1 package Peta::NN::Data;
2 # ABSTRACT: records with named fields, to train models on and measure them by
3  
4 # What models are trained on and measured by: records with named fields.
5 #
6 # A record is one thing a model is to learn something about: a noun with its
7 # gender and its plural, a word with its language. A model is told which
8 # field it reads, which it answers and which it is given beside (from, to,
9 # given), and takes its training pairs from here.
10 #
11 # Two things about a record are not fields but its place in the work:
12 #
13 # held out it is not trained on, so that there is something left to
14 # measure a model by. Which records are held out is decided by
15 # a field's value (by default the first field), not by chance:
16 # the same record is on the same side on every run and on every
17 # machine, and records that share the value stay together.
18 # marked it belongs to a named group, such as the `core` a model has
19 # to get right without exception.
20 #
21 # A Data object is a view: where(), shown(), held() and marked() give a
22 # narrower view of the same records, and nothing is copied.
23  
24 3 ▶use v5.36;
Subroutine BEGIN: called CC=1 SCAR=0.0
25 3 ▶use utf8;
Subroutine BEGIN: called CC=1 SCAR=0.0
26  
27 3 our $VERSION = '0.2610090';
28  
29 3 my $WORD = 0xFFFFFFFF; # the hash below is in 32 bits, the same on every perl
30  
31 # records a list of tables, one per record
32 # fields the names of their fields, in the order they are thought of
33 6 ▶sub new ($class, %arg) {
Subroutine new: called CC=4 SCAR=14.2
Pod documented
34 6 ▶ my $records = $arg{records} // die "records => [ { field => value, ... }, ... ] is required\n";
Condition50%1/2
$arg{"records"} // die("records => [ { field => value, ... }, ... ] is required\n")
l!l
60
MC/DC0%0/1
$arg{"records"} // die("records => [ { field => value, ... }, ... ] is required\n")
$arg{"records"}
Truth table50%1/2
A
A= $arg{"records"}
Aresult
11
00
35 6 ▶ my $fields = $arg{fields} // [ sort keys %{ $records->[0] // {} } ];
Condition75%3/4
$arg{"fields"} // [sort(keys %{$records->[0] // {};})]
l!l
42
$records->[0] // {}
l!l
20
MC/DC100%1/1
$arg{"fields"} // [sort(keys %{$records->[0] // {};})]
$arg{"fields"}
MC/DC0%0/1
$records->[0] // {}
$records->[0]
Truth table100%2/2
A
A= $arg{"fields"}
Aresult
11
00
Truth table50%1/2
A
A= $records->[0]
Aresult
11
00
36 6 return bless { records => $records, fields => [@$fields], held => {}, mark => {} }, $class;
37 }
38  
39 # From a file of tab-separated lines, one record a line. Lines that start
40 # with # are comments, and a missing last field is empty.
41 # fields the names of the columns
42 4 ▶sub read ($class, $file, %arg) {
Subroutine read: called CC=7 SCAR=19.5
Pod documented
43 4 ▶ my $fields = $arg{fields} // die "fields => [ names of the columns ] is required\n";
Condition50%1/2
$arg{"fields"} // die("fields => [ names of the columns ] is required\n")
l!l
40
MC/DC0%0/1
$arg{"fields"} // die("fields => [ names of the columns ] is required\n")
$arg{"fields"}
Truth table50%1/2
A
A= $arg{"fields"}
Aresult
11
00
44 4 ▶ open my $in, '<:encoding(UTF-8)', $file or die "cannot read $file: $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open my $in, "<:encoding(UTF-8)", $file or die "cannot read ${file}: $!\n" 3 1 4
45 3 my @records;
46 3 ▶ while (my $line = <$in>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $in)) 810 2 812
47 810 ▶ next if $line =~ /\A#/;
Branch: 1 branch
BranchTrueFalseTotal
if $line =~ /\A#/u 3 807 810
48 807 chomp $line;
49 807 my @values = split /\t/, $line, -1;
50 807 ▶ die "$file line $.: " . scalar(@values) . " fields where there are " . scalar(@$fields) . " names\n" if @values > @$fields;
Branch: 1 branch
BranchTrueFalseTotal
if @values > @$fields 1 806 807
51 806 ▶ push @records, { map { $fields->[$_] => $values[$_] // '' } 0 .. $#$fields };
Condition100%2/2
$values[$_] // ""
l!l
2422802
MC/DC100%1/1
$values[$_] // ""
$values[$_]
Truth table100%2/2
A
A= $values[$_]
Aresult
11
00
52 }
53 2 close $in;
54 2 return $class->new(records => \@records, fields => $fields);
55 }
56  
57 # A view of some of these records, with what is known about them.
58 45 ▶sub _view ($self, $records) {
Subroutine _view: called CC=1 SCAR=0.0
59 45 return bless { %$self, records => $records }, ref $self;
60 }
61  
62 12 ▶sub records ($self) { return @{ $self->{records} } }
Subroutine records: called CC=1 SCAR=0.0
Pod documented
63 12 ▶sub count ($self) { return scalar @{ $self->{records} } }
Subroutine count: called CC=1 SCAR=0.0
Pod documented
64 3 ▶sub fields ($self) { return @{ $self->{fields} } }
Subroutine fields: called CC=1 SCAR=0.0
Pod documented
65  
66 # The values of one field, record by record.
67 7 ▶sub values_of ($self, $field) {
Subroutine values_of: called CC=1 SCAR=0.0
Pod documented
68 7 $self->_known($field);
69 6 return map { $_->{$field} } @{ $self->{records} };
70 }
71  
72 39 ▶sub _known ($self, @fields) {
Subroutine _known: called CC=3 SCAR=11.0
73 39 my %have = map { $_ => 1 } @{ $self->{fields} };
74 39 ▶ $have{$_} or die "there is no field '$_' (there are: @{ $self->{fields} })\n" for @fields;
Branch: 1 branch
BranchTrueFalseTotal
$have{$_} or die qq[there is no field '${_}' (there are: @{$self->{"fields"};})\n] 73 4 77
75 35 return;
76 }
77  
78 # A new field, worked out from each record: derive(name => sub ($record) { ... }).
79 1 ▶sub derive ($self, %new) {
Subroutine derive: called CC=4 SCAR=13.9
Pod documented
80 1 for my $field (sort keys %new) {
81 1 $_->{$field} = $new{$field}->($_) for @{ $self->{records} };
82 1 ▶ push @{ $self->{fields} }, $field if !grep { $_ eq $field } @{ $self->{fields} };
Branch: 1 branch
BranchTrueFalseTotal
unless grep {$_ eq $field;} @{$$self{"fields"};} 1 0 1
83 }
84 1 return $self;
85 }
86  
87 # The records a sub accepts.
88 2 ▶sub where ($self, $accept) {
Subroutine where: called CC=1 SCAR=0.0
Pod documented
89 2 return $self->_view([ grep { $accept->($_) } @{ $self->{records} } ]);
90 }
91  
92 # A number from 0 up to 1 for a value: the same wherever the value occurs,
93 # and nothing like that of a value that differs by a letter (FNV-1a, stirred).
94 3624 ▶sub _place ($value) {
Subroutine _place: called CC=2 SCAR=6.9
95 3624 my $hash = 2166136261;
96 3624 $hash = (($hash ^ ord) * 16777619) & $WORD for split //, $value;
97 3624 $hash ^= $hash >> 15;
98 3624 $hash = ($hash * 2246822519) & $WORD;
99 3624 $hash ^= $hash >> 13;
100 3624 return $hash / ($WORD + 1);
101 }
102  
103 # Hold a share of the records out of training. Which ones follows from the
104 # value of a field (by => name, default the first field), so records that
105 # share it are held out together or not at all; `by` may also be a sub that
106 # gives the value for a record. The records of a mark can be exempted
107 # (never => name): what has to be right without exception is trained on.
108 10 ▶sub hold_out ($self, $share, %arg) {
Subroutine hold_out: called CC=11 SCAR=24.0
Pod documented
109 10 ▶ die "the share to hold out is a number above 0 and below 1, not '$share'\n" if !($share > 0 && $share < 1);
Branch: 1 branch
BranchTrueFalseTotal
unless $share > 0 and $share < 1 1 9 10
Condition66%2/3
$share > 0 and $share < 1
!ll&&!rl&&r
019
MC/DC50%1/2
$share > 0 and $share < 1
$share > 0 $share < 1
Truth table66%2/3
A and B
A= $share > 0B= $share < 1
ABresult
0X0
100
111
110 9 ▶ my $by = $arg{by} // $self->{fields}[0];
Condition66%2/3
$arg{"by"} // $self->{"fields"}[0]
l!l&&r!l&&!r
360
MC/DC0%0/2
$arg{"by"} // $self->{"fields"}[0]
$arg{"by"} $self->{"fields"}[0]
Truth table66%2/3
A // B
A= $arg{"by"}B= $self->{"fields"}[0]
ABresult
1X1
011
000
111 9 ▶ $self->_known($by) if !ref $by;
Branch: 1 branch
BranchTrueFalseTotal
unless ref $by 8 1 9
112 8 ▶ my $never = defined $arg{never} ? $self->{mark}{ $arg{never} } // die "no records are marked '$arg{never}'; mark them before holding out\n" : {};
Branch: 1 branch
BranchTrueFalseTotal
defined $arg{'never'} ? : 2 6 8
Condition100%2/2
$self->{"mark"}{$arg{"never"}} // die(qq[no records are marked '$arg{"never"}'; mark them before holding out\n])
l!l
11
MC/DC100%1/1
$self->{"mark"}{$arg{"never"}} // die(qq[no records are marked '$arg{"never"}'; mark them before holding out\n])
$self->{"mark"}{$arg{"never"}}
Truth table100%2/2
A
A= $self->{"mark"}{$arg{"never"}}
Aresult
11
00
113 7 %{ $self->{held} } = ();
114 7 for my $record (@{ $self->{records} }) {
115 4015 ▶ next if $never->{$record};
Branch: 1 branch
BranchTrueFalseTotal
if $never->{$record} 391 3624 4015
116 3624 ▶ my $value = ref $by ? $by->($record) : $record->{$by};
Branch: 1 branch
BranchTrueFalseTotal
ref $by ? : 403 3221 3624
117 3624 ▶ $self->{held}{$record} = 1 if _place($value) < $share;
Branch: 1 branch
BranchTrueFalseTotal
if _place($value) < $share 904 2720 3624
118 }
119 7 return $self;
120 }
121  
122 # Hold out the records a sub accepts, and no others: for data that has to be
123 # on the same sides as other data.
124 1 ▶sub hold_out_if ($self, $accept) {
Subroutine hold_out_if: called CC=1 SCAR=0.0
Pod documented
125 1 %{ $self->{held} } = map { $_ => 1 } grep { $accept->($_) } @{ $self->{records} };
126 1 return $self;
127 }
128  
129 # The records that are trained on, and those that are held out.
130 16 ▶sub shown ($self) { return $self->_view([ grep { !$self->{held}{$_} } @{ $self->{records} } ]) }
Subroutine shown: called CC=1 SCAR=0.0
Pod documented
131 15 ▶sub held ($self) { return $self->_view([ grep { $self->{held}{$_} } @{ $self->{records} } ]) }
Subroutine held: called CC=1 SCAR=0.0
Pod documented
132  
133 # Name a group of records: mark(core => sub ($record) { ... }). A record that
134 # is marked is not thereby trained on or held out; that is another question.
135 2 ▶sub mark ($self, %group) {
Subroutine mark: called CC=2 SCAR=6.9
Pod documented
136 2 for my $name (sort keys %group) {
137 3 $self->{mark}{$name} = { map { $_ => 1 } grep { $group{$name}->($_) } @{ $self->{records} } };
138 }
139 2 return $self;
140 }
141  
142 7 ▶sub marks ($self) { return sort keys %{ $self->{mark} } }
Subroutine marks: called CC=1 SCAR=0.0
Pod documented
143  
144 # The records of a named group.
145 9 ▶sub marked ($self, $name) {
Subroutine marked: called CC=2 SCAR=6.9
Pod documented
146 9 ▶ my $group = $self->{mark}{$name} // die "no records are marked '$name' (marked are: @{[ $self->marks ]})\n";
Condition100%2/2
$self->{"mark"}{$name} // die("no records are marked '${name}' (marked are: @{[$self->marks];})\n")
l!l
81
MC/DC100%1/1
$self->{"mark"}{$name} // die("no records are marked '${name}' (marked are: @{[$self->marks];})\n")
$self->{"mark"}{$name}
Truth table100%2/2
A
A= $self->{"mark"}{$name}
Aresult
11
00
147 8 return $self->_view([ grep { $group->{$_} } @{ $self->{records} } ]);
148 }
149  
150 2 ▶sub is_marked ($self, $name, $record) { return ($self->{mark}{$name} // {})->{$record} ? 1 : 0 }
Subroutine is_marked: called CC=3 SCAR=11.2
Pod documented
Branch: 1 branch
BranchTrueFalseTotal
+($self->{'mark'}{$name} // {})->{$record} ? : 1 1 2
Condition50%1/2
$self->{"mark"}{$name} // {}
l!l
20
MC/DC0%0/1
$self->{"mark"}{$name} // {}
$self->{"mark"}{$name}
Truth table50%1/2
A
A= $self->{"mark"}{$name}
Aresult
11
00
151 1609 ▶sub is_held ($self, $record) { return $self->{held}{$record} ? 1 : 0 }
Subroutine is_held: called CC=2 SCAR=6.9
Pod documented
Branch: 1 branch
BranchTrueFalseTotal
$self->{'held'}{$record} ? : 490 1119 1609
152  
153 # Some of the records, the same ones every time: at most $count of them,
154 # spread evenly over the whole.
155 4 ▶sub sample ($self, $count) {
Subroutine sample: called CC=2 SCAR=6.9
Pod documented
156 4 my $records = $self->{records};
157 4 ▶ return $self->_view([@$records]) if $count >= @$records;
Branch: 1 branch
BranchTrueFalseTotal
if $count >= @$records 1 3 4
158 3 return $self->_view([ map { $records->[ int($_ * @$records / $count) ] } 0 .. $count - 1 ]);
159 }
160  
161 # What a model learns from: for each record [input, answer, parameters ...].
162 # from the field a model reads
163 # to the field it answers
164 # given the fields it is told beside, in order
165 25 ▶sub pairs ($self, %arg) {
Subroutine pairs: called CC=4 SCAR=13.9
Pod documented
166 25 ▶ my $from = $arg{from} // die "from => the field a model reads is required\n";
Condition50%1/2
$arg{"from"} // die("from => the field a model reads is required\n")
l!l
250
MC/DC0%0/1
$arg{"from"} // die("from => the field a model reads is required\n")
$arg{"from"}
Truth table50%1/2
A
A= $arg{"from"}
Aresult
11
00
167 25 ▶ my $to = $arg{to} // die "to => the field a model answers is required\n";
Condition100%2/2
$arg{"to"} // die("to => the field a model answers is required\n")
l!l
241
MC/DC100%1/1
$arg{"to"} // die("to => the field a model answers is required\n")
$arg{"to"}
Truth table100%2/2
A
A= $arg{"to"}
Aresult
11
00
168 24 ▶ my @given = @{ $arg{given} // [] };
Condition100%2/2
$arg{"given"} // []
l!l
222
MC/DC100%1/1
$arg{"given"} // []
$arg{"given"}
Truth table100%2/2
A
A= $arg{"given"}
Aresult
11
00
169 24 $self->_known($from, $to, @given);
170 22 return [ map { [ @$_{ $from, $to, @given } ] } @{ $self->{records} } ];
171 }
172  
173 3 1;
174  
175 __END__
« blib/lib/Peta/NN/Codec.pm ↑ Summary blib/lib/Peta/NN/Backend/WebGPU.pm »