blib/lib/Peta/NN/Data.pm untested

filter: Statementstmt 0.0% 0 / 129 Branchbran 0.0% 0 / 24 Conditioncond 0.0% 0 / 32 MC/DCmcdc 0.0% 0 / 17 Subroutinesub 0.0% 0 / 24 Podpod 0.0% 0 / 22 totaltotal 0.0% 0 / 248 CC 13 SCAR 26.5
CC
13
Cov
0%
hold_out
56.0
read
12.0
derive
6.0
CRAP 14.2
1 package Peta::NN::Data;
2 # ABSTRACT: records with named fields, to train models on and measure them by
3  
4 # What models are trained on and measured by: records with named fields.
5 #
6 # A record is one thing a model is to learn something about: a noun with its
7 # gender and its plural, a word with its language. A model is told which
8 # field it reads, which it answers and which it is given beside (from, to,
9 # given), and takes its training pairs from here.
10 #
11 # Two things about a record are not fields but its place in the work:
12 #
13 # held out it is not trained on, so that there is something left to
14 # measure a model by. Which records are held out is decided by
15 # a field's value (by default the first field), not by chance:
16 # the same record is on the same side on every run and on every
17 # machine, and records that share the value stay together.
18 # marked it belongs to a named group, such as the `core` a model has
19 # to get right without exception.
20 #
21 # A Data object is a view: where(), shown(), held() and marked() give a
22 # narrower view of the same records, and nothing is copied.
23  
24 use v5.36;
25 use utf8;
26  
27 our $VERSION = '0.2610090';
28  
29 my $WORD = 0xFFFFFFFF; # the hash below is in 32 bits, the same on every perl
30  
31 # records a list of tables, one per record
32 # fields the names of their fields, in the order they are thought of
33 sub new ($class, %arg) {
34 my $records = $arg{records} // die "records => [ { field => value, ... }, ... ] is required\n";
35 my $fields = $arg{fields} // [ sort keys %{ $records->[0] // {} } ];
36 return bless { records => $records, fields => [@$fields], held => {}, mark => {} }, $class;
37 }
38  
39 # From a file of tab-separated lines, one record a line. Lines that start
40 # with # are comments, and a missing last field is empty.
41 # fields the names of the columns
42 sub read ($class, $file, %arg) {
43 my $fields = $arg{fields} // die "fields => [ names of the columns ] is required\n";
44 open my $in, '<:encoding(UTF-8)', $file or die "cannot read $file: $!\n";
45 my @records;
46 while (my $line = <$in>) {
47 next if $line =~ /\A#/;
48 chomp $line;
49 my @values = split /\t/, $line, -1;
50 die "$file line $.: " . scalar(@values) . " fields where there are " . scalar(@$fields) . " names\n" if @values > @$fields;
51 push @records, { map { $fields->[$_] => $values[$_] // '' } 0 .. $#$fields };
52 }
53 close $in;
54 return $class->new(records => \@records, fields => $fields);
55 }
56  
57 # A view of some of these records, with what is known about them.
58 sub _view ($self, $records) {
59 return bless { %$self, records => $records }, ref $self;
60 }
61  
62 sub records ($self) { return @{ $self->{records} } }
63 sub count ($self) { return scalar @{ $self->{records} } }
64 sub fields ($self) { return @{ $self->{fields} } }
65  
66 # The values of one field, record by record.
67 sub values_of ($self, $field) {
68 $self->_known($field);
69 return map { $_->{$field} } @{ $self->{records} };
70 }
71  
72 sub _known ($self, @fields) {
73 my %have = map { $_ => 1 } @{ $self->{fields} };
74 $have{$_} or die "there is no field '$_' (there are: @{ $self->{fields} })\n" for @fields;
75 return;
76 }
77  
78 # A new field, worked out from each record: derive(name => sub ($record) { ... }).
79 sub derive ($self, %new) {
80 for my $field (sort keys %new) {
81 $_->{$field} = $new{$field}->($_) for @{ $self->{records} };
82 push @{ $self->{fields} }, $field if !grep { $_ eq $field } @{ $self->{fields} };
83 }
84 return $self;
85 }
86  
87 # The records a sub accepts.
88 sub where ($self, $accept) {
89 return $self->_view([ grep { $accept->($_) } @{ $self->{records} } ]);
90 }
91  
92 # A number from 0 up to 1 for a value: the same wherever the value occurs,
93 # and nothing like that of a value that differs by a letter (FNV-1a, stirred).
94 sub _place ($value) {
95 my $hash = 2166136261;
96 $hash = (($hash ^ ord) * 16777619) & $WORD for split //, $value;
97 $hash ^= $hash >> 15;
98 $hash = ($hash * 2246822519) & $WORD;
99 $hash ^= $hash >> 13;
100 return $hash / ($WORD + 1);
101 }
102  
103 # Hold a share of the records out of training. Which ones follows from the
104 # value of a field (by => name, default the first field), so records that
105 # share it are held out together or not at all; `by` may also be a sub that
106 # gives the value for a record. The records of a mark can be exempted
107 # (never => name): what has to be right without exception is trained on.
108 sub hold_out ($self, $share, %arg) {
109 die "the share to hold out is a number above 0 and below 1, not '$share'\n" if !($share > 0 && $share < 1);
110 my $by = $arg{by} // $self->{fields}[0];
111 $self->_known($by) if !ref $by;
112 my $never = defined $arg{never} ? $self->{mark}{ $arg{never} } // die "no records are marked '$arg{never}'; mark them before holding out\n" : {};
113 %{ $self->{held} } = ();
114 for my $record (@{ $self->{records} }) {
115 next if $never->{$record};
116 my $value = ref $by ? $by->($record) : $record->{$by};
117 $self->{held}{$record} = 1 if _place($value) < $share;
118 }
119 return $self;
120 }
121  
122 # Hold out the records a sub accepts, and no others: for data that has to be
123 # on the same sides as other data.
124 sub hold_out_if ($self, $accept) {
125 %{ $self->{held} } = map { $_ => 1 } grep { $accept->($_) } @{ $self->{records} };
126 return $self;
127 }
128  
129 # The records that are trained on, and those that are held out.
130 sub shown ($self) { return $self->_view([ grep { !$self->{held}{$_} } @{ $self->{records} } ]) }
131 sub held ($self) { return $self->_view([ grep { $self->{held}{$_} } @{ $self->{records} } ]) }
132  
133 # Name a group of records: mark(core => sub ($record) { ... }). A record that
134 # is marked is not thereby trained on or held out; that is another question.
135 sub mark ($self, %group) {
136 for my $name (sort keys %group) {
137 $self->{mark}{$name} = { map { $_ => 1 } grep { $group{$name}->($_) } @{ $self->{records} } };
138 }
139 return $self;
140 }
141  
142 sub marks ($self) { return sort keys %{ $self->{mark} } }
143  
144 # The records of a named group.
145 sub marked ($self, $name) {
146 my $group = $self->{mark}{$name} // die "no records are marked '$name' (marked are: @{[ $self->marks ]})\n";
147 return $self->_view([ grep { $group->{$_} } @{ $self->{records} } ]);
148 }
149  
150 sub is_marked ($self, $name, $record) { return ($self->{mark}{$name} // {})->{$record} ? 1 : 0 }
151 sub is_held ($self, $record) { return $self->{held}{$record} ? 1 : 0 }
152  
153 # Some of the records, the same ones every time: at most $count of them,
154 # spread evenly over the whole.
155 sub sample ($self, $count) {
156 my $records = $self->{records};
157 return $self->_view([@$records]) if $count >= @$records;
158 return $self->_view([ map { $records->[ int($_ * @$records / $count) ] } 0 .. $count - 1 ]);
159 }
160  
161 # What a model learns from: for each record [input, answer, parameters ...].
162 # from the field a model reads
163 # to the field it answers
164 # given the fields it is told beside, in order
165 sub pairs ($self, %arg) {
166 my $from = $arg{from} // die "from => the field a model reads is required\n";
167 my $to = $arg{to} // die "to => the field a model answers is required\n";
168 my @given = @{ $arg{given} // [] };
169 $self->_known($from, $to, @given);
170 return [ map { [ @$_{ $from, $to, @given } ] } @{ $self->{records} } ];
171 }
172  
173 1;
174  
175 __END__