| 1 | package Peta::NN::Data; | ||
| 2 | # ABSTRACT: records with named fields, to train models on and measure them by | ||
| 3 | |||
| 4 | # What models are trained on and measured by: records with named fields. | ||
| 5 | # | ||
| 6 | # A record is one thing a model is to learn something about: a noun with its | ||
| 7 | # gender and its plural, a word with its language. A model is told which | ||
| 8 | # field it reads, which it answers and which it is given beside (from, to, | ||
| 9 | # given), and takes its training pairs from here. | ||
| 10 | # | ||
| 11 | # Two things about a record are not fields but its place in the work: | ||
| 12 | # | ||
| 13 | # held out it is not trained on, so that there is something left to | ||
| 14 | # measure a model by. Which records are held out is decided by | ||
| 15 | # a field's value (by default the first field), not by chance: | ||
| 16 | # the same record is on the same side on every run and on every | ||
| 17 | # machine, and records that share the value stay together. | ||
| 18 | # marked it belongs to a named group, such as the `core` a model has | ||
| 19 | # to get right without exception. | ||
| 20 | # | ||
| 21 | # A Data object is a view: where(), shown(), held() and marked() give a | ||
| 22 | # narrower view of the same records, and nothing is copied. | ||
| 23 | |||
| 24 | use v5.36; | ||
| 25 | use utf8; | ||
| 26 | |||
| 27 | our $VERSION = '0.2610090'; | ||
| 28 | |||
| 29 | my $WORD = 0xFFFFFFFF; # the hash below is in 32 bits, the same on every perl | ||
| 30 | |||
| 31 | # records a list of tables, one per record | ||
| 32 | # fields the names of their fields, in the order they are thought of | ||
| 33 | sub new ($class, %arg) { | ||
| 34 | my $records = $arg{records} // die "records => [ { field => value, ... }, ... ] is required\n"; | ||
| 35 | my $fields = $arg{fields} // [ sort keys %{ $records->[0] // {} } ]; | ||
| 36 | return bless { records => $records, fields => [@$fields], held => {}, mark => {} }, $class; | ||
| 37 | } | ||
| 38 | |||
| 39 | # From a file of tab-separated lines, one record a line. Lines that start | ||
| 40 | # with # are comments, and a missing last field is empty. | ||
| 41 | # fields the names of the columns | ||
| 42 | sub read ($class, $file, %arg) { | ||
| 43 | my $fields = $arg{fields} // die "fields => [ names of the columns ] is required\n"; | ||
| 44 | open my $in, '<:encoding(UTF-8)', $file or die "cannot read $file: $!\n"; | ||
| 45 | my @records; | ||
| 46 | while (my $line = <$in>) { | ||
| 47 | next if $line =~ /\A#/; | ||
| 48 | chomp $line; | ||
| 49 | my @values = split /\t/, $line, -1; | ||
| 50 | die "$file line $.: " . scalar(@values) . " fields where there are " . scalar(@$fields) . " names\n" if @values > @$fields; | ||
| 51 | push @records, { map { $fields->[$_] => $values[$_] // '' } 0 .. $#$fields }; | ||
| 52 | } | ||
| 53 | close $in; | ||
| 54 | return $class->new(records => \@records, fields => $fields); | ||
| 55 | } | ||
| 56 | |||
| 57 | # A view of some of these records, with what is known about them. | ||
| 58 | sub _view ($self, $records) { | ||
| 59 | return bless { %$self, records => $records }, ref $self; | ||
| 60 | } | ||
| 61 | |||
| 62 | sub records ($self) { return @{ $self->{records} } } | ||
| 63 | sub count ($self) { return scalar @{ $self->{records} } } | ||
| 64 | sub fields ($self) { return @{ $self->{fields} } } | ||
| 65 | |||
| 66 | # The values of one field, record by record. | ||
| 67 | sub values_of ($self, $field) { | ||
| 68 | $self->_known($field); | ||
| 69 | return map { $_->{$field} } @{ $self->{records} }; | ||
| 70 | } | ||
| 71 | |||
| 72 | sub _known ($self, @fields) { | ||
| 73 | my %have = map { $_ => 1 } @{ $self->{fields} }; | ||
| 74 | $have{$_} or die "there is no field '$_' (there are: @{ $self->{fields} })\n" for @fields; | ||
| 75 | return; | ||
| 76 | } | ||
| 77 | |||
| 78 | # A new field, worked out from each record: derive(name => sub ($record) { ... }). | ||
| 79 | sub derive ($self, %new) { | ||
| 80 | for my $field (sort keys %new) { | ||
| 81 | $_->{$field} = $new{$field}->($_) for @{ $self->{records} }; | ||
| 82 | push @{ $self->{fields} }, $field if !grep { $_ eq $field } @{ $self->{fields} }; | ||
| 83 | } | ||
| 84 | return $self; | ||
| 85 | } | ||
| 86 | |||
| 87 | # The records a sub accepts. | ||
| 88 | sub where ($self, $accept) { | ||
| 89 | return $self->_view([ grep { $accept->($_) } @{ $self->{records} } ]); | ||
| 90 | } | ||
| 91 | |||
| 92 | # A number from 0 up to 1 for a value: the same wherever the value occurs, | ||
| 93 | # and nothing like that of a value that differs by a letter (FNV-1a, stirred). | ||
| 94 | sub _place ($value) { | ||
| 95 | my $hash = 2166136261; | ||
| 96 | $hash = (($hash ^ ord) * 16777619) & $WORD for split //, $value; | ||
| 97 | $hash ^= $hash >> 15; | ||
| 98 | $hash = ($hash * 2246822519) & $WORD; | ||
| 99 | $hash ^= $hash >> 13; | ||
| 100 | return $hash / ($WORD + 1); | ||
| 101 | } | ||
| 102 | |||
| 103 | # Hold a share of the records out of training. Which ones follows from the | ||
| 104 | # value of a field (by => name, default the first field), so records that | ||
| 105 | # share it are held out together or not at all; `by` may also be a sub that | ||
| 106 | # gives the value for a record. The records of a mark can be exempted | ||
| 107 | # (never => name): what has to be right without exception is trained on. | ||
| 108 | sub hold_out ($self, $share, %arg) { | ||
| 109 | die "the share to hold out is a number above 0 and below 1, not '$share'\n" if !($share > 0 && $share < 1); | ||
| 110 | my $by = $arg{by} // $self->{fields}[0]; | ||
| 111 | $self->_known($by) if !ref $by; | ||
| 112 | my $never = defined $arg{never} ? $self->{mark}{ $arg{never} } // die "no records are marked '$arg{never}'; mark them before holding out\n" : {}; | ||
| 113 | %{ $self->{held} } = (); | ||
| 114 | for my $record (@{ $self->{records} }) { | ||
| 115 | next if $never->{$record}; | ||
| 116 | my $value = ref $by ? $by->($record) : $record->{$by}; | ||
| 117 | $self->{held}{$record} = 1 if _place($value) < $share; | ||
| 118 | } | ||
| 119 | return $self; | ||
| 120 | } | ||
| 121 | |||
| 122 | # Hold out the records a sub accepts, and no others: for data that has to be | ||
| 123 | # on the same sides as other data. | ||
| 124 | sub hold_out_if ($self, $accept) { | ||
| 125 | %{ $self->{held} } = map { $_ => 1 } grep { $accept->($_) } @{ $self->{records} }; | ||
| 126 | return $self; | ||
| 127 | } | ||
| 128 | |||
| 129 | # The records that are trained on, and those that are held out. | ||
| 130 | sub shown ($self) { return $self->_view([ grep { !$self->{held}{$_} } @{ $self->{records} } ]) } | ||
| 131 | sub held ($self) { return $self->_view([ grep { $self->{held}{$_} } @{ $self->{records} } ]) } | ||
| 132 | |||
| 133 | # Name a group of records: mark(core => sub ($record) { ... }). A record that | ||
| 134 | # is marked is not thereby trained on or held out; that is another question. | ||
| 135 | sub mark ($self, %group) { | ||
| 136 | for my $name (sort keys %group) { | ||
| 137 | $self->{mark}{$name} = { map { $_ => 1 } grep { $group{$name}->($_) } @{ $self->{records} } }; | ||
| 138 | } | ||
| 139 | return $self; | ||
| 140 | } | ||
| 141 | |||
| 142 | sub marks ($self) { return sort keys %{ $self->{mark} } } | ||
| 143 | |||
| 144 | # The records of a named group. | ||
| 145 | sub marked ($self, $name) { | ||
| 146 | my $group = $self->{mark}{$name} // die "no records are marked '$name' (marked are: @{[ $self->marks ]})\n"; | ||
| 147 | return $self->_view([ grep { $group->{$_} } @{ $self->{records} } ]); | ||
| 148 | } | ||
| 149 | |||
| 150 | sub is_marked ($self, $name, $record) { return ($self->{mark}{$name} // {})->{$record} ? 1 : 0 } | ||
| 151 | sub is_held ($self, $record) { return $self->{held}{$record} ? 1 : 0 } | ||
| 152 | |||
| 153 | # Some of the records, the same ones every time: at most $count of them, | ||
| 154 | # spread evenly over the whole. | ||
| 155 | sub sample ($self, $count) { | ||
| 156 | my $records = $self->{records}; | ||
| 157 | return $self->_view([@$records]) if $count >= @$records; | ||
| 158 | return $self->_view([ map { $records->[ int($_ * @$records / $count) ] } 0 .. $count - 1 ]); | ||
| 159 | } | ||
| 160 | |||
| 161 | # What a model learns from: for each record [input, answer, parameters ...]. | ||
| 162 | # from the field a model reads | ||
| 163 | # to the field it answers | ||
| 164 | # given the fields it is told beside, in order | ||
| 165 | sub pairs ($self, %arg) { | ||
| 166 | my $from = $arg{from} // die "from => the field a model reads is required\n"; | ||
| 167 | my $to = $arg{to} // die "to => the field a model answers is required\n"; | ||
| 168 | my @given = @{ $arg{given} // [] }; | ||
| 169 | $self->_known($from, $to, @given); | ||
| 170 | return [ map { [ @$_{ $from, $to, @given } ] } @{ $self->{records} } ]; | ||
| 171 | } | ||
| 172 | |||
| 173 | 1; | ||
| 174 | |||
| 175 | __END__ |