blib/lib/Lingua/Word/Parser.pm

filter: Statementstmt 88.8% 191 / 215 Branchbran 66.6% 40 / 60 Conditioncond 76.9% 10 / 13 MC/DCmcdc 62.5% 5 / 8 Subroutinesub 90.4% 19 / 21 Podpod 100% 5 / 5 totaltotal 83.8% 270 / 322 CC 54 SCAR 25.3
CC
54
Cov
84%
_db_fetch
72.0
power
9.0
_reconstruct
9.0
CRAP 12.5
↑ Summary
1 package Lingua::Word::Parser;
2 2 our $AUTHORITY = 'cpan:GENE';
3  
4 # ABSTRACT: Parse a word into scored known and unknown parts
5  
6 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 2 our $VERSION = '0.0901';
10  
11 2 ▶use Bit::Vector ();
Subroutine BEGIN: called CC=1 SCAR=0.0
12 2 ▶use DBI ();
Subroutine BEGIN: called CC=1 SCAR=0.0
13 2 ▶use IO::File ();
Subroutine BEGIN: called CC=1 SCAR=0.0
14  
15 2 ▶use Memoize qw(memoize);
Subroutine BEGIN: called CC=1 SCAR=0.0
16 2 memoize('_grouping');
17 2 memoize('_rle');
18  
19  
20  
21 2 ▶sub new {
Subroutine new: called CC=3 SCAR=11.2
Pod documented
22 2 my $class = shift;
23 2 my %args = @_;
24 my $self = {
25 file => $args{file},
26 dbhost => $args{dbhost} || 'localhost',
27 dbtype => $args{dbtype} || 'mysql',
28 dbname => $args{dbname},
29 dbuser => $args{dbuser},
30 dbpass => $args{dbpass},
31 lex => $args{lex},
32 word => $args{word},
33 2 ▶ known => {},
Condition50%2/4
$args{'dbhost'} || 'localhost'
l!l
02
$args{'dbtype'} || 'mysql'
l!l
02
MC/DC0%0/1
$args{'dbhost'} || 'localhost'
$args{'dbhost'}
MC/DC0%0/1
$args{'dbtype'} || 'mysql'
$args{'dbtype'}
Truth table50%1/2
A
A= $args{'dbhost'}
Aresult
11
00
Truth table50%1/2
A
A= $args{'dbtype'}
Aresult
11
00
34 masks => {},
35 combos => [],
36 score => {},
37 };
38 2 bless $self, $class;
39 2 $self->_init(%args);
40 2 return $self;
41 }
42 2 ▶sub _init {
Subroutine _init: called CC=4 SCAR=14.5
43 2 my ($self, %args) = @_;
44  
45 # Set the length of our word.
46 2 $self->{wlen} = length $self->{word};
47  
48 # Set lex if given data.
49 2 ▶ if ( $self->{file} && -e $self->{file} ) {
Branch: 2 branches
BranchTrueFalseTotal
if ($self->{'file'} and -e $self->{'file'}) { } 1 1 2
elsif ($self->{'dbname'}) { } 0 1 1
Condition66%2/3
$self->{"file"} and -e $self->{"file"}
!ll&&!rl&&r
101
MC/DC50%1/2
$self->{"file"} and -e $self->{"file"}
$self->{"file"} -e $self->{"file"}
Truth table66%2/3
A and B
A= $self->{"file"}B= -e $self->{"file"}
ABresult
0X0
100
111
50 1 $self->_fetch_lex;
51 }
52 elsif( $self->{dbname} )
53 {
54 0 $self->_db_fetch;
55 }
56 }
57  
58 1 ▶sub _fetch_lex {
Subroutine _fetch_lex: called CC=3 SCAR=11.0
59 1 my $self = shift;
60  
61 1 my $i = 0;
62  
63 # Open the given file for reading...
64 1 my $fh = IO::File->new;
65 1 ▶ $fh->open($self->{file}, '<') or die "Can't read file: '$self->{file}'";
Branch: 1 branch
BranchTrueFalseTotal
$fh->open($self->{"file"}, "<") or die qq[Can't read file: '$self->{"file"}'] 1 0 1
66 1 for ( <$fh> ) {
67 9 $i++;
68 # Split space-separated entries.
69 9 chomp;
70 9 my ($re, $defn) = split /\s+/, $_, 2;
71 # Add the entry to the lexicon.
72 9 $self->{lex}{$i} = { defn => $defn, re => qr/$re/ };
73 }
74 1 $fh->close;
75  
76 1 return $self->{lex};
77 }
78  
79 0 ▶sub _db_fetch {
Subroutine _db_fetch: not called CC=8 SCAR=42.8
80 0 my $self = shift;
81  
82 0 my $dsn;
83 0 ▶ if (lc($self->{dbtype}) eq 'sqlite') {
Branch: 1 branch
BranchTrueFalseTotal
if (lc $self->{'dbtype'} eq 'sqlite') { } 0 0 0
84 0 $dsn = "DBI:$self->{dbtype}:dbname=$self->{dbname};$self->{dbhost}";
85 }
86 else {
87 0 $dsn = "DBI:$self->{dbtype}:$self->{dbname};$self->{dbhost}";
88 }
89  
90 0 ▶ my $dbh = DBI->connect( $dsn, $self->{dbuser}, $self->{dbpass}, { RaiseError => 1, AutoCommit => 1 } )
Branch: 1 branch
BranchTrueFalseTotal
my $dbh = "DBI"->connect($dsn, $self->{"dbuser"}, $self->{"dbpass"}, {"RaiseError", 1, "AutoCommit", 1}) or die qq[Unable to connect to $self->{"dbname"}: $DBI::errstr\n] 0 0 0
91 or die "Unable to connect to $self->{dbname}: $DBI::errstr\n";
92  
93 0 my $sql = 'SELECT id, affix, definition FROM fragment';
94  
95 0 my $sth = $dbh->prepare($sql);
96 0 ▶ $sth->execute or die "Unable to execute '$sql': $DBI::errstr\n";
Branch: 1 branch
BranchTrueFalseTotal
$sth->execute or die "Unable to execute '${sql}': $DBI::errstr\n" 0 0 0
97  
98 0 ▶ while( my @row = $sth->fetchrow_array ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my(@row) = $sth->fetchrow_array) 0 0 0
99 0 my ($id, $part, $defn) = @row;
100 0 $self->{lex}{$id} = { re => qr/$part/, defn => $defn };
101 }
102 0 ▶ die "Fetch terminated early: $DBI::errstr\n" if $DBI::errstr;
Branch: 1 branch
BranchTrueFalseTotal
if $DBI::errstr 0 0 0
103  
104 0 ▶ $sth->finish or die "Unable to finish '$sql': $DBI::errstr\n";
Branch: 1 branch
BranchTrueFalseTotal
$sth->finish or die "Unable to finish '${sql}': $DBI::errstr\n" 0 0 0
105  
106 0 ▶ $dbh->disconnect or die "Unable to disconnect from $self->{dbname}: $DBI::errstr\n";
Branch: 1 branch
BranchTrueFalseTotal
$dbh->disconnect or die qq[Unable to disconnect from $self->{"dbname"}: $DBI::errstr\n] 0 0 0
107 }
108  
109  
110 1 ▶sub knowns {
Subroutine knowns: called CC=4 SCAR=13.9
Pod documented
111 1 my $self = shift;
112  
113 # The identifier for the known and masks lists.
114 1 my $id = 0;
115  
116 1 for my $i (values %{ $self->{lex} }) {
117 9 ▶ while ($self->{word} =~ /$i->{re}/g) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->{"word"} =~ /$i->{"re"}/g) 10 9 19
118 # Match positions.
119 10 my ($m, $n) = ($-[0], $+[0]);
120 # Get matched word-part.
121 10 my $part = substr $self->{word}, $m, $n - $m;
122  
123 10 ▶ next if $n == $m; # skip zero-width matches instead of padding the mask
Branch: 1 branch
BranchTrueFalseTotal
if $n == $m 0 10 10
124  
125 # Create the part-of-word bitmask.
126 10 my $mask = 0 x $m; # Before known
127 10 $mask .= 1 x ($n - $m); # Known part
128 10 $mask .= 0 x ($self->{wlen} - $n); # After known
129  
130 # Output our progress.
131 # warn sprintf "%s %s - %s, %s (%d %d), %s\n",
132 # $mask,
133 # $i->{re},
134 # substr($self->{word}, 0, $m),
135 # $part,
136 # $m,
137 # $n - 1,
138 # substr($self->{word}, $n),
139 # ;
140  
141 # Save the known as a member of a list keyed by starting position.
142 $self->{known}{$id} = {
143 part => $part,
144 span => [$m, $n - 1],
145 defn => $i->{defn},
146 10 mask => $mask,
147 };
148  
149 # Save the relationship between mask and id.
150 10 $self->{masks}{$mask} = $id++;
151 }
152 }
153  
154 1 return $self->{known};
155 }
156  
157  
158 1 ▶sub power {
Subroutine power: called CC=9 SCAR=22.0
Pod documented
159 1 my $self = shift;
160  
161 # Precompute each mask's [start, end) span once
162 1 my @spans;
163 1 for my $mask (keys %{ $self->{masks} }) {
164 10 $mask =~ /^(0*)(1+)/;
165 10 push @spans, { start => length($1), end => length($1) + length($2), mask => $mask };
166 }
167 1 @spans = sort { $a->{start} <=> $b->{start} } @spans;
168  
169 # Recursively extend only with spans that don't overlap what's already chosen
170 1 my @combos;
171 my $extend;
172 216 ▶ $extend = sub {
Subroutine __ANON__: called CC=5 SCAR=16.1
173 216 my ($from_idx, $chosen) = @_;
174 216 ▶ push @combos, [ map { $_->{mask} } @$chosen ] if @$chosen;
Branch: 1 branch
BranchTrueFalseTotal
if @$chosen 215 1 216
175 216 for my $idx ($from_idx .. $#spans) {
176 306 my $span = $spans[$idx];
177 306 ▶ next if @$chosen && $span->{start} < $chosen->[-1]{end};
Branch: 1 branch
BranchTrueFalseTotal
if @$chosen and $span->{"start"} < $chosen->[-1]{"end"} 91 215 306
Condition100%3/3
@$chosen and $span->{"start"} < $chosen->[-1]{"end"}
!ll&&!rl&&r
1020591
MC/DC100%2/2
@$chosen and $span->{"start"} < $chosen->[-1]{"end"}
@$chosen $span->{"start"} < $chosen->[-1]{"end"}
Truth table100%3/3
A and B
A= @$chosenB= $span->{"start"} < $chosen->[-1]{"end"}
ABresult
0X0
100
111
178 215 $extend->($idx + 1, [ @$chosen, $span ]);
179 }
180 1 };
181 1 $extend->(0, []);
182  
183 # Drop non-maximal combos: if some known span could still be added to a
184 # combo without overlapping anything already in it, that combo is a
185 # strictly weaker duplicate and shouldn't be scored on its own.
186 1 my @maximal;
187 1 COMBO: for my $combo (@combos) {
188 215 my %chosen = map { $_ => 1 } @$combo;
189 768 my @chosen_spans = sort { $a->{start} <=> $b->{start} }
190 215 grep { $chosen{ $_->{mask} } } @spans;
191  
192 215 for my $span (@spans) {
193 821 ▶ next if $chosen{ $span->{mask} };
Branch: 1 branch
BranchTrueFalseTotal
if $chosen{$span->{"mask"}} 282 539 821
194 539 my $overlaps = 0;
195 539 for my $cs (@chosen_spans) {
196 1218 ▶ if ( $span->{start} < $cs->{end} && $cs->{start} < $span->{end} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($span->{"start"} < $cs->{"end"} and $cs->{"start"} < $span->{"end"}) 336 882 1218
Condition100%3/3
$span->{"start"} < $cs->{"end"} and $cs->{"start"} < $span->{"end"}
!ll&&!rl&&r
444438336
MC/DC100%2/2
$span->{"start"} < $cs->{"end"} and $cs->{"start"} < $span->{"end"}
$span->{"start"} < $cs->{"end"} $cs->{"start"} < $span->{"end"}
Truth table100%3/3
A and B
A= $span->{"start"} < $cs->{"end"}B= $cs->{"start"} < $span->{"end"}
ABresult
0X0
100
111
197 336 $overlaps = 1;
198 336 last;
199 }
200 }
201 539 ▶ next COMBO unless $overlaps; # an addable span exists -- not maximal
Branch: 1 branch
BranchTrueFalseTotal
unless $overlaps 203 336 539
202 }
203 12 push @maximal, $combo;
204 }
205  
206 1 $self->{combos} = \@maximal;
207  
208 1 return $self->{combos};
209 }
210  
211  
212 1 ▶sub score {
Subroutine score: called CC=3 SCAR=11.0
Pod documented
213 1 my $self = shift;
214 1 my ( $open_separator, $close_separator ) = @_;
215  
216 1 my $parts = $self->score_parts( $open_separator, $close_separator );
217  
218 1 for my $mask ( keys %$parts ) {
219 8 my $familiarity = sprintf "%.2f chunks / %.2f chars", @{ $self->_familiarity($mask) };
220  
221 8 for my $element ( @{ $parts->{$mask} } ) {
222 my $score = sprintf "%d:%d chunks / %d:%d chars",
223 $element->{score}{knowns}, $element->{score}{unknowns},
224 24 $element->{score}{knownc}, $element->{score}{unknownc};
225  
226 24 my $part = join ', ', @{ $element->{partition} };
227  
228 24 my $defn = join ', ', @{ $element->{definition} };
229  
230 24 push @{ $self->{score}{$mask} }, {
231 score => $score,
232 familiarity => $familiarity,
233 partition => $part,
234 definition => $defn,
235 };
236 }
237 }
238  
239 1 return $self->{score};
240 }
241  
242 32 ▶sub _familiarity {
Subroutine _familiarity: called CC=4 SCAR=13.9
243 32 my ( $self, $mask ) = @_;
244  
245 32 my @chunks = grep { $_ ne "" } split /(0+)/, $mask;
246  
247 # Figure out how many chars are only 1s and
248 # Figure out how many chunks are made up of 1s:
249 32 my $char_1s = 0;
250 32 my $chunk_1s = 0;
251 32 for my $chunk (@chunks) {
252 120 ▶ $char_1s += $chunk =~ /0/ ? 0 : length($chunk);
Branch: 1 branch
BranchTrueFalseTotal
$chunk =~ /0/ ? : 44 76 120
253 120 ▶ $chunk_1s += $chunk =~ /0/ ? 0 : 1;
Branch: 1 branch
BranchTrueFalseTotal
$chunk =~ /0/ ? : 44 76 120
254 }
255  
256 32 return [ $chunk_1s / @chunks, $char_1s / length($mask) ];
257 }
258  
259  
260 2 ▶sub score_parts {
Subroutine score_parts: called CC=7 SCAR=19.5
Pod documented
261 2 my $self = shift;
262 2 my ( $open_separator, $close_separator, $line_terminator ) = @_;
263  
264 2 ▶ $line_terminator = '' unless defined $line_terminator;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $line_terminator 2 0 2
265  
266 # Visit each combination...
267 2 my $i = 0;
268 2 for my $c (@{ $self->{combos} }) {
269 24 $i++;
270 24 my $together = $self->_or_together(@$c);
271  
272 # Breakdown knowns vs unknowns and knowncharacters vs unknowncharacters.
273 24 my %count = (
274 knowns => 0,
275 unknowns => 0,
276 knownc => 0,
277 unknownc => 0,
278 );
279  
280 24 for my $x ( reverse sort @$c ) {
281 # Run-length encode an "un-digitized" string.
282 128 my $y = _rle($x);
283 128 my ( $knowns, $unknowns, $knownc, $unknownc ) = _grouping($y);
284 # Accumulate the counters!
285 128 $count{knowns} += $knowns;
286 128 $count{unknowns} += $unknowns;
287 128 $count{knownc} += $knownc;
288 128 $count{unknownc} += $unknownc;
289 }
290  
291 24 my ( $s, $m ) = _reconstruct( $self->{word}, $c, $open_separator, $close_separator );
292  
293 24 my $defn = [];
294 24 for my $k ( @$m )
295 {
296 128 for my $j ( keys %{ $self->{known} } )
297 {
298 1280 ▶ push @$defn, $self->{known}{$j}{defn} if $self->{known}{$j}{mask} eq $k;
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"known"}{$j}{"mask"} eq $k 128 1152 1280
299 }
300 }
301  
302 24 push @{ $self->{score_parts}{$together} }, {
303 score => \%count,
304 partition => $s,
305 definition => $defn,
306 familiarity => $self->_familiarity($together),
307 };
308 }
309  
310 2 return $self->{score_parts};
311 }
312  
313 10 ▶sub _grouping {
Subroutine _grouping: called CC=4 SCAR=13.9
314 10 my $scored = shift;
315 10 my @groups = $scored =~ /([ku]\d+)/g;
316 10 my ( $knowns, $unknowns ) = ( 0, 0 );
317 10 my ( $knownc, $unknownc ) = ( 0, 0 );
318 10 for ( @groups ) {
319 26 ▶ if ( /k(\d+)/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if (/k(\d+)/) 10 16 26
320 10 $knowns++;
321 10 $knownc += $1;
322 }
323 26 ▶ if ( /u(\d+)/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if (/u(\d+)/) 16 10 26
324 16 $unknowns++;
325 16 $unknownc += $1;
326 }
327 }
328 10 return $knowns, $unknowns, $knownc, $unknownc;
329 }
330  
331 10 ▶sub _rle {
Subroutine _rle: called CC=1 SCAR=0.0
332 10 my $scored = shift;
333 # Run-length encode an "un-digitized" string.
334 10 $scored =~ s/1/k/g; # Undigitize
335 10 $scored =~ s/0/u/g; # "
336 # Count contiguous chars.
337 10 $scored =~ s/(.)\1*/$1 . length(substr($scored, $-[0], $+[0]-$-[0]))/ge;
338 10 return $scored;
339 }
340  
341 0 ▶sub _does_not_overlap {
Subroutine _does_not_overlap: not called CC=2 SCAR=17.9
342 0 my $self = shift;
343  
344 # Get our masks to check.
345 0 my ($mask, $check) = @_;
346  
347 # Create the bitstrings to compare.
348 0 my $bitmask = Bit::Vector->new_Bin($self->{wlen}, $mask);
349 0 my $orclone = Bit::Vector->new_Bin($self->{wlen}, $check);
350 0 my $xorclone = Bit::Vector->new_Bin($self->{wlen}, $check);
351  
352 # Compute or and xor for the strings.
353 0 $orclone->Or($bitmask, $orclone);
354 0 $xorclone->Xor($bitmask, $xorclone);
355  
356 # Return the "or & xor equivalent sibling."
357 0 ▶ return $xorclone->equal($orclone) ? $orclone->to_Bin : 0;
Branch: 1 branch
BranchTrueFalseTotal
$xorclone->equal($orclone) ? : 0 0 0
358 }
359  
360 24 ▶sub _or_together {
Subroutine _or_together: called CC=2 SCAR=6.9
361 24 my $self = shift;
362  
363 # Get our masks to score.
364 24 my @masks = @_;
365  
366 # Initialize the bitmask to return, to zero.
367 24 my $result = Bit::Vector->new_Bin($self->{wlen}, (0 x $self->{wlen}));
368  
369 24 for my $mask (@masks) {
370 # Create the bitstrings to compare.
371 128 my $bitmask = Bit::Vector->new_Bin($self->{wlen}, $mask);
372  
373 # Get the union of the bit strings.
374 128 $result->Or($result, $bitmask);
375 }
376  
377 # Return the "or sum."
378 24 return $result->to_Bin;
379 }
380  
381 24 ▶sub _reconstruct {
Subroutine _reconstruct: called CC=9 SCAR=22.0
382 24 my ( $word, $masks, $open_separator, $close_separator ) = @_;
383  
384 24 ▶ $open_separator = '<' unless defined $open_separator;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $open_separator 12 12 24
385 24 ▶ $close_separator = '>' unless defined $close_separator;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $close_separator 12 12 24
386  
387 24 my $strings = [];
388 24 my $my_masks = [];
389  
390 24 for my $mask (reverse sort @$masks) {
391 128 my $i = 0;
392 128 my $last = 0;
393 128 my $string = '';
394 128 for my $m ( split //, $mask ) {
395 1280 ▶ if ( $m ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($m) { } 208 1072 1280
396 208 ▶ $string .= $open_separator unless $last;
Branch: 1 branch
BranchTrueFalseTotal
unless $last 128 80 208
397 208 $string .= substr( $word, $i, 1 );
398 208 $last = 1;
399 }
400 else {
401 1072 ▶ $string .= $close_separator if $last;
Branch: 1 branch
BranchTrueFalseTotal
if $last 104 968 1072
402 1072 $string .= substr( $word, $i, 1 );
403 1072 $last = 0;
404 }
405 1280 $i++;
406 }
407 128 ▶ $string .= $close_separator if $last;
Branch: 1 branch
BranchTrueFalseTotal
if $last 24 104 128
408 128 push @$strings, $string;
409 128 push @$my_masks, $mask;
410 }
411  
412 24 return $strings, $my_masks;
413 }
414  
415 2 1;
416  
417 __END__
↑ Summary