CAB/Analyzer/Automaton/Dyn.pm untested

filter: Statementstmt 0.0% 0 / 126 Branchbran 0.0% 0 / 56 Conditioncond 0.0% 0 / 79 MC/DCmcdc 0.0% 0 / 51 Subroutinesub 0.0% 0 / 24 Podpod 0.0% 0 / 17 totaltotal 0.0% 0 / 353 CC 29 SCAR 38.6
CC
29
Cov
0%
dynSubCode
156.0
load
30.0
loadLabels
30.0
CRAP 47.3
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Automaton::Dyn.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: generic analysis automaton API
6  
7 package DTA::CAB::Analyzer::Automaton::Dyn;
8 use DTA::CAB::Analyzer::Dyn;
9 use DTA::CAB::Unify ':all';
10 use Gfsm;
11 use Encode qw(encode decode);
12 use IO::File;
13 #use File::Basename qw();
14 use Carp;
15  
16 use strict;
17  
18 ##==============================================================================
19 ## Globals
20 ##==============================================================================
21  
22 our @ISA = qw(DTA::CAB::Analyzer::Dyn);
23  
24 ## $DEFAULT_ANALYZE_GET
25 ## + code string for {analyzeGet}
26 ## + eval()d in list context, may return multiples
27 ## + available vars:
28 ## $tok => token object being analyzed
29 ## $aut => analyzer (automaton)
30 #our $DEFAULT_ANALYZE_GET = '$_[0]{xlit} ? $_[0]{xlit}{latin1Text} : $_[0]{text}';
31 our $DEFAULT_ANALYZE_GET = '$tok->{xlit} ? $tok->{xlit}{latin1Text} : $tok->{text}';
32  
33 ## $DEFAULT_ANALYZE_SET
34 ## + default code string for {analyzeSet}
35 ## + available vars:
36 ## $tok => token object being analyzed
37 ## $a => analyses (array-ref, maybe blessed)
38 ## $aut => analyzer (automaton)
39 #our $DEFAULT_ANALYZE_SET = '$_[0]{$anl->{label}}=$_[1]';
40 our $DEFAULT_ANALYZE_SET = '$tok->{$aut->{label}}=$wa';
41  
42 ##==============================================================================
43 ## Constructors etc.
44 ##==============================================================================
45  
46 ## $obj = CLASS_OR_OBJ->new(%args)
47 ## + object structure:
48 ## (
49 ## ##-- Filename Options
50 ## fstFile => $filename, ##-- source FST file (default: none)
51 ## labFile => $filename, ##-- source labels file (default: none)
52 ## dictFile => $filename, ##-- source dict file (default: none): clobbers $dict->{dictFile} if defined
53 ##
54 ## ##-- Exception lexicon options
55 ## dict => $dict, ##-- exception lexicon as a DTA::CAB::Analyzer::Dict object or option hash
56 ## ## + default=undef
57 ## dictClass => $class, ##-- fallback class for new dict (default='DTA::CAB::Analyzer::Dict')
58 ##
59 ## ##-- Analysis Output
60 ## analyzeGet => $code, ##-- accessor: coderef or string: source text (default=$DEFAULT_ANALYZE_GET; return undef for no analysis)
61 ## analyzeSet => $code, ##-- accessor: coderef or string: set analyses (default=$DEFAULT_ANALYZE_SET)
62 ## wantAnalysisLo => $bool, ##-- set to true to include 'lo' keys in analyses (default: true)
63 ## wantAnalysisLemma => $bool, ##-- set to true to include 'lemma' keys in analyses (default: false)
64 ##
65 ## ##-- Analysis Options
66 ## eow => $sym, ##-- EOW symbol for analysis FST
67 ## check_symbols => $bool, ##-- check for unknown symbols? (default=1)
68 ## labenc => $enc, ##-- encoding of labels file (default='auto')
69 ## #dictenc => $enc, ##-- dictionary encoding (default='UTF-8') (set $aut->{dict}{encoding} instead)
70 ## auto_connect => $bool, ##-- whether to call $result->_connect() after every lookup (default=0)
71 ## tolower => $bool, ##-- if true, all input words will be bashed to lower-case (default=0)
72 ## tolowerNI => $bool, ##-- if true, all non-initial characters of inputs will be lower-cased (default=0)
73 ## toupperI => $bool, ##-- if true, initial character will be upper-cased (default=0)
74 ## bashWS => $str, ##-- if defined, input whitespace will be bashed to '$str' (default='_')
75 ## attInput => $bool, ##-- if true, respect AT&T lextools-style escapes in input (default=0)
76 ## allowTextRegex => $re, ##-- if defined, only tokens with matching 'text' will be analyzed (default: none)
77 ## ## : useful: /^(?:(?:[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+)|(?:[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$/
78 ## ## : == DTA::CAB::Analyzer::_am_wordlike_regex()
79 ##
80 ## ##-- Analysis objects
81 ## fst => $gfst, ##-- (child classes only) e.g. a Gfsm::Automaton::Dyn object (default=new)
82 ## lab => $lab, ##-- (child classes only) e.g. a Gfsm::Alphabet object (default=new)
83 ## labh => \%sym2lab, ##-- (?) label hash: $sym2lab{$labSym} = $labId;
84 ## laba => \@lab2sym, ##-- (?) label array: $lab2sym[$labId] = $labSym;
85 ## labc => \@chr2lab, ##-- (?)chr-label array: $chr2lab[ord($chr)] = $labId;, by unicode char number (e.g. unpack('U0U*'))
86 ## result=>$resultfst, ##-- (child classes only) e.g. result fst
87 ##
88 ## ##-- INHERITED from DTA::CAB::Analyzer
89 ## label => $label, ##-- analyzer label (default: from analyzer class name)
90 ## typeKeys => \@keys, ##-- type-wise keys to expand
91 ## )
92 sub new {
93 my $that = shift;
94 my $aut = $that->SUPER::new(
95 ##-- filenames
96 fstFile => undef,
97 labFile => undef,
98 dictFile => undef,
99  
100 ##-- analysis objects
101 fst=>undef,
102 lab=>undef,
103 result=>undef,
104 labh=>{},
105 laba=>[],
106 labc=>[],
107 dict=>undef,
108 dictClass=>'DTA::CAB::Analyzer::Dict',
109  
110 ##-- options
111 eow =>'',
112 check_symbols => 1,
113 labenc => 'auto',
114 #dictenc => 'utf8',
115 auto_connect => 0,
116 tolower => 0,
117 tolowerNI => 0,
118 toupperI => 0,
119 bashWS => '_',
120 attInput => 0,
121 allowTextRegex => undef, #DTA::CAB::Analyzer::_am_wordlike_regex(),
122  
123 ##-- analysis I/O
124 analyzeSrc => 'text',
125 wantAnalysisLo => 1,
126 wantAnalysisLemma => 0,
127  
128 ##-- user args
129 @_
130 );
131 return $aut;
132 }
133  
134 ## $aut = $aut->clear()
135 sub clear {
136 my $aut = shift;
137  
138 ##-- analysis sub(s)
139 $aut->dropClosures();
140  
141 ##-- analysis objects
142 delete($aut->{fst});
143 delete($aut->{lab});
144 delete($aut->{result});
145 %{$aut->{labh}} = qw();
146 @{$aut->{laba}} = qw();
147 @{$aut->{labc}} = qw();
148  
149 return $aut;
150 }
151  
152  
153 ##==============================================================================
154 ## Methods: Generic
155 ##==============================================================================
156  
157 ## $class = $aut->fstClass()
158 ## + default FST class for loadFst() method
159 sub fstClass { return 'Gfsm::Automaton::Dyn'; }
160  
161 ## $class = $aut->labClass()
162 ## + default labels class for loadLabels() method
163 sub labClass { return 'Gfsm::Alphabet'; }
164  
165 ## $bool = $aut->fstOk()
166 ## + should return false iff fst is undefined or "empty"
167 sub fstOk { return defined($_[0]{fst}) && $_[0]{fst}->n_states>0; }
168  
169 ## $bool = $aut->labOk()
170 ## + should return false iff label-set is undefined or "empty"
171 sub labOk { return defined($_[0]{lab}) && $_[0]{lab}->size>0; }
172  
173 ## $bool = $aut->dictOk()
174 ## + should return false iff dict is undefined or "empty"
175 sub dictOk { return $_[0]{dict} && $_[0]{dict}->dictOk; }
176  
177 ##==============================================================================
178 ## Methods: I/O
179 ##==============================================================================
180  
181 ##--------------------------------------------------------------
182 ## Methods: I/O: Input: all
183  
184 ## $bool = $aut->ensureLoaded()
185 ## + ensures automaton data is loaded from default files
186 sub ensureLoaded {
187 my $aut = shift;
188 my $rc = 1;
189 ##-- ensure: fst
190 if ( defined($aut->{fstFile}) && !$aut->fstOk ) {
191 $rc &&= $aut->loadFst($aut->{fstFile});
192 }
193 ##-- ensure: lab
194 if ( defined($aut->{labFile}) && !$aut->labOk ) {
195 $rc &&= $aut->loadLabels($aut->{labFile});
196 }
197 ##-- ensure: dict
198 if ( (defined($aut->{dictFile}) || ($aut->{dict} && $aut->{dict}{dictFile})) && !$aut->dictOk ) {
199 $rc &&= $aut->loadDict();
200 }
201 ##-- ensure: closures
202 $rc &&= $aut->ensureDynSubs();
203 return $rc;
204 }
205  
206 ## $aut = $aut->load(fst=>$fstFile, lab=>$labFile, dict=>$dictFile)
207 sub load {
208 my ($aut,%args) = @_;
209 return 0 if (!grep {defined($_)} @args{qw(fst lab dict)});
210 my $rc = $aut;
211 $rc &&= $aut->loadFst($args{fst}) if (defined($args{fst}));
212 $rc &&= $aut->loadLabels($args{lab}) if (defined($args{lab}));
213 $rc &&= $aut->loadDict($args{dict}) if (defined($args{dict}));
214 return $rc;
215 }
216  
217 ##--------------------------------------------------------------
218 ## Methods: I/O: Input: FST
219  
220 ## $aut = $aut->loadFst($fstfile)
221 sub loadFst {
222 my ($aut,$fstfile) = @_;
223 $aut->info("loading FST file '$fstfile'");
224 $aut->{fst} = $aut->fstClass->new() if (!defined($aut->{fst}));
225 $aut->{fst}->load($fstfile)
226 or $aut->logconfess("loadFst(): load failed for '$fstfile': $!");
227 $aut->{result} = $aut->{fst}->shadow; #if (defined($aut->{result}) && $aut->{fst}->can('shadow'));
228 delete($aut->{_analyze});
229 return $aut;
230 }
231  
232 ##--------------------------------------------------------------
233 ## Methods: I/O: Input: Labels
234  
235 ## $aut = $aut->loadLabels($labfile)
236 sub loadLabels {
237 my ($aut,$labfile) = @_;
238 $aut->info("loading labels file '$labfile'");
239 $aut->{lab} = $aut->labClass->new() if (!defined($aut->{lab}));
240 $aut->{lab}->load($labfile)
241 or $aut->logconfess("loadLabels(): load failed for '$labfile': $!");
242 if (!$aut->{labenc} || $aut->{labenc} eq 'auto') {
243 ##-- guess label encoding
244 my $buf = join('',@{$aut->{lab}->toArray});
245 $aut->{labenc} = utf8::decode($buf) ? 'utf8' : 'latin1';
246 $aut->debug("loadLabels(): guessed label encoding '$aut->{labenc}'");
247 }
248 $aut->{lab}->utf8(1)
249 if ($aut->{lab}->can('utf8') && (($aut->{labenc}||'') =~ /^utf\-?8$/i));
250 $aut->parseLabels();
251 delete($aut->{_analyze});
252 return $aut;
253 }
254  
255 ## $aut = $aut->parseLabels()
256 ## + sets up $aut->{labh}, $aut->{laba}, $aut->{labc}
257 ## + fixes encoding difficulties in $aut->{labh}, $aut->{laba}
258 sub parseLabels {
259 my $aut = shift;
260 my $laba = $aut->{laba};
261 @$laba = @{$aut->{lab}->asArray};
262 my ($i);
263 foreach $i (grep { defined($laba->[$_]) } 0..$#$laba) {
264 $laba->[$i] = decode($aut->{labenc}, $laba->[$i]) if ($aut->{labenc});
265 $aut->{labh}{$laba->[$i]} = $i;
266 }
267 ##-- setup labc: $labId = $labc->[ord($c)]; ##-- single unicode characater
268 ## : @labIds = @$labc[unpack('U0U*',$s)]; ##-- batch lookup for strings (fast)
269 my @csyms = grep {defined($_) && length($_)==1} @$laba; ##-- @csyms = ($sym1, ...) s.t. each sym has len==1
270 @{$aut->{labc}}[map {ord($_)} @csyms] = @{$aut->{labh}}{@csyms};
271 ##
272 return $aut;
273 }
274  
275 ##--------------------------------------------------------------
276 ## Methods: I/O: Input: Dictionary
277  
278 ## $aut = $aut->loadDict()
279 ## $aut = $aut->loadDict($dictfile)
280 sub loadDict {
281 my ($aut,$dictfile) = @_;
282 $dictfile = $aut->{dictFile} if (!defined($dictfile));
283 $dictfile = $aut->{dict}{dictFile} if (!defined($dictfile));
284 return $aut if (!defined($dictfile)); ##-- no dict file to load
285 $aut->info("loading exception lexicon from '$dictfile'");
286  
287 ##-- sanitize dict object
288 my $dclass = (ref($aut->{dict})||$aut->{dictClass}||'DTA::CAB::Analyzer::Dict');
289 my $dict = $aut->{dict} = bless(_unifyClobber($dclass->new,$aut->{dict},undef), $dclass);
290 $dict->{label} = $aut->{label}."_dict"; ##-- force sub-analyzer label
291 $dict->{dictFile} = $dictfile; ##-- clobber sub-analyzer file
292  
293 ##-- load dict object
294 $dict->ensureLoaded();
295 return undef if (!$dict->dictOk);
296 return $aut;
297 }
298  
299 ##==============================================================================
300 ## Methods: Persistence
301 ##==============================================================================
302  
303 ##======================================================================
304 ## Methods: Persistence: Perl
305  
306 ## @keys = $class_or_obj->noSaveKeys()
307 ## + returns list of keys not to be saved
308 sub noSaveKeys {
309 my $that = shift;
310 return ($that->SUPER::noSaveKeys, qw(dict fst lab laba labc labh result));
311 }
312  
313 ## $saveRef = $obj->savePerlRef()
314 ## + inherited from DTA::CAB::Persistent
315  
316 ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref)
317 ## + implicitly calls $obj->clear()
318 sub loadPerlRef {
319 my ($that,$ref) = @_;
320 my $obj = $that->SUPER::loadPerlRef($ref);
321 $obj->clear();
322 return $obj;
323 }
324  
325 ##==============================================================================
326 ## Methods: Analysis
327 ##==============================================================================
328  
329 ##------------------------------------------------------------------------
330 ## Methods: Analysis: Generic
331  
332 ## $bool = $anl->canAnalyze()
333 ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty)
334 sub canAnalyze {
335 return $_[0]->dictOk || ($_[0]->labOk && $_[0]->fstOk);
336 }
337  
338  
339 ##==============================================================================
340 ## Methods: Analysis: v1.x
341 ##==============================================================================
342  
343 ##------------------------------------------------------------------------
344 ## Methods: Analysis: v1.x: dynamic (newer)
345  
346 sub dynSubCode {
347 my ($anl,$which) = @_;
348 return $anl->SUPER::dynSubCode($which) if ($which ne 'Types');
349  
350 my $aget = $anl->{analyzeGet} || $DEFAULT_ANALYZE_GET;
351 my $aset = $anl->{analyzeSet} || $DEFAULT_ANALYZE_SET;
352  
353 return 'sub {
354 my ($aut,$doc,$types,$opts) = @_;
355 $types = $doc->types if (!$types);
356  
357 ##-- common vars
358 my $dict = $aut->dictOk ? $aut->{dict}->dictHash : undef;
359 my $fst = $aut->{fst};
360 my $fst_ok = $aut->fstOk();
361 my $result = $aut->{result};
362 my $lab = $aut->{lab};
363 my $labc = $aut->{labc};
364 my $laba = $aut->{laba};
365 my $labenc = $aut->{labenc};
366 my @eowlab = (defined($aut->{eow}) && $aut->{eow} ne "" ? ($aut->{labh}{$aut->{eow}}) : qw());
367  
368 ##-- ananalysis options
369 my @analyzeOptionKeys = (qw(check_symbols auto_connect),
370 #qw(tolower tolowerNI toupperI bashWS attInput),
371 qw(wantAnalysisLo wantAnalysisLemma),
372 qw(max_paths max_weight max_ops),
373 );
374  
375 my ($tok,@w,$w,$wa);
376 my ($uword,$ua,$ulword,@wlabs,$lemma);
377  
378 foreach $tok (values(%$types)) {
379 '.($anl->{allowTextRegex} ? ('next if ($tok->{text} !~ /'.$anl->{allowTextRegex}.'/);') : '').' ##-- dyn:allowTextRegex
380 @w = grep {defined($_)} '.$aget.'; ##-- dyn:analyzeGet
381 next if (!@w); ##-- accessor returned undef: skip this token
382 $wa = [];
383 foreach $w (@w) {
384 ##-- BEGIN analyzeWord
385 $opts->{src} = $w; ##-- set $opts->{src} (hack for setLookupOptions())
386  
387 ##-- set default options
388 $opts->{$_} = $aut->{$_} foreach (grep {!defined($opts->{$_})} @analyzeOptionKeys);
389 $aut->setLookupOptions($opts) if ($aut->can("setLookupOptions"));
390  
391 ##---- analyze
392 ##-- normalize word
393 '.($anl->{tolower}
394 ? '$uword = lc($w); ##-- dyn:tolower'
395 : ($anl->{tolowerNI}
396 ? '($uword = $w) =~ s/^(.)(.*)$/$1\L$2\E/; ##-- dyn:tolowerNI'
397 : ('$uword = $w; ##-- dyn:no-lower'))).' ##-- dyn:norm:lower
398  
399 '.($anl->{toupperI} ? '$uword = ucfirst($uword); ##-- dyn:toupperI' : '').' ##-- dyn:norm:upper
400 '.($anl->{bashWS} ? '$uword =~ s/\s+/$opts->{bashWS}/g; ##-- dyn:bashWS' : '').' ##-- dyn:norm:bashWS
401  
402 ##-- check for (normalized) word in dict
403 if ($dict && defined($ua=$dict->{$uword})) {
404 push(@$wa,
405 sort {($a->{w}||0) <=> ($b->{w}||0)}
406 map {DTA::CAB::Analyzer::Dict::parseFstString($_)}
407 grep {$_ ne ""}
408 split(/\t/,$ua)
409 );
410 } elsif ($fst_ok) {
411 ##-- not in dict: fst lookup (if fst is kosher)
412  
413 ##-- dyn:str2labels
414 '.($anl->{attInput}
415 ? ('##-- get labels: att-style (requires gfsm v0.0.10-pre11, gfsm-perl v0.0217)
416 $ulword = $uword;
417 utf8::downgrade($ulword);
418 @wlabs = (@{$lab->string_to_labels($ulword, $aut->{check_symbols}, 1)}, @eowlab);')
419 : ($anl->{check_symbols}
420 ? ('##-- get labels: by character: verbose
421 @wlabs = (@$labc[unpack("U0U*",$uword)],@eowlab);
422 foreach (grep { !defined($wlabs[$_]) } (0..$#wlabs)) {
423 $aut->warn("ignoring unknown character \`", substr($uword,$_,1), "\' in word \`$w\' (normalized to \`$uword\').\n");
424 }
425 @wlabs = grep {defined($_)} @wlabs;')
426 : ('##-- get labels: by character: quiet
427 @wlabs = grep {defined($_)} (@$labc[unpack("U0U*",$uword)],@eowlab);'))).' ##-- dyn:string_to_labels
428  
429 ##-- fst lookup
430 $aut->{fst}->lookup(\@wlabs, $result);
431 '.($anl->{auto_connect} ? '$result->_connect();' : '').' ##-- dyn:auto_connect
432 #$result->_rmepsilon() if ($opts->{auto_rmeps});
433  
434 ##-- parse analyses
435 push(@$wa,
436 map {
437 {(
438 '.($anl->{wantAnalysisLo} ? 'lo=>$uword,' : '').' ##-- dyn:wantAnalysisLo
439 "hi"=> (defined($labenc)
440 ? Encode::decode($labenc,$lab->labels_to_string($_->{hi},0,1))
441 : $lab->labels_to_string($_->{hi},0,1)),
442 "w" => $_->{w},
443 )}
444 } @{$result->paths($Gfsm::LSUpper)}
445 );
446 }
447 }
448  
449 ##-- parse lemmata?
450 '.($anl->{wantAnalysisLemma} ?
451 'foreach (@$wa) {
452 $lemma = $_->{hi};
453 if (defined($lemma) && $lemma ne "") {
454 $lemma =~ s/\[.*$//; ##-- trim everything after first non-character symbol
455 $lemma =~ s/(?:\/\w+)|(?:[\\\¬\~\|\=\+\#])//g;
456 substr($lemma,1) = lc(substr($lemma,1));
457 } else {
458 $lemma = $uword;
459 }
460 $lemma =~ s/^\s*//;
461 $lemma =~ s/\s*$//;
462 $lemma =~ s/\s+/_/g;
463 $_->{lemma} = $lemma;
464 }'
465 : '').' ##-- dyn: wantAnalysisLemma
466 ##-- END analyzeWord
467  
468 undef($wa) if (!@$wa);
469 '.($aset).'; ##-- dyn:analyzeSet
470 }
471 return $doc;
472 };';
473 }
474  
475  
476 1; ##-- be happy
477  
478 __END__