CAB/Analyzer/Automaton.pm untested

filter: Statementstmt 0.0% 0 / 199 Branchbran 0.0% 0 / 80 Conditioncond 0.0% 0 / 58 MC/DCmcdc 0.0% 0 / 38 Subroutinesub 0.0% 0 / 23 Podpod 0.0% 0 / 16 totaltotal 0.0% 0 / 414 CC 41 SCAR 60.1
CC
41
Cov
0%
analyzeTypes
870.0
loadLabels
30.0
load
20.0
CRAP 405.8
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Automaton.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: generic analysis automaton API
6  
7 package DTA::CAB::Analyzer::Automaton;
8 use DTA::CAB::Analyzer ':child';
9 use DTA::CAB::Unify ':all';
10 use Gfsm;
11 use Encode qw(encode decode);
12 use IO::File;
13 #use File::Basename qw();
14 use Carp;
15  
16 use strict;
17  
18 ##==============================================================================
19 ## Globals
20 ##==============================================================================
21  
22 our @ISA = qw(DTA::CAB::Analyzer);
23  
24 ## $DEFAULT_ANALYZE_GET
25 ## + default coderef or eval-able string for {analyzeGet}
26 our $DEFAULT_ANALYZE_GET = '$_->{$lab} ? qw() : '._am_xlit;
27  
28 ## $DEFAULT_ANALYZE_SET
29 ## + default coderef or eval-able string for {analyzeSet}
30 ## + vars:
31 ## $anl => analyzer (automaton)
32 ## $lab => analyzer label
33 ## $_ => token
34 ## $wa => analyses (array-ref)
35 our $DEFAULT_ANALYZE_SET = '$_->{$lab} = ($wa && @$wa ? [@$wa] : undef);';
36  
37 ##==============================================================================
38 ## Constructors etc.
39 ##==============================================================================
40  
41 ## $obj = CLASS_OR_OBJ->new(%args)
42 ## + object structure:
43 ## (
44 ## ##-- Filename Options
45 ## fstFile => $filename, ##-- source FST file (default: none)
46 ## labFile => $filename, ##-- source labels file (default: none)
47 ##
48 ## ##-- Exception lexicon options (OBSOLETE for CAB >=v1.15)
49 ## #dictFile => $filename, ##-- source dict file (default: none): clobbers $dict->{dictFile} if defined
50 ## #dict => $dict, ##-- exception lexicon as a DTA::CAB::Analyzer::Dict object or option hash
51 ## # ## + default=undef
52 ## #dictClass => $class, ##-- fallback class for new dict (default='DTA::CAB::Analyzer::Dict')
53 ##
54 ## ##-- Analysis Output
55 ## analyzeGet => $code, ##-- accessor: coderef or string: source text (default=$DEFAULT_ANALYZE_GET; return undef for no analysis)
56 ## analyzeSet => $code, ##-- accessor: coderef or string: set analyses (default=$DEFAULT_ANALYZE_SET)
57 ## analyzePre => $code, ##-- coderef or string: pre-analysis code to call (called as $code->($word))
58 ## analyzePost => $code, ##-- coderef or string: post-analysis code to call (called as $code->($word))
59 ## wantAnalysisLo => $bool, ##-- set to true to include 'lo' keys in analyses (default: true)
60 ## #wantAnalysisFst => $bool, ##-- set to true to include 'fst' key in analyses (default: false)
61 ## wantAnalysisLemma => $bool, ##-- set to true to include 'lemma' keys in analyses (default: false)
62 ##
63 ## ##-- Analysis Options
64 ## eow => $sym, ##-- EOW symbol for analysis FST
65 ## check_symbols => $bool, ##-- check for unknown symbols? (default=1)
66 ## labenc => $enc, ##-- encoding of labels file (default='auto' [utf8 if applicable, otherwise latin1])
67 ## #dictenc => $enc, ##-- dictionary encoding (default='UTF-8') (set $aut->{dict}{encoding} instead)
68 ## auto_connect => $bool, ##-- whether to call $result->_connect() after every lookup (default=0)
69 ## tolower => $bool, ##-- if true, all input words will be bashed to lower-case (default=0)
70 ## tolowerNI => $bool, ##-- if true, all non-initial characters of inputs will be lower-cased (default=0)
71 ## toupperI => $bool, ##-- if true, initial character will be upper-cased (default=0)
72 ## capsFallback => $bool, ##-- if true, all-caps words will also be analyzed as ucfirst(lc($w)) (default=0)
73 ## qsFallback => $bool, ##-- if true, "-'s" suffixes will also be analyzed as "-s" (default=0)
74 ## bashWS => $str, ##-- if defined, input whitespace will be bashed to '$str' (default='_')
75 ## attInput => $bool, ##-- if true, respect AT&T lextools-style escapes in input (default=0)
76 ## attOutput => $bool, ##-- if true, generate AT&T escapes in output (default=1)
77 ## allowTextRegex => $re, ##-- if defined, only tokens with matching 'text' will be analyzed (default: none)
78 ## ## : useful: /^(?:(?:[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+)|(?:[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$/
79 ## ## : == DTA::CAB::Analyzer::_am_wordlike_regex()
80 ##
81 ## ##-- Analysis objects
82 ## fst => $gfst, ##-- (child classes only) e.g. a Gfsm::Automaton object (default=new)
83 ## lab => $lab, ##-- (child classes only) e.g. a Gfsm::Alphabet object (default=new)
84 ## labh => \%sym2lab, ##-- (?) label hash: $sym2lab{$labSym} = $labId;
85 ## laba => \@lab2sym, ##-- (?) label array: $lab2sym[$labId] = $labSym;
86 ## labc => \@chr2lab, ##-- (?)chr-label array: $chr2lab[ord($chr)] = $labId;, by unicode char number (e.g. unpack('U0U*'))
87 ## warned_symbols => \%sym2undef, ##-- tracks unknown symbols we've already warned about (for check_symbols != 0)
88 ##
89 ## ##-- INHERITED from DTA::CAB::Analyzer
90 ## label => $label, ##-- analyzer label (default: from analyzer class name)
91 ## typeKeys => \@keys, ##-- type-wise keys to expand
92 ## )
93 sub new {
94 my $that = shift;
95 my $aut = $that->SUPER::new(
96 ##-- filenames
97 fstFile => undef,
98 labFile => undef,
99  
100 ##-- analysis objects
101 fst=>undef,
102 lab=>undef,
103 result=>undef,
104 labh=>{},
105 laba=>[],
106 labc=>[],
107 warned_symbols=>{},
108  
109 ##-- options
110 eow =>'',
111 check_symbols => 1,
112 labenc => 'auto',
113 auto_connect => 0,
114 tolower => 0,
115 tolowerNI => 0,
116 toupperI => 0,
117 capsFallback => 0,
118 qsFallback => 0,
119 bashWS => '_',
120 attInput => 0,
121 attOutput => 1,
122 allowTextRegex => undef, #'(?:^[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+$)|(?:^[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+$)',
123 #'^(?:(?:[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+)|(?:[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$'
124 # == DTA::CAB::Analyzer::_am_wordlike_regex()
125 ##-- analysis I/O
126 analyzeSrc => 'text',
127 wantAnalysisLo => 1,
128 wantAnalysisLemma => 0,
129  
130 ##-- user args
131 @_
132 );
133 return $aut;
134 }
135  
136 ## $aut = $aut->clear()
137 sub clear {
138 my $aut = shift;
139  
140 ##-- analysis sub(s)
141 $aut->dropClosures();
142  
143 ##-- analysis objects
144 delete($aut->{fst});
145 delete($aut->{lab});
146 delete($aut->{result}); ##-- should be undef anyways
147 %{$aut->{labh}} = qw();
148 @{$aut->{laba}} = qw();
149 @{$aut->{labc}} = qw();
150  
151 return $aut;
152 }
153  
154 ##==============================================================================
155 ## Methods: Generic
156 ##==============================================================================
157  
158 ## $class = $aut->fstClass()
159 ## + default FST class for loadFst() method
160 sub fstClass { return 'Gfsm::Automaton'; }
161  
162 ## $class = $aut->labClass()
163 ## + default labels class for loadLabels() method
164 sub labClass { return 'Gfsm::Alphabet'; }
165  
166 ## $bool = $aut->fstOk()
167 ## + should return false iff fst is undefined or "empty"
168 sub fstOk { return defined($_[0]{fst}) && $_[0]{fst}->n_states>0; }
169  
170 ## $bool = $aut->labOk()
171 ## + should return false iff label-set is undefined or "empty"
172 sub labOk { return defined($_[0]{lab}) && $_[0]{lab}->size>0; }
173  
174 ##==============================================================================
175 ## Methods: I/O
176 ##==============================================================================
177  
178 ##--------------------------------------------------------------
179 ## Methods: I/O: Input: all
180  
181 ## $bool = $aut->ensureLoaded()
182 ## + ensures automaton data is loaded from default files
183 sub ensureLoaded {
184 my $aut = shift;
185 my $rc = 1;
186 ##-- ensure: fst
187 if ( defined($aut->{fstFile}) && !$aut->fstOk ) {
188 $rc &&= $aut->loadFst($aut->{fstFile});
189 }
190 ##-- ensure: lab
191 if ( defined($aut->{labFile}) && !$aut->labOk ) {
192 $rc &&= $aut->loadLabels($aut->{labFile});
193 }
194 return $rc;
195 }
196  
197 ## $aut = $aut->load(fst=>$fstFile, lab=>$labFile)
198 sub load {
199 my ($aut,%args) = @_;
200 return 0 if (!grep {defined($_)} @args{qw(fst lab dict)});
201 my $rc = $aut;
202 $rc &&= $aut->loadFst($args{fst}) if (defined($args{fst}));
203 $rc &&= $aut->loadLabels($args{lab}) if (defined($args{lab}));
204 return $rc;
205 }
206  
207 ##--------------------------------------------------------------
208 ## Methods: I/O: Input: FST
209  
210 ## $aut = $aut->loadFst($fstfile)
211 sub loadFst {
212 my ($aut,$fstfile) = @_;
213 $aut->info("loading FST file '$fstfile'");
214 $aut->{fst} = $aut->fstClass->new() if (!defined($aut->{fst}));
215 $aut->{fst}->load($fstfile)
216 or $aut->logconfess("loadFst(): load failed for '$fstfile': $!");
217 delete($aut->{result});
218 #$aut->{result} = $aut->{fst}->shadow; #if (defined($aut->{result}) && $aut->{fst}->can('shadow'));
219 delete($aut->{_analyze});
220 return $aut;
221 }
222  
223 ## $result = $aut->resultFst()
224 ## + returns empty result FST
225 sub resultFst {
226 return $_[0]{fst} ? $_[0]{fst}->shadow : undef;
227 }
228  
229 ##--------------------------------------------------------------
230 ## Methods: I/O: Input: Labels
231  
232 ## $aut = $aut->loadLabels($labfile)
233 sub loadLabels {
234 my ($aut,$labfile) = @_;
235 $aut->info("loading labels file '$labfile'");
236 $aut->{lab} = $aut->labClass->new() if (!defined($aut->{lab}));
237 $aut->{lab}->load($labfile)
238 or $aut->logconfess("loadLabels(): load failed for '$labfile': $!");
239 if (!$aut->{labenc} || $aut->{labenc} eq 'auto') {
240 ##-- guess label encoding
241 my $buf = join('',@{$aut->{lab}->toArray});
242 $aut->{labenc} = utf8::decode($buf) ? 'utf8' : 'latin1';
243 $aut->debug("loadLabels(): guessed label encoding '$aut->{labenc}'");
244 }
245 $aut->{lab}->utf8(1)
246 if ($aut->{lab}->can('utf8') && (($aut->{labenc}||'') =~ /^utf\-?8$/i));
247 $aut->parseLabels();
248 delete($aut->{_analyze});
249 return $aut;
250 }
251  
252 ## $aut = $aut->parseLabels()
253 ## + sets up $aut->{labh}, $aut->{laba}, $aut->{labc}
254 ## + fixes encoding difficulties in $aut->{labh}, $aut->{laba}
255 sub parseLabels {
256 my $aut = shift;
257 my $laba = $aut->{laba};
258 @$laba = @{$aut->{lab}->asArray};
259 my ($i);
260 foreach $i (grep { defined($laba->[$_]) } 0..$#$laba) {
261 $laba->[$i] = decode($aut->{labenc}, $laba->[$i]) if ($aut->{labenc});
262 $aut->{labh}{$laba->[$i]} = $i;
263 }
264 ##-- setup labc: $labId = $labc->[ord($c)]; ##-- single unicode characater
265 ## : @labIds = @$labc[unpack('U0U*',$s)]; ##-- batch lookup for strings (fast)
266 my @csyms = grep {defined($_) && length($_)==1} @$laba; ##-- @csyms = ($sym1, ...) s.t. each sym has len==1
267 @{$aut->{labc}}[map {ord($_)} @csyms] = @{$aut->{labh}}{@csyms};
268 ##
269 return $aut;
270 }
271  
272 ##==============================================================================
273 ## Methods: Persistence
274 ##==============================================================================
275  
276 ##======================================================================
277 ## Methods: Persistence: Perl
278  
279 ## @keys = $class_or_obj->noSaveKeys()
280 ## + returns list of keys not to be saved
281 sub noSaveKeys {
282 my $that = shift;
283 return ($that->SUPER::noSaveKeys, qw(dict fst lab laba labc labh result));
284 }
285  
286 ## $saveRef = $obj->savePerlRef()
287 ## + inherited from DTA::CAB::Persistent
288  
289 ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref)
290 ## + implicitly calls $obj->clear()
291 sub loadPerlRef {
292 my ($that,$ref) = @_;
293 my $obj = $that->SUPER::loadPerlRef($ref);
294 $obj->clear();
295 return $obj;
296 }
297  
298 ##==============================================================================
299 ## Methods: Analysis
300 ##==============================================================================
301  
302 ##------------------------------------------------------------------------
303 ## Methods: Analysis: Generic
304  
305 ## $bool = $anl->canAnalyze()
306 ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty)
307 sub canAnalyze {
308 return ($_[0]->labOk && $_[0]->fstOk);
309 }
310  
311  
312 ##==============================================================================
313 ## Methods: Analysis: v1.x
314 ##==============================================================================
315  
316  
317 ## $doc = $anl->analyzeTypes($doc,\%types,\%opts)
318 ## + perform type-wise analysis of all (text) types in %types (= %{$doc->{types}})
319 sub analyzeTypes {
320 my ($aut,$doc,$types,$opts) = @_;
321 return if (!$aut->canAnalyze);
322 $types = $doc->types if (!$types);
323  
324 ##-- setup common variables
325 my $fst = $aut->{fst};
326 my $fst_ok = $aut->fstOk();
327 my $result = $aut->resultFst();
328 my $lab = $aut->{lab};
329 my $labc = $aut->{labc};
330 my $laba = $aut->{laba};
331 my $labenc = $aut->{labenc};
332 my @eowlab = (defined($aut->{eow}) && $aut->{eow} ne '' ? ($aut->{labh}{$aut->{eow}}) : qw());
333 my $allowTextRegex = defined($aut->{allowTextRegex}) ? qr($aut->{allowTextRegex}) : undef;
334  
335 ##-- object analysis options
336 my $check_symbols = $aut->{check_symbols};
337 my $auto_connect = $aut->{auto_connect};
338 my $tolower = $aut->{tolower};
339 my $tolowerNI = $aut->{tolowerNI};
340 my $toupperI = $aut->{toupperI};
341 my $capsFallback = $aut->{capsFallback};
342 my $qsFallback = $aut->{qsFallback};
343 my $bashWS = $aut->{bashWS};
344 my $attInput = $aut->{attInput};
345 my $attOutput = $aut->{attOutput};
346 my $wantAnalysisLo = $aut->{wantAnalysisLo};
347 #my $wantAnalysisFst = $aut->{wantAnalysisFst};
348 my $wantAnalysisLemma = $aut->{wantAnalysisLemma};
349  
350 ##-- closures
351 my (@wa);
352 my $aget_code = defined($aut->{analyzeGet}) ? $aut->{analyzeGet} : $DEFAULT_ANALYZE_GET;
353 my $aset_code = defined($aut->{analyzeSet}) ? $aut->{analyzeSet} : $DEFAULT_ANALYZE_SET;
354 my $ax_pre = join('',
355 map {"my $_;\n"}
356 '$lab=$anl->{label}',
357 '$wa=$opts{wa}',
358 );
359 my $aget = $aut->accessClosure($aget_code, pre=>$ax_pre, wa=>\@wa, cache=>0);
360 my $aset = $aut->accessClosure($aset_code, pre=>$ax_pre, wa=>\@wa, cache=>0);
361 ##
362 my $apre = $aut->{analyzePre} ? $aut->accessClosure($aut->{analyzePre}) : undef;
363 my $apost = $aut->{analyzePost} ? $aut->accessClosure($aut->{analyzePost}) : undef;
364  
365 ##-- ye olde loope
366 my (@w,$w,$uword,$ulword,@wlabs,$ua,$lemma);
367 foreach (values(%$types)) {
368 next if (defined($allowTextRegex) && $_->{text} !~ $allowTextRegex); ##-- text-sensitive regex
369 @w=$aget->();
370 push(@w, map {ucfirst(lc($_))} grep {/^[[:upper:]]{2,}$/} @w) if ($capsFallback); ##-- handle "FRANKFURT" as "Frankfurt"
371 push(@w, map {m/^(.+)[\'\x{2018}\x{2019}]s$/ ? "${1}s" : qw()} @w) if ($qsFallback); ##-- handle "Goethe's" as "Goethes"
372 #$aut->trace("analyzeTypes(): w=(", join(' ', @w), ")") if ($aut->{label} eq 'eqphox'); ##-- DEBUG
373  
374 next if (!@w); ##-- no lookup-inputs for token (e.g. if $_->{$label} was already populated, e.g. from exlex)
375 @wa = qw();
376 foreach $w (@w) {
377 ##-------- BEGIN analyzeWord
378 #$aut->setLookupOptions($wopts) if ($aut->can('setLookupOptions')); ##-- OBSOLETE: use code-string in 'analyzePre' key instead!
379 $apre->($w) if ($apre);
380  
381 ##---- analyze
382 ##-- normalize
383 $uword = $w // '';
384 if ($tolower) { $uword = lc($uword); }
385 elsif ($tolowerNI) { $uword =~ s/^(.)(.*)$/$1\L$2\E/; }
386 if ($toupperI) { $uword = ucfirst($uword); }
387 if (defined($bashWS)) { $uword =~ s/\s+/$bashWS/g; }
388  
389 if ($fst_ok) {
390 ##-- fst lookup (if fst available and no previous analysis was found)
391  
392 ##-- fst: get labels
393 if ($attInput) {
394 ##-- fst: labels: att-style (requires gfsm v0.0.10-pre11, gfsm-perl v0.0217)
395 $ulword = $uword;
396 utf8::downgrade($ulword);
397 @wlabs = (@{$lab->string_to_labels($ulword, $check_symbols, 1)}, @eowlab);
398 }
399 elsif ($check_symbols) {
400 ##-- fst: labels: by character: verbose
401 @wlabs = (@$labc[unpack('U0U*',$uword)],@eowlab);
402 foreach (grep { !defined($wlabs[$_]) && !exists($aut->{warned_symbols}{substr($uword,$_,1)}) } (0..$#wlabs)) {
403 $aut->warn("ignoring unknown character '", substr($uword,$_,1), "' in word '$w' (normalized to '$uword').\n");
404 $aut->{warned_symbols}{substr($uword,$_,1)} = undef;
405 }
406 @wlabs = grep {defined($_)} @wlabs;
407 }
408 else {
409 ##-- fst: labels: by character: quiet
410 @wlabs = grep {defined($_)} (@$labc[unpack('U0U*',$uword)],@eowlab);
411 }
412  
413 ##-- fst: lookup
414 $aut->{fst}->lookup(\@wlabs, $result);
415 $result->_connect() if ($auto_connect);
416 #$result->_rmepsilon() if ($wopts->{auto_rmeps});
417  
418 ##-- parse analyses
419 push(@wa,
420 map {
421 {(
422 #($wantAnalysisFst ? (fst=>$result->clone) : qw()),
423 ($wantAnalysisLo ? (lo=>$uword) : qw()),
424 'hi'=> (defined($labenc)
425 ? decode($labenc,$lab->labels_to_string($_->{hi},0,1))
426 : $lab->labels_to_string($_->{hi},0,1)),
427 'w' => $_->{w},
428 )}
429 } @{$result->paths($Gfsm::LSUpper)}
430 );
431  
432 }
433 }##-- /foreach $w (@w)
434  
435 ##-- un-escape output
436 if (!$attOutput) {
437 foreach (@wa) {
438 $_->{hi} =~ s/\\(.)/$1/g;
439 $_->{hi} =~ s/\[([^\]]+)\]/$1/g;
440 }
441 }
442  
443 ##-- parse lemmata
444 if ($wantAnalysisLemma) {
445 foreach (@wa) {
446 $lemma = $_->{hi};
447 if (defined($lemma) && $lemma ne '') {
448 $lemma =~ s/\[.*$//; ##-- trim everything after first non-character symbol
449 $lemma =~ s/(?:\/\w+)|(?:[\\\¬\~\|\=\+\#])//g;
450 substr($lemma,1) = lc(substr($lemma,1));
451 } else {
452 $lemma = $uword;
453 }
454 $lemma =~ s/^\s*//;
455 $lemma =~ s/\s*$//;
456 $lemma =~ s/\s+/_/g;
457 $_->{lemma} = $lemma;
458 }
459 }
460 ##-------- END analyzeWord
461  
462 ##-- set analyses
463 $aset->();
464 $apost->() if ($apost);
465 }
466  
467 return $doc;
468 }
469  
470  
471 1; ##-- be happy
472  
473 __END__