CAB/Analyzer/Moot.pm untested

filter: Statementstmt 0.0% 0 / 157 Branchbran 0.0% 0 / 74 Conditioncond 0.0% 0 / 96 MC/DCmcdc 0.0% 0 / 54 Subroutinesub 0.0% 0 / 24 Podpod 0.0% 0 / 17 totaltotal 0.0% 0 / 422 CC 38 SCAR 57.2
CC
38
Cov
0%
analysisCodeDEBUG
702.0
ensureDict
20.0
analyzeSentences
20.0
CRAP 304.0
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Moot.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: generic Moot analysis API
6  
7 package DTA::CAB::Analyzer::Moot;
8 use DTA::CAB::Analyzer ':child';
9 use DTA::CAB::Datum ':all';
10  
11 use Moot;
12 use Encode qw(encode decode);
13 use IO::File;
14 use Carp;
15  
16 #use Devel::Cycle; ##-- debug
17  
18 use strict;
19  
20 ##==============================================================================
21 ## Globals
22 ##==============================================================================
23  
24 our @ISA = qw(DTA::CAB::Analyzer);
25  
26 ##==============================================================================
27 ## Closure stuff
28  
29 ## $DEFAULT_ANALYZE_CODE
30 ## + default analysis code
31 ## + available variables:
32 ## $anl # Analyzer::Moot object
33 ## $_ # sentence to be analyzed
34 our $DEFAULT_ANALYZE_CODE = '
35 package '. __PACKAGE__ .';
36 my $moot=$anl;
37 my $lab =$moot->{label};
38 my $hmm =$moot->{hmm};
39 my $tagx=$moot->{tagx};
40 my $utf8=$moot->{hmmUtf8};
41 my $prune=$moot->{prune};
42 my $lctext=$moot->{lctext};
43 my $notag=$moot->{notag};
44 my $use_dmoot=$moot->{use_dmoot};
45 my $moot_lang=$moot->{lang};
46 my $xpne=$moot->{xpne};
47 my $xpfm=$moot->{xpfm};
48 my $fmtag=$moot->{fmtag};
49 my ($s,$msent,$w,$mw,$t,$at,$lang,$val);
50 sub {
51 $s = $_;
52 $msent = [map {
53 $w = $_;
54 $mw = $w->{$lab} = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{moot} if present
55 $mw->{text} = (defined($mw->{word}) ? $mw->{word} : '._am_tag('($use_dmoot ? $_->{dmoot} : undef)', _am_xlit).') if (!defined($mw->{text}));
56 $mw->{text} = lc($mw->{text}) if ($lctext);
57 $mw->{analyses} = [{tag=>"NE",details=>"NE.xp",prob=>0}] if ($xpne && ($w->{xp}//"") =~ /\b((?:pers)Name)\b/i); #place
58 $mw->{analyses} = [{tag=>$fmtag,details=>"$fmtag.xp",prob=>0}] if ($xpfm && ($w->{xp}//"") =~ /\bforeign\b/i);
59 $val = undef; ##-- temporary for _am_tagh_moota_uniq()
60 $mw->{analyses} = ['._am_tagh_list2moota_uniq('map {$_ ? @$_ : qw()}
61 @$w{qw(mlatin tokpp toka)},
62 ($use_dmoot && $w->{xlit} && !$w->{xlit}{isLatinExt} ? [$fmtag, "XY"] : qw()),
63 ($use_dmoot && $w->{dmoot} ? $w->{dmoot}{morph}
64 : ($w->{morph}, ($w->{rw} ? (map {$_->{morph}} @{$w->{rw}}) : qw())))'
65 ).'
66 ] if (!defined($mw->{analyses}));
67 foreach (@{$mw->{analyses}}) {
68 ##-- tag-translation hack: apply BEFORE sending to moot!
69 $_->{tag}=$t if (defined($t=$tagx->{$_->{tag}}));
70 }
71 $mw
72 } @{$s->{tokens}}];
73 return if (!@$msent); ##-- ignore empty sentences
74  
75 $hmm->tag_sentence($msent, $utf8) if (!$notag);
76  
77 ##-- language-guesser hack
78 if ($moot_lang && ($lang=$s->{lang}) && $lang ne $moot_lang) {
79 $_->{tag} = "$fmtag.$lang" foreach (grep {$_->{tag} !~ /^\$/} @$msent);
80 }
81  
82 foreach (@$msent) {
83 $_->{word}=$_->{text};
84 delete($_->{text});
85 #$_->{tag}=$t if (defined($t=$tagx->{$_->{tag}//""})); ##-- do NOT translate output tags (breaks en-wsj, 2016-06-09)
86 $_->{tag}="NE" if ($xpne && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "NE.xp");
87 $_->{tag}=$fmtag if ($xpfm && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "FM.xp");
88 if ($prune) {
89 $t = $_->{tag}//"";
90 @{$_->{analyses}} = grep {$_->{tag} eq $t} @{$_->{analyses}};
91 }
92 }
93 }
94 ';
95  
96  
97 ##==============================================================================
98 ## Constructors etc.
99 ##==============================================================================
100  
101 ## $obj = CLASS_OR_OBJ->new(%args)
102 ## + object structure:
103 ## (
104 ## ##-- Filename Options
105 ## hmmFile => $filename, ##-- default: none (REQUIRED)
106 ## tagxFile => $tagxFile, ##-- tag-translation file (hack)
107 ##
108 ## ##-- Analysis Options
109 ## hmmArgs => \%args, ##-- clobber Moot::HMM->new() defaults (default: none)
110 ## hmmUtf8 => $bool, ##-- use hmm utf8 mode? (default=true)
111 ##
112 ## analyzeCode => $code, ##-- pseudo-closure: analyze current sentence $_
113 ## label => $lab, ##-- destination key (default='moot')
114 ## prune => $bool, ##-- if true (default), prune analyses after tagging
115 ## lctext => $bool, ##-- if true, input text will be bashed to lower-case (default: false)
116 ## notag => $bool, ##-- if true, hmm tagger won't actually be called; read from global analyzer options as "${lab}.notag"
117 ## use_dmoot => $bool, ##-- if true, hmm tagger will try to get text & analyses from token {dmoot} key, otherwise it will be ignored
118 ## xpne => $bool, ##-- if true, force 'NE' tags whenever $w->{xp} =~ /\b(?:pers)Name\b/i (default=true) #NOT 'placeName', the tags are often appositions
119 ## xpfm => $bool, ##-- if true, force 'FM' tags whenever $w->{xp} =~ /\bforeign\b/i (default=true)
120 ## lang => $lang, ##-- only tag sentences marked as language "$lang"; read from global analyzer options as "${lab}.lang" (default='de')
121 ## fmtag => $fmtag, ##-- prefix for "foreign-material" tags; default='FM'
122 ##
123 ## ##-- Analysis Objects
124 ## hmm => $hmm, ##-- a moot::HMM object
125 ## tagx => \%tagx, ##-- tag-translation table (loaded via DTA::CAB::Analyzer::Dict from $tagxFile)
126 ## )
127 sub new {
128 my $that = shift;
129 my $moot = $that->SUPER::new(
130 ##-- filenames
131 hmmFile => undef,
132 tagxFile => undef,
133  
134 ##-- options
135 hmmArgs => {
136 #verbose=>Moot::vlWarnings,
137 #relax => 1,
138 },
139 hmmUtf8 => 1,
140  
141 #prune => 1,
142 #uniqueAnalyses=>0,
143  
144 ##-- analysis I/O
145 #analysisClass => 'DTA::CAB::Analyzer::Moot::Analysis',
146 label => 'moot',
147 analyzeCode => $DEFAULT_ANALYZE_CODE,
148 lctext => 0,
149 #notag => undef,
150 #use_dmoot => undef,
151 lang => 'de',
152 xpne => 1,
153 xpfm => 1,
154 fmtag => 'FM',
155  
156 #analyzeCostFuncs => {},
157 #requireAnalyses => 0,
158 #wantTaggedWord => 1,
159  
160 ##-- analysis objects
161 #hmm => undef,
162  
163 ##-- user args
164 @_
165 );
166 return $moot;
167 }
168  
169 ## $moot = $moot->clear()
170 sub clear {
171 my $moot = shift;
172  
173 ##-- analysis sub(s)
174 $moot->dropClosures();
175  
176 ##-- analysis objects
177 delete($moot->{hmm});
178 delete($moot->{tagx});
179  
180 return $moot;
181 }
182  
183 ## @keys = $anl->typeKeys(\%opts)
184 ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes()
185 ## + override returns empty list
186 sub typeKeys {
187 return qw();
188 }
189  
190 ##==============================================================================
191 ## Methods: Generic
192 ##==============================================================================
193  
194 ## $bool = $moot->hmmOk()
195 ## + should return false iff HMM is undefined or "empty"
196 ## + default version checks for non-empty 'lexprobs' and 'n_tags'
197 sub hmmOk {
198 return defined($_[0]{hmm}) && $_[0]{hmm}->n_tags()>1 && $_[0]{hmm}->n_toks()>1;
199 #&& $_[0]{hmm}{lexprobs}->size > 1;
200 }
201  
202 ## $class = $moot->hmmClass()
203 ## + returns class for $moot->{hmm} object
204 ## + default just returns 'Moot::HMM'
205 sub hmmClass { return 'Moot::HMM'; }
206  
207 ##==============================================================================
208 ## Methods: I/O
209 ##==============================================================================
210  
211 ##--------------------------------------------------------------
212 ## Methods: I/O: Input: all
213  
214 ## $bool = $moot->ensureLoaded()
215 ## + ensures model data is loaded from default files (if available)
216 sub ensureLoaded {
217 my $moot = shift;
218 my $rc = 1; ##-- allow empty models
219  
220 ##-- ensure: hmm
221 $rc &&= $moot->loadHMM($moot->{hmmFile}) if (defined($moot->{hmmFile}) && !$moot->hmmOk);
222  
223 ##-- ensure: dict: tagx
224 $rc &&= $moot->ensureDict('tagx',{}) if (!$moot->{tagx});
225  
226 return $rc;
227 }
228  
229 ##--------------------------------------------------------------
230 ## Methods: I/O: Input: HMM
231  
232 ## $moot = $moot->loadHMM($model_file)
233 BEGIN { *loadHMM = *loadHmm = \&loadHMM; }
234 sub loadHMM {
235 my ($moot,$model) = @_;
236 my $hmmClass = $moot->hmmClass;
237 $moot->info("loading HMM model file '$model' using HMM class '$hmmClass'");
238 if (!defined($moot->{hmm})) {
239 $moot->{hmm} = $hmmClass->new()
240 or $moot->logconfess("could not create HMM object of class '$hmmClass': $!");
241 $moot->{hmm}->config($moot->{hmmArgs}) if ($moot->{hmmArgs});
242 }
243 $moot->{hmm}->load($model)
244 or $moot->logconfess("loadHMM(): load failed for '$model': $!");
245 $moot->dropClosures();
246 return $moot;
247 }
248  
249 ##--------------------------------------------------------------
250 ## Methods: I/O: Input: Dictionaries: generic
251  
252 ## $bool = $a->ensureDict($dictName,\%dictDefault)
253 sub ensureDict {
254 my ($a,$name,$default) = @_;
255 return 1 if ($a->{$name}); ##-- already defined
256 return $a->loadDict($name,$a->{"${name}File"}) if ($a->{"${name}File"});
257 $a->{$name} = $default ? {%$default} : {};
258 return 1;
259 }
260  
261 ## \%dictHash_or_undef = $a->loadDict($dictName,$dictFile)
262 sub loadDict {
263 my ($a,$name,$dfile) = @_;
264 delete($a->{$name});
265 my $dclass = 'DTA::CAB::Analyzer::Dict';
266 $a->info("loading map from '$dfile' as $dclass");
267  
268 ##-- hack: generate a temporary dict object
269 my $dict = $dclass->new(label=>($a->{label}.".dict.$name"), dictFile=>$dfile);
270 $dict->ensureLoaded();
271 return undef if (!$dict->dictOk);
272  
273 ##-- clobber dict
274 $a->{$name} = $dict->dictHash;
275 }
276  
277  
278  
279 ##==============================================================================
280 ## Methods: Persistence
281 ##==============================================================================
282  
283 ##======================================================================
284 ## Methods: Persistence: Perl
285  
286 ## @keys = $class_or_obj->noSaveKeys()
287 ## + returns list of keys not to be saved
288 sub noSaveKeys {
289 my $that = shift;
290 return ($that->SUPER::noSaveKeys, qw(hmm));
291 }
292  
293 ## $saveRef = $obj->savePerlRef()
294 ## + inherited from DTA::CAB::Persistent
295  
296 ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref)
297 ## + implicitly calls $obj->clear()
298 sub loadPerlRef {
299 my ($that,$ref) = @_;
300 my $obj = $that->SUPER::loadPerlRef($ref);
301 $obj->clear();
302 return $obj;
303 }
304  
305 ##==============================================================================
306 ## Methods: Analysis
307 ##==============================================================================
308  
309 ##------------------------------------------------------------------------
310 ## Methods: Analysis: Generic
311  
312 ## $bool = $anl->canAnalyze()
313 ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty)
314 sub canAnalyze {
315 return $_[0]->hmmOk();
316 }
317  
318  
319 ##------------------------------------------------------------------------
320 ## Methods: Analysis: v1.x: API
321  
322 ## $bool = $anl->doAnalyze(\%opts, $name)
323 ## + override: only allow analyzeSentences()
324 sub doAnalyze {
325 my $anl = shift;
326 return 0 if (defined($_[1]) && $_[1] ne 'Sentences');
327 return $anl->SUPER::doAnalyze(@_);
328 }
329  
330 ## $doc = $anl->analyzeSentences($doc,\%opts)
331 ## + perform sentence-wise analysis of all sentences $doc->{body}[$si]
332 ## + no default implementation
333 sub analyzeSentences {
334 my ($moot,$doc,$opts) = @_;
335 return undef if (!$moot->ensureLoaded()); ##-- uh-oh...
336 return $doc if (!$moot->canAnalyze); ##-- ok...
337 $doc = toDocument($doc);
338  
339 ##-- inherit global options
340 my $notag = $moot->{notag};
341 my $use_dmoot = $moot->{use_dmoot};
342 $moot->{notag} //= $opts->{"$moot->{label}.notag"};
343 $moot->{use_dmoot} //= $opts->{"$moot->{label}.use_dmoot"};
344 $moot->{use_dmoot} //= 1;
345 $moot->{lang} = $opts->{"$moot->{label}.lang"} if ($opts->{"$moot->{label}.lang"});
346  
347 ##-- setup access closures
348 my $acode_str = $moot->analysisCode();
349 my $acode_sub = $moot->accessClosure($acode_str);
350  
351 ##-- ye olde loope
352 foreach (@{$doc->{body}}) {
353 $acode_sub->();
354 }
355  
356 ##-- restore local options
357 $moot->{notag} = $notag;
358 $moot->{use_dmoot} = $use_dmoot;
359  
360 ##-- DEBUG
361 #print STDERR "find_cycle()\n";
362 #find_cycle([$moot,$doc,$acode_sub]);
363 ##--/DEBUG
364  
365 return $doc;
366 }
367  
368 ##------------------------------------------------------------------------
369 ## Methods: Analysis: Closure Utilities
370  
371 ## $asub_code = $moot->analysisCode()
372 ## + analysis closure for passing to Analyzer::accessClosure()
373 ## + default just returns $moot->{analyzeCode} || $DEFAULT_ANALYZE_CODE
374 sub analysisCode {
375 my $moot = shift;
376 return
377 #analysisCodeDEBUG($moot) ||
378 $moot->{analyzeCode} || $DEFAULT_ANALYZE_CODE;
379 }
380  
381 ##------------------------------------------------------------------------
382 ## Methods: Analysis: DEBUG
383  
384 ## analysisCodeDEBUG() : from $DEFAULT_ANALYZE_CODE
385 sub analysisCodeDEBUG {
386 my $anl = shift;
387  
388 ## copy+paste from debugger "print $DEFAULT_ANALYZE_CODE" after breaking first e.g. in analysisCode() above
389 package DTA::CAB::Analyzer::Moot;
390 my $moot=$anl;
391 my $lab =$moot->{label};
392 my $hmm =$moot->{hmm};
393 my $tagx=$moot->{tagx};
394 my $utf8=$moot->{hmmUtf8};
395 my $prune=$moot->{prune};
396 my $lctext=$moot->{lctext};
397 my $notag=$moot->{notag};
398 my $use_dmoot=$moot->{use_dmoot};
399 my $moot_lang=$moot->{lang};
400 my $xpne=$moot->{xpne};
401 my $xpfm=$moot->{xpfm};
402 my $fmtag=$moot->{fmtag};
403 my ($s,$msent,$w,$mw,$t,$at,$lang,$val);
404 sub {
405 $s = $_;
406 $msent = [map {
407 $w = $_;
408 $mw = $w->{$lab} = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{moot} if present
409 $mw->{text} = (defined($mw->{word}) ? $mw->{word} : (($use_dmoot ? $_->{dmoot} : undef) ? ($use_dmoot ? $_->{dmoot} : undef)->{tag} : ($_->{xlit} ? $_->{xlit}{latin1Text} : $_->{text}) ##== _am_xlit
410 ) ##== _am_tag
411 ) if (!defined($mw->{text}));
412 $mw->{text} = lc($mw->{text}) if ($lctext);
413 $mw->{analyses} = [{tag=>"NE",details=>"NE.xp",prob=>0}] if ($xpne && ($w->{xp}//"") =~ /\b((?:pers)Name)\b/i); #place
414 $mw->{analyses} = [{tag=>$fmtag,details=>"$fmtag.xp",prob=>0}] if ($xpfm && ($w->{xp}//"") =~ /\bforeign\b/i);
415 $val = undef; ##-- temporary for _am_tagh_moota_uniq()
416 $mw->{analyses} = [(map {$val && $val->{details} eq $_->{details} ? qw() : ($val=$_)} sort {($a->{details}//"") cmp ($b->{details}//"") || ($a->{prob}//0) <=> ($b->{prob}//0)} (map {{details=>$_->{hi}, prob=>($_->{w}||0), tag=>($_->{hi} =~ /\[\_?((?:[A-Za-z0-9.]+|\$[^\]]+))\]/ ? $1 : $_->{hi})} ##-- _am_tagh_fst2moota
417 } map {ref($_) ? $_ : {hi=>$_}} map {$_ ? @$_ : qw()}
418 @$w{qw(mlatin tokpp toka)},
419 ($use_dmoot && $w->{xlit} && !$w->{xlit}{isLatinExt} ? [$fmtag, "XY"] : qw()),
420 ($use_dmoot && $w->{dmoot} ? $w->{dmoot}{morph}
421 : ($w->{morph}, ($w->{rw} ? (map {$_->{morph}} @{$w->{rw}}) : qw())))) ##-- _am_tagh_list2moota
422 ) ##== _am_tagh_moota_uniq
423  
424 ] if (!defined($mw->{analyses}));
425 foreach (@{$mw->{analyses}}) {
426 ##-- tag-translation hack: apply BEFORE sending to moot!
427 $_->{tag}=$t if (defined($t=$tagx->{$_->{tag}}));
428 }
429 $mw
430 } @{$s->{tokens}}];
431 return if (!@$msent); ##-- ignore empty sentences
432  
433 $hmm->tag_sentence($msent, $utf8) if (!$notag);
434  
435 ##-- language-guesser hack
436 if ($moot_lang && ($lang=$s->{lang}) && $lang ne $moot_lang) {
437 $_->{tag} = "$fmtag.$lang" foreach (grep {$_->{tag} !~ /^\$/} @$msent);
438 }
439  
440 foreach (@$msent) {
441 $_->{word}=$_->{text};
442 delete($_->{text});
443 #$_->{tag}=$t if (defined($t=$tagx->{$_->{tag}//""})); ##-- do NOT translate output tags (breaks en-wsj, 2016-06-09)
444 $_->{tag}="NE" if ($xpne && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "NE.xp");
445 $_->{tag}=$fmtag if ($xpfm && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "FM.xp");
446 if ($prune) {
447 $t = $_->{tag}//"";
448 @{$_->{analyses}} = grep {$_->{tag} eq $t} @{$_->{analyses}};
449 }
450 }
451 }
452 }
453  
454 1; ##-- be happy
455  
456 __END__