CAB/Analyzer/Moot/Boltzmann.pm untested

filter: Statementstmt 0.0% 0 / 103 Branchbran 0.0% 0 / 44 Conditioncond 0.0% 0 / 48 MC/DCmcdc 0.0% 0 / 29 Subroutinesub 0.0% 0 / 9 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 238 CC 23 SCAR 58.9
CC
23
Cov
0%
analyzeDebug
506.0
analysisCode
6.0
new
2.0
CRAP 359.9
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Moot::Boltzmann.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Moot analysis API for word n-gram disambiguation using dynamic lexicon
6  
7 package DTA::CAB::Analyzer::Moot::Boltzmann;
8 use DTA::CAB::Analyzer ':child';
9 use DTA::CAB::Analyzer::Moot;
10 use Moot;
11  
12 use strict;
13  
14 ##==============================================================================
15 ## Globals
16 ##==============================================================================
17  
18 our @ISA = qw(DTA::CAB::Analyzer::Moot);
19  
20 ##==============================================================================
21 ## Constructors etc.
22 ##==============================================================================
23  
24 ## $obj = CLASS_OR_OBJ->new(%args)
25 ## + object structure:
26 ## (
27 ## ##-- new in Analyzer::Moot::Boltzmann
28 ## #?
29 ##
30 ## ##==== Inherited from Analyzer::Moot
31 ## ##-- Filename Options
32 ## hmmFile => $filename, ##-- default: none (REQUIRED)
33 ##
34 ## ##-- Analysis Options
35 ## hmmArgs => \%args, ##-- clobber moot::HMM->new() defaults (default: verbose=>$moot::HMMvlWarnings)
36 ## hmmUtf8 => $bool, ##-- use hmm utf8 mode? (default=true)
37 ## ##
38 ## analyzeCostFuncs =>\%fnc, ##-- maps source 'analyses' key(s) to cost-munging functions
39 ## ## @fnc = ($akey=>$perlcode_str, ...)
40 ## ## + used by $dmoot->analysisCode() method
41 ## ## + evaluates $perlcode_str as subroutine body to derive analysis
42 ## ## 'weights' from source-key weights
43 ## ## + $perlcode_str may use variables:
44 ## ## #$moot ##-- current Analyzer::Moot object
45 ## ## #$tag ##-- source analysis tag
46 ## ## #$details ##-- source analysis 'details' "$hi <$w>"
47 ## ## $cost ##-- source analysis weight (munged to "$_->{w}")
48 ## ## $text ##-- source token text
49 ## ## $_ ##-- source analysis
50 ## ## + Default:
51 ## ## xlit => 2/length($text)
52 ## ## eqpho => 1/length($text)
53 ## ## eqphox => .5*$_->{w}/length($text)
54 ## ## rw => $_->{w}/length($text)
55 ##
56 ## #uniqueAnalyses => $bool, ##-- if true, only cost-minimal analyses for each tag will be added (default=1)
57 ## #requireAnalyses => $bool, ##-- if true all tokens MUST have non-empty analyses (useful for Boltzmann; default=0)
58 ## #prune => $bool, ##-- if true, prune analyses after tagging (default (override)=false)
59 ##
60 ## ##-- Analysis Objects
61 ## hmm => $hmm, ##-- a moot::HMM object
62 sub new {
63 my $that = shift;
64 my $dmoot = $that->SUPER::new(
65 ##-- options (override)
66 hmmArgs => {
67 newtag_str=>'@NEW',
68 newtag_f=>0.5,
69 Ftw_eps =>0.0, ##-- moot default=0.5
70 invert_lexp=>1,
71 hash_ngrams=>1,
72 relax=>0,
73 dynlex_base=>2,
74 dynlex_beta=>1,
75 },
76  
77 ##-- these flow into analysisCode()
78 #uniqueAnalyses =>1,
79 #requireAnalyses =>1,
80 #prune =>0,
81 analyzeCostFuncs => {
82 xlit=>'2.0/length($text)',
83 eqpho=>'1.0/length($text)',
84 eqphox=>'(1+0.1*$cost)/length($text)',
85 rw=>'$cost/length($text)',
86 },
87  
88 ##-- letter-case options
89 lctags => 0, ##-- if true, analysis tags are forced to lower-case
90  
91 ##-- analysis I/O
92 label => 'dmoot',
93 analyzeCode => undef, ##-- see analysisCode() method, below
94  
95 ##-- analysis objects
96 #hmm => undef,
97  
98 ##-- debugging
99 #analyzeDebug=>1,
100  
101 ##-- user args
102 @_
103 );
104 return $dmoot;
105 }
106  
107 ## $moot = $moot->clear()
108 ## + inherited from DTA::CAB::Analyzer::Moot
109  
110 ##==============================================================================
111 ## Methods: Generic
112 ##==============================================================================
113  
114 ## $bool = $moot->hmmOk()
115 ## + should return false iff HMM is undefined or "empty"
116 ## + default version checks for non-empty 'n_tags'
117 sub hmmOk {
118 return defined($_[0]{hmm}) && $_[0]{hmm}->n_tags()>1;
119 }
120  
121 ## $class = $moot->hmmClass()
122 ## + returns class for $moot->{hmm} object
123 ## + default just returns 'moot::HMM::Boltzmann'
124 sub hmmClass { return 'Moot::HMM::Boltzmann'; }
125  
126 ##==============================================================================
127 ## Methods: Analysis
128 ##==============================================================================
129  
130 ##------------------------------------------------------------------------
131 ## Methods: Analysis: Closure Utilities
132  
133 ## $asub_code = $dmoot->analysisCode()
134 ## + analysis closure for passing to Analyzer::accessClosure()
135 sub analysisCode {
136 my $dmoot = shift;
137 return $dmoot->analyzeDebug() if (0 && $dmoot->{analyzeDebug}); ##-- DEBUG
138  
139 my %acfunc = %{$dmoot->{analyzeCostFuncs}};
140 foreach (keys %acfunc) {
141 $acfunc{$_} =~ s/\$cost/\$_->{w}/g;
142 $acfunc{$_} =~ s/\$\{cost\}/\$_->{w}/g;
143 }
144  
145 return '
146 package '. __PACKAGE__ .';
147 my $dmoot=$anl;
148 my $lab =$dmoot->{label};
149 my $hmm =$dmoot->{hmm};
150 my $utf8 =$dmoot->{hmmUtf8};
151 my $lctags =$dmoot->{lctags};
152 my $dynbase =log($hmm->dynlex_base());
153 my $logbase =log($hmm->dynlex_base());
154 my $dynbeta =$hmm->dynlex_beta();
155 my ($msent,$w,$mw,$text,$tmp, $analysesOk);
156 sub {
157 $msent = [map {
158 $w = $_;
159 $analysesOk=1;
160 $mw = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{dmoot} if present
161 $text = $mw->{text} = (defined($mw->{word}) ? $mw->{word} : $w->{text}) if (!defined($text=$mw->{text}));
162 if (!$mw->{analyses}) {
163 if (($w->{xp}//"") =~ /\bpersName\b/i) {
164 ##-- special case for explicit TEI <persName> tag: use unicruft for latinExt=1 to scrub out long s
165 $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt});
166 $mw->{analyses} = [{tag=>$text, prob=>0}];
167 } elsif (($w->{xp}//"") =~ /\bforeign\b/i) {
168 ##-- special case for explicit TEI <foreign> tag: leave literal
169 $mw->{analyses} = [{tag=>$text, prob=>0}];
170 } elsif ($w->{exlex}) {
171 ##-- special case for exception lexicon: remaining alternatives
172 $mw->{analyses} = (ref($w->{exlex})
173 ? [map {{tag=>$_->{text}, prob=>-log($_->{freq}||$dynbase)/($logbase*$dynbeta)}} @{$w->{exlex}}] ##-- non-deterministic: ARRAY
174 : [{tag=>$w->{exlex}, prob=>0}]); ##-- deterministic: string
175 #} elsif (($w->{xr}//"") =~ /\baq\b/) {
176 # ##-- CAB <= v1.81 special case for antiqua-rendition in fraktur text via (($w->{xr}//"") =~ /\baq\b/) :: DISABLED for CAB v1.82
177 # $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt});
178 # $mw->{analyses} = [{tag=>$text, prob=>0}];
179 } elsif ($w->{msafe}) {
180 ##-- safe contemporary form: leave as-is
181 $mw->{analyses} = [{tag=>'._am_xlit('$w').', prob=>0}];
182 } elsif ($w->{xlit} && !$w->{xlit}{isLatinExt}) {
183 ##-- word with non-latin characters: leave as-is
184 $mw->{analyses} = [{tag=>$text, prob=>0}];
185 } elsif ($_->{lang} && (ref($_->{lang}) ? (!grep {$_ eq "de"} @{$_->{lang}}) : ($_->{lang} ne "de"))) {
186 ##-- foreign material: leave as-is
187 $mw->{analyses} = [{tag=>'._am_xlit('$w').', prob=>0}];
188 } else {
189 $tmp=undef;
190 $mw->{analyses} = [
191 '._am_dmoot_list2moota(_am_fst_sort(
192 _am_fst_uniq( ##-- only include cost-minimal unique analyses
193 '('.join(",\n\t",
194 _am_xlit_fst('$w',"($acfunc{xlit})"),
195 _am_fst_wcp_listref('$w->{eqphox}',"($acfunc{eqphox})"),
196 _am_fst_wcp_listref('$w->{rw}',"($acfunc{rw})"),
197 )
198 .')',
199 '$tmp'
200 )
201 )
202 ).'
203 ];
204 foreach (@{$mw->{analyses}}) {
205 $_->{tag} =~ s/\[(.[^\]]*)\]/$1/g; ##-- un-escape: brackets
206 $_->{tag} =~ s/\\\\(.)/$1/g; ##-- un-escape: backslashes
207 }
208 $analysesOk=0 if (!@{$mw->{analyses}});
209 }
210 $_->{details} = $_->{tag} foreach (@{$mw->{analyses}});
211 }
212 if ($lctags) { $_->{tag}=lc($_->{tag}) foreach (@{$mw->{analyses}}); }
213  
214 if (!$analysesOk) {
215 $dmoot->logwarn("no candidate analyses found for token text \\"$text\\": skipping sentence!");
216 return;
217 }
218  
219 $mw
220 } @{$_->{tokens}}];
221  
222 $hmm->tag_sentence($msent, $utf8);
223  
224 foreach (@$msent) {
225 delete($_->{text});
226 }
227 }
228 ';
229 }
230  
231 ##-- DEBUG
232 sub analyzeDebug {
233 my $anl = shift;
234  
235 my $dmoot=$anl;
236 my $lab =$dmoot->{label};
237 my $hmm =$dmoot->{hmm};
238 my $utf8 =$dmoot->{hmmUtf8};
239 my $lctags =$dmoot->{lctags};
240 my $dynbase =log($hmm->dynlex_base());
241 my $logbase =log($hmm->dynlex_base());
242 my $dynbeta =$hmm->dynlex_beta();
243 my ($msent,$w,$mw,$text,$tmp, $analysesOk);
244 return sub {
245 $msent = [map {
246 $w = $_;
247 $analysesOk=1;
248 $mw = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{dmoot} if present
249 $text = $mw->{text} = (defined($mw->{word}) ? $mw->{word} : $w->{text}) if (!defined($text=$mw->{text}));
250 if (!$mw->{analyses}) {
251 if ($w->{exlex}) {
252 ##-- special case for exception lexicon: clobber all other alternatives
253 $mw->{analyses} = (ref($w->{exlex})
254 ? [map {{tag=>$_->{text}, prob=>-log($_->{freq}||$dynbase)/($logbase*$dynbeta)}} @{$w->{exlex}}] ##-- non-deterministic: ARRAY
255 : [{tag=>$w->{exlex}, prob=>0}]); ##-- deterministic: string
256 } elsif ($w->{xr} && $w->{xr} =~ /\baq\b/) {
257 ##-- special case for antiqua typesetting in fraktur text: use unicruft for latinExt=1 to scrub out long s
258 $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt});
259 $mw->{analyses} = [{tag=>$text, prob=>0}];
260 } elsif ($w->{msafe}) {
261 ##-- safe contemporary form: leave as-is
262 $mw->{analyses} = [{tag=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit
263 , prob=>0}];
264 } elsif ($w->{xlit} && !$w->{xlit}{isLatinExt}) {
265 ##-- word with non-latin characters: leave as-is
266 $mw->{analyses} = [{tag=>$text, prob=>0}];
267 } elsif ($_->{lang} && (ref($_->{lang}) ? (!grep {$_ eq "de"} @{$_->{lang}}) : ($_->{lang} ne "de"))) {
268 ##-- foreign material: leave as-is
269 $mw->{analyses} = [{tag=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit
270 , prob=>0}];
271 } else {
272 $tmp=undef;
273 $mw->{analyses} = [
274 (map {{tag=>$_->{hi}, prob=>($_->{w}||0)} ##-- _am_dmoot_fst2moota
275 } (sort {($a->{w}||0) <=> ($b->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} (map {$tmp && $tmp->{hi} eq $_->{hi} ? qw() : ($tmp=$_)} sort {($a->{hi}||"") cmp ($b->{hi}||"") || ($a->{w}||0) <=> ($b->{w}||0)} ({hi=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit
276 , w=>(8.06*(length($text)**-1))} ##== _am_id_fst
277 ,
278 ($w->{eqphox} ? (map {{ %{$_}, w=>((3.42+0.05*$_->{w})*(length($text)**-1)) } ##-- _am_fst_wcp
279 } @{$w->{eqphox}}) ##-- _am_fst_wcp_list
280 : qw()) ##-- _am_fst_wcp_listref
281 ,
282 ($w->{rw} ? (map {{ %{$_}, w=>((1.07+0.04*$_->{w})*(length($text)**-1)) } ##-- _am_fst_wcp
283 } @{$w->{rw}}) ##-- _am_fst_wcp_list
284 : qw()) ##-- _am_fst_wcp_listref
285 )) ##== _am_fst_uniq
286 ) ##== _am_fst_sort
287 ) ##-- _am_dmoot_list2moota
288  
289 ];
290 foreach (@{$mw->{analyses}}) {
291 $_->{tag} =~ s/\[(.[^\]]*)\]/$1/g; ##-- un-escape: brackets
292 $_->{tag} =~ s/\\(.)/$1/g; ##-- un-escape: backslashes
293 }
294 $analysesOk=0 if (!@{$mw->{analyses}});
295 }
296 $_->{details} = $_->{tag} foreach (@{$mw->{analyses}});
297 }
298 if ($lctags) { $_->{tag}=lc($_->{tag}) foreach (@{$mw->{analyses}}); }
299  
300 if (!$analysesOk) {
301 $dmoot->logwarn("no candidate analyses found for token text \"$text\": skipping sentence!");
302 return;
303 }
304  
305 $mw
306 } @{$_->{tokens}}];
307  
308 $hmm->tag_sentence($msent, $utf8);
309  
310 foreach (@$msent) {
311 delete($_->{text});
312 }
313 };
314 }
315  
316 ##==============================================================================
317 ## Methods: I/O
318 ##==============================================================================
319  
320 ##--------------------------------------------------------------
321 ## Methods: I/O: Input: all
322  
323 ## $bool = $moot->ensureLoaded()
324 ## + ensures model data is loaded from default files
325 ## + inherited from DTA::CAB::Analyzer::Moot
326  
327 ##--------------------------------------------------------------
328 ## Methods: I/O: Input: HMM
329  
330 ## $moot = $moot->loadHMM($model_file)
331 ## + inherited from DTA::CAB::Analyzer::Moot
332  
333 ##==============================================================================
334 ## Methods: Persistence
335 ##==============================================================================
336  
337 ##======================================================================
338 ## Methods: Persistence: Perl
339  
340 ## @keys = $class_or_obj->noSaveKeys()
341 ## + returns list of keys not to be saved
342 ## + inherited from DTA::CAB::Analyzer::Moot
343  
344 ## $saveRef = $obj->savePerlRef()
345 ## + inherited from DTA::CAB::Persistent
346  
347 ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref)
348 ## + implicitly calls $obj->clear()
349 ## + inherited from DTA::CAB::Analyzer::Moot
350  
351 ##==============================================================================
352 ## Methods: Analysis
353 ## + inherited from DTA::CAB::Analyzer::Moot
354  
355  
356 1; ##-- be happy
357  
358 __END__