| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Moot::Boltzmann.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Moot analysis API for word n-gram disambiguation using dynamic lexicon | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Moot::Boltzmann; | ||
| 8 | use DTA::CAB::Analyzer ':child'; | ||
| 9 | use DTA::CAB::Analyzer::Moot; | ||
| 10 | use Moot; | ||
| 11 | |||
| 12 | use strict; | ||
| 13 | |||
| 14 | ##============================================================================== | ||
| 15 | ## Globals | ||
| 16 | ##============================================================================== | ||
| 17 | |||
| 18 | our @ISA = qw(DTA::CAB::Analyzer::Moot); | ||
| 19 | |||
| 20 | ##============================================================================== | ||
| 21 | ## Constructors etc. | ||
| 22 | ##============================================================================== | ||
| 23 | |||
| 24 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 25 | ## + object structure: | ||
| 26 | ## ( | ||
| 27 | ## ##-- new in Analyzer::Moot::Boltzmann | ||
| 28 | ## #? | ||
| 29 | ## | ||
| 30 | ## ##==== Inherited from Analyzer::Moot | ||
| 31 | ## ##-- Filename Options | ||
| 32 | ## hmmFile => $filename, ##-- default: none (REQUIRED) | ||
| 33 | ## | ||
| 34 | ## ##-- Analysis Options | ||
| 35 | ## hmmArgs => \%args, ##-- clobber moot::HMM->new() defaults (default: verbose=>$moot::HMMvlWarnings) | ||
| 36 | ## hmmUtf8 => $bool, ##-- use hmm utf8 mode? (default=true) | ||
| 37 | ## ## | ||
| 38 | ## analyzeCostFuncs =>\%fnc, ##-- maps source 'analyses' key(s) to cost-munging functions | ||
| 39 | ## ## @fnc = ($akey=>$perlcode_str, ...) | ||
| 40 | ## ## + used by $dmoot->analysisCode() method | ||
| 41 | ## ## + evaluates $perlcode_str as subroutine body to derive analysis | ||
| 42 | ## ## 'weights' from source-key weights | ||
| 43 | ## ## + $perlcode_str may use variables: | ||
| 44 | ## ## #$moot ##-- current Analyzer::Moot object | ||
| 45 | ## ## #$tag ##-- source analysis tag | ||
| 46 | ## ## #$details ##-- source analysis 'details' "$hi <$w>" | ||
| 47 | ## ## $cost ##-- source analysis weight (munged to "$_->{w}") | ||
| 48 | ## ## $text ##-- source token text | ||
| 49 | ## ## $_ ##-- source analysis | ||
| 50 | ## ## + Default: | ||
| 51 | ## ## xlit => 2/length($text) | ||
| 52 | ## ## eqpho => 1/length($text) | ||
| 53 | ## ## eqphox => .5*$_->{w}/length($text) | ||
| 54 | ## ## rw => $_->{w}/length($text) | ||
| 55 | ## | ||
| 56 | ## #uniqueAnalyses => $bool, ##-- if true, only cost-minimal analyses for each tag will be added (default=1) | ||
| 57 | ## #requireAnalyses => $bool, ##-- if true all tokens MUST have non-empty analyses (useful for Boltzmann; default=0) | ||
| 58 | ## #prune => $bool, ##-- if true, prune analyses after tagging (default (override)=false) | ||
| 59 | ## | ||
| 60 | ## ##-- Analysis Objects | ||
| 61 | ## hmm => $hmm, ##-- a moot::HMM object | ||
| 62 | sub new { | ||
| 63 | my $that = shift; | ||
| 64 | my $dmoot = $that->SUPER::new( | ||
| 65 | ##-- options (override) | ||
| 66 | hmmArgs => { | ||
| 67 | newtag_str=>'@NEW', | ||
| 68 | newtag_f=>0.5, | ||
| 69 | Ftw_eps =>0.0, ##-- moot default=0.5 | ||
| 70 | invert_lexp=>1, | ||
| 71 | hash_ngrams=>1, | ||
| 72 | relax=>0, | ||
| 73 | dynlex_base=>2, | ||
| 74 | dynlex_beta=>1, | ||
| 75 | }, | ||
| 76 | |||
| 77 | ##-- these flow into analysisCode() | ||
| 78 | #uniqueAnalyses =>1, | ||
| 79 | #requireAnalyses =>1, | ||
| 80 | #prune =>0, | ||
| 81 | analyzeCostFuncs => { | ||
| 82 | xlit=>'2.0/length($text)', | ||
| 83 | eqpho=>'1.0/length($text)', | ||
| 84 | eqphox=>'(1+0.1*$cost)/length($text)', | ||
| 85 | rw=>'$cost/length($text)', | ||
| 86 | }, | ||
| 87 | |||
| 88 | ##-- letter-case options | ||
| 89 | lctags => 0, ##-- if true, analysis tags are forced to lower-case | ||
| 90 | |||
| 91 | ##-- analysis I/O | ||
| 92 | label => 'dmoot', | ||
| 93 | analyzeCode => undef, ##-- see analysisCode() method, below | ||
| 94 | |||
| 95 | ##-- analysis objects | ||
| 96 | #hmm => undef, | ||
| 97 | |||
| 98 | ##-- debugging | ||
| 99 | #analyzeDebug=>1, | ||
| 100 | |||
| 101 | ##-- user args | ||
| 102 | @_ | ||
| 103 | ); | ||
| 104 | return $dmoot; | ||
| 105 | } | ||
| 106 | |||
| 107 | ## $moot = $moot->clear() | ||
| 108 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 109 | |||
| 110 | ##============================================================================== | ||
| 111 | ## Methods: Generic | ||
| 112 | ##============================================================================== | ||
| 113 | |||
| 114 | ## $bool = $moot->hmmOk() | ||
| 115 | ## + should return false iff HMM is undefined or "empty" | ||
| 116 | ## + default version checks for non-empty 'n_tags' | ||
| 117 | sub hmmOk { | ||
| 118 | return defined($_[0]{hmm}) && $_[0]{hmm}->n_tags()>1; | ||
| 119 | } | ||
| 120 | |||
| 121 | ## $class = $moot->hmmClass() | ||
| 122 | ## + returns class for $moot->{hmm} object | ||
| 123 | ## + default just returns 'moot::HMM::Boltzmann' | ||
| 124 | sub hmmClass { return 'Moot::HMM::Boltzmann'; } | ||
| 125 | |||
| 126 | ##============================================================================== | ||
| 127 | ## Methods: Analysis | ||
| 128 | ##============================================================================== | ||
| 129 | |||
| 130 | ##------------------------------------------------------------------------ | ||
| 131 | ## Methods: Analysis: Closure Utilities | ||
| 132 | |||
| 133 | ## $asub_code = $dmoot->analysisCode() | ||
| 134 | ## + analysis closure for passing to Analyzer::accessClosure() | ||
| 135 | sub analysisCode { | ||
| 136 | my $dmoot = shift; | ||
| 137 | return $dmoot->analyzeDebug() if (0 && $dmoot->{analyzeDebug}); ##-- DEBUG | ||
| 138 | |||
| 139 | my %acfunc = %{$dmoot->{analyzeCostFuncs}}; | ||
| 140 | foreach (keys %acfunc) { | ||
| 141 | $acfunc{$_} =~ s/\$cost/\$_->{w}/g; | ||
| 142 | $acfunc{$_} =~ s/\$\{cost\}/\$_->{w}/g; | ||
| 143 | } | ||
| 144 | |||
| 145 | return ' | ||
| 146 | package '. __PACKAGE__ .'; | ||
| 147 | my $dmoot=$anl; | ||
| 148 | my $lab =$dmoot->{label}; | ||
| 149 | my $hmm =$dmoot->{hmm}; | ||
| 150 | my $utf8 =$dmoot->{hmmUtf8}; | ||
| 151 | my $lctags =$dmoot->{lctags}; | ||
| 152 | my $dynbase =log($hmm->dynlex_base()); | ||
| 153 | my $logbase =log($hmm->dynlex_base()); | ||
| 154 | my $dynbeta =$hmm->dynlex_beta(); | ||
| 155 | my ($msent,$w,$mw,$text,$tmp, $analysesOk); | ||
| 156 | sub { | ||
| 157 | $msent = [map { | ||
| 158 | $w = $_; | ||
| 159 | $analysesOk=1; | ||
| 160 | $mw = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{dmoot} if present | ||
| 161 | $text = $mw->{text} = (defined($mw->{word}) ? $mw->{word} : $w->{text}) if (!defined($text=$mw->{text})); | ||
| 162 | if (!$mw->{analyses}) { | ||
| 163 | if (($w->{xp}//"") =~ /\bpersName\b/i) { | ||
| 164 | ##-- special case for explicit TEI <persName> tag: use unicruft for latinExt=1 to scrub out long s | ||
| 165 | $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt}); | ||
| 166 | $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 167 | } elsif (($w->{xp}//"") =~ /\bforeign\b/i) { | ||
| 168 | ##-- special case for explicit TEI <foreign> tag: leave literal | ||
| 169 | $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 170 | } elsif ($w->{exlex}) { | ||
| 171 | ##-- special case for exception lexicon: remaining alternatives | ||
| 172 | $mw->{analyses} = (ref($w->{exlex}) | ||
| 173 | ? [map {{tag=>$_->{text}, prob=>-log($_->{freq}||$dynbase)/($logbase*$dynbeta)}} @{$w->{exlex}}] ##-- non-deterministic: ARRAY | ||
| 174 | : [{tag=>$w->{exlex}, prob=>0}]); ##-- deterministic: string | ||
| 175 | #} elsif (($w->{xr}//"") =~ /\baq\b/) { | ||
| 176 | # ##-- CAB <= v1.81 special case for antiqua-rendition in fraktur text via (($w->{xr}//"") =~ /\baq\b/) :: DISABLED for CAB v1.82 | ||
| 177 | # $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt}); | ||
| 178 | # $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 179 | } elsif ($w->{msafe}) { | ||
| 180 | ##-- safe contemporary form: leave as-is | ||
| 181 | $mw->{analyses} = [{tag=>'._am_xlit('$w').', prob=>0}]; | ||
| 182 | } elsif ($w->{xlit} && !$w->{xlit}{isLatinExt}) { | ||
| 183 | ##-- word with non-latin characters: leave as-is | ||
| 184 | $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 185 | } elsif ($_->{lang} && (ref($_->{lang}) ? (!grep {$_ eq "de"} @{$_->{lang}}) : ($_->{lang} ne "de"))) { | ||
| 186 | ##-- foreign material: leave as-is | ||
| 187 | $mw->{analyses} = [{tag=>'._am_xlit('$w').', prob=>0}]; | ||
| 188 | } else { | ||
| 189 | $tmp=undef; | ||
| 190 | $mw->{analyses} = [ | ||
| 191 | '._am_dmoot_list2moota(_am_fst_sort( | ||
| 192 | _am_fst_uniq( ##-- only include cost-minimal unique analyses | ||
| 193 | '('.join(",\n\t", | ||
| 194 | _am_xlit_fst('$w',"($acfunc{xlit})"), | ||
| 195 | _am_fst_wcp_listref('$w->{eqphox}',"($acfunc{eqphox})"), | ||
| 196 | _am_fst_wcp_listref('$w->{rw}',"($acfunc{rw})"), | ||
| 197 | ) | ||
| 198 | .')', | ||
| 199 | '$tmp' | ||
| 200 | ) | ||
| 201 | ) | ||
| 202 | ).' | ||
| 203 | ]; | ||
| 204 | foreach (@{$mw->{analyses}}) { | ||
| 205 | $_->{tag} =~ s/\[(.[^\]]*)\]/$1/g; ##-- un-escape: brackets | ||
| 206 | $_->{tag} =~ s/\\\\(.)/$1/g; ##-- un-escape: backslashes | ||
| 207 | } | ||
| 208 | $analysesOk=0 if (!@{$mw->{analyses}}); | ||
| 209 | } | ||
| 210 | $_->{details} = $_->{tag} foreach (@{$mw->{analyses}}); | ||
| 211 | } | ||
| 212 | if ($lctags) { $_->{tag}=lc($_->{tag}) foreach (@{$mw->{analyses}}); } | ||
| 213 | |||
| 214 | if (!$analysesOk) { | ||
| 215 | $dmoot->logwarn("no candidate analyses found for token text \\"$text\\": skipping sentence!"); | ||
| 216 | return; | ||
| 217 | } | ||
| 218 | |||
| 219 | $mw | ||
| 220 | } @{$_->{tokens}}]; | ||
| 221 | |||
| 222 | $hmm->tag_sentence($msent, $utf8); | ||
| 223 | |||
| 224 | foreach (@$msent) { | ||
| 225 | delete($_->{text}); | ||
| 226 | } | ||
| 227 | } | ||
| 228 | '; | ||
| 229 | } | ||
| 230 | |||
| 231 | ##-- DEBUG | ||
| 232 | sub analyzeDebug { | ||
| 233 | my $anl = shift; | ||
| 234 | |||
| 235 | my $dmoot=$anl; | ||
| 236 | my $lab =$dmoot->{label}; | ||
| 237 | my $hmm =$dmoot->{hmm}; | ||
| 238 | my $utf8 =$dmoot->{hmmUtf8}; | ||
| 239 | my $lctags =$dmoot->{lctags}; | ||
| 240 | my $dynbase =log($hmm->dynlex_base()); | ||
| 241 | my $logbase =log($hmm->dynlex_base()); | ||
| 242 | my $dynbeta =$hmm->dynlex_beta(); | ||
| 243 | my ($msent,$w,$mw,$text,$tmp, $analysesOk); | ||
| 244 | return sub { | ||
| 245 | $msent = [map { | ||
| 246 | $w = $_; | ||
| 247 | $analysesOk=1; | ||
| 248 | $mw = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{dmoot} if present | ||
| 249 | $text = $mw->{text} = (defined($mw->{word}) ? $mw->{word} : $w->{text}) if (!defined($text=$mw->{text})); | ||
| 250 | if (!$mw->{analyses}) { | ||
| 251 | if ($w->{exlex}) { | ||
| 252 | ##-- special case for exception lexicon: clobber all other alternatives | ||
| 253 | $mw->{analyses} = (ref($w->{exlex}) | ||
| 254 | ? [map {{tag=>$_->{text}, prob=>-log($_->{freq}||$dynbase)/($logbase*$dynbeta)}} @{$w->{exlex}}] ##-- non-deterministic: ARRAY | ||
| 255 | : [{tag=>$w->{exlex}, prob=>0}]); ##-- deterministic: string | ||
| 256 | } elsif ($w->{xr} && $w->{xr} =~ /\baq\b/) { | ||
| 257 | ##-- special case for antiqua typesetting in fraktur text: use unicruft for latinExt=1 to scrub out long s | ||
| 258 | $text = $w->{xlit}{latin1Text} if ($w->{xlit} && $w->{xlit}{isLatinExt}); | ||
| 259 | $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 260 | } elsif ($w->{msafe}) { | ||
| 261 | ##-- safe contemporary form: leave as-is | ||
| 262 | $mw->{analyses} = [{tag=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit | ||
| 263 | , prob=>0}]; | ||
| 264 | } elsif ($w->{xlit} && !$w->{xlit}{isLatinExt}) { | ||
| 265 | ##-- word with non-latin characters: leave as-is | ||
| 266 | $mw->{analyses} = [{tag=>$text, prob=>0}]; | ||
| 267 | } elsif ($_->{lang} && (ref($_->{lang}) ? (!grep {$_ eq "de"} @{$_->{lang}}) : ($_->{lang} ne "de"))) { | ||
| 268 | ##-- foreign material: leave as-is | ||
| 269 | $mw->{analyses} = [{tag=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit | ||
| 270 | , prob=>0}]; | ||
| 271 | } else { | ||
| 272 | $tmp=undef; | ||
| 273 | $mw->{analyses} = [ | ||
| 274 | (map {{tag=>$_->{hi}, prob=>($_->{w}||0)} ##-- _am_dmoot_fst2moota | ||
| 275 | } (sort {($a->{w}||0) <=> ($b->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} (map {$tmp && $tmp->{hi} eq $_->{hi} ? qw() : ($tmp=$_)} sort {($a->{hi}||"") cmp ($b->{hi}||"") || ($a->{w}||0) <=> ($b->{w}||0)} ({hi=>($w->{xlit} ? $w->{xlit}{latin1Text} : $w->{text}) ##== _am_xlit | ||
| 276 | , w=>(8.06*(length($text)**-1))} ##== _am_id_fst | ||
| 277 | , | ||
| 278 | ($w->{eqphox} ? (map {{ %{$_}, w=>((3.42+0.05*$_->{w})*(length($text)**-1)) } ##-- _am_fst_wcp | ||
| 279 | } @{$w->{eqphox}}) ##-- _am_fst_wcp_list | ||
| 280 | : qw()) ##-- _am_fst_wcp_listref | ||
| 281 | , | ||
| 282 | ($w->{rw} ? (map {{ %{$_}, w=>((1.07+0.04*$_->{w})*(length($text)**-1)) } ##-- _am_fst_wcp | ||
| 283 | } @{$w->{rw}}) ##-- _am_fst_wcp_list | ||
| 284 | : qw()) ##-- _am_fst_wcp_listref | ||
| 285 | )) ##== _am_fst_uniq | ||
| 286 | ) ##== _am_fst_sort | ||
| 287 | ) ##-- _am_dmoot_list2moota | ||
| 288 | |||
| 289 | ]; | ||
| 290 | foreach (@{$mw->{analyses}}) { | ||
| 291 | $_->{tag} =~ s/\[(.[^\]]*)\]/$1/g; ##-- un-escape: brackets | ||
| 292 | $_->{tag} =~ s/\\(.)/$1/g; ##-- un-escape: backslashes | ||
| 293 | } | ||
| 294 | $analysesOk=0 if (!@{$mw->{analyses}}); | ||
| 295 | } | ||
| 296 | $_->{details} = $_->{tag} foreach (@{$mw->{analyses}}); | ||
| 297 | } | ||
| 298 | if ($lctags) { $_->{tag}=lc($_->{tag}) foreach (@{$mw->{analyses}}); } | ||
| 299 | |||
| 300 | if (!$analysesOk) { | ||
| 301 | $dmoot->logwarn("no candidate analyses found for token text \"$text\": skipping sentence!"); | ||
| 302 | return; | ||
| 303 | } | ||
| 304 | |||
| 305 | $mw | ||
| 306 | } @{$_->{tokens}}]; | ||
| 307 | |||
| 308 | $hmm->tag_sentence($msent, $utf8); | ||
| 309 | |||
| 310 | foreach (@$msent) { | ||
| 311 | delete($_->{text}); | ||
| 312 | } | ||
| 313 | }; | ||
| 314 | } | ||
| 315 | |||
| 316 | ##============================================================================== | ||
| 317 | ## Methods: I/O | ||
| 318 | ##============================================================================== | ||
| 319 | |||
| 320 | ##-------------------------------------------------------------- | ||
| 321 | ## Methods: I/O: Input: all | ||
| 322 | |||
| 323 | ## $bool = $moot->ensureLoaded() | ||
| 324 | ## + ensures model data is loaded from default files | ||
| 325 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 326 | |||
| 327 | ##-------------------------------------------------------------- | ||
| 328 | ## Methods: I/O: Input: HMM | ||
| 329 | |||
| 330 | ## $moot = $moot->loadHMM($model_file) | ||
| 331 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 332 | |||
| 333 | ##============================================================================== | ||
| 334 | ## Methods: Persistence | ||
| 335 | ##============================================================================== | ||
| 336 | |||
| 337 | ##====================================================================== | ||
| 338 | ## Methods: Persistence: Perl | ||
| 339 | |||
| 340 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 341 | ## + returns list of keys not to be saved | ||
| 342 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 343 | |||
| 344 | ## $saveRef = $obj->savePerlRef() | ||
| 345 | ## + inherited from DTA::CAB::Persistent | ||
| 346 | |||
| 347 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 348 | ## + implicitly calls $obj->clear() | ||
| 349 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 350 | |||
| 351 | ##============================================================================== | ||
| 352 | ## Methods: Analysis | ||
| 353 | ## + inherited from DTA::CAB::Analyzer::Moot | ||
| 354 | |||
| 355 | |||
| 356 | 1; ##-- be happy | ||
| 357 | |||
| 358 | __END__ |