| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Moot.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: generic Moot analysis API | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Moot; | ||
| 8 | use DTA::CAB::Analyzer ':child'; | ||
| 9 | use DTA::CAB::Datum ':all'; | ||
| 10 | |||
| 11 | use Moot; | ||
| 12 | use Encode qw(encode decode); | ||
| 13 | use IO::File; | ||
| 14 | use Carp; | ||
| 15 | |||
| 16 | #use Devel::Cycle; ##-- debug | ||
| 17 | |||
| 18 | use strict; | ||
| 19 | |||
| 20 | ##============================================================================== | ||
| 21 | ## Globals | ||
| 22 | ##============================================================================== | ||
| 23 | |||
| 24 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 25 | |||
| 26 | ##============================================================================== | ||
| 27 | ## Closure stuff | ||
| 28 | |||
| 29 | ## $DEFAULT_ANALYZE_CODE | ||
| 30 | ## + default analysis code | ||
| 31 | ## + available variables: | ||
| 32 | ## $anl # Analyzer::Moot object | ||
| 33 | ## $_ # sentence to be analyzed | ||
| 34 | our $DEFAULT_ANALYZE_CODE = ' | ||
| 35 | package '. __PACKAGE__ .'; | ||
| 36 | my $moot=$anl; | ||
| 37 | my $lab =$moot->{label}; | ||
| 38 | my $hmm =$moot->{hmm}; | ||
| 39 | my $tagx=$moot->{tagx}; | ||
| 40 | my $utf8=$moot->{hmmUtf8}; | ||
| 41 | my $prune=$moot->{prune}; | ||
| 42 | my $lctext=$moot->{lctext}; | ||
| 43 | my $notag=$moot->{notag}; | ||
| 44 | my $use_dmoot=$moot->{use_dmoot}; | ||
| 45 | my $moot_lang=$moot->{lang}; | ||
| 46 | my $xpne=$moot->{xpne}; | ||
| 47 | my $xpfm=$moot->{xpfm}; | ||
| 48 | my $fmtag=$moot->{fmtag}; | ||
| 49 | my ($s,$msent,$w,$mw,$t,$at,$lang,$val); | ||
| 50 | sub { | ||
| 51 | $s = $_; | ||
| 52 | $msent = [map { | ||
| 53 | $w = $_; | ||
| 54 | $mw = $w->{$lab} = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{moot} if present | ||
| 55 | $mw->{text} = (defined($mw->{word}) ? $mw->{word} : '._am_tag('($use_dmoot ? $_->{dmoot} : undef)', _am_xlit).') if (!defined($mw->{text})); | ||
| 56 | $mw->{text} = lc($mw->{text}) if ($lctext); | ||
| 57 | $mw->{analyses} = [{tag=>"NE",details=>"NE.xp",prob=>0}] if ($xpne && ($w->{xp}//"") =~ /\b((?:pers)Name)\b/i); #place | ||
| 58 | $mw->{analyses} = [{tag=>$fmtag,details=>"$fmtag.xp",prob=>0}] if ($xpfm && ($w->{xp}//"") =~ /\bforeign\b/i); | ||
| 59 | $val = undef; ##-- temporary for _am_tagh_moota_uniq() | ||
| 60 | $mw->{analyses} = ['._am_tagh_list2moota_uniq('map {$_ ? @$_ : qw()} | ||
| 61 | @$w{qw(mlatin tokpp toka)}, | ||
| 62 | ($use_dmoot && $w->{xlit} && !$w->{xlit}{isLatinExt} ? [$fmtag, "XY"] : qw()), | ||
| 63 | ($use_dmoot && $w->{dmoot} ? $w->{dmoot}{morph} | ||
| 64 | : ($w->{morph}, ($w->{rw} ? (map {$_->{morph}} @{$w->{rw}}) : qw())))' | ||
| 65 | ).' | ||
| 66 | ] if (!defined($mw->{analyses})); | ||
| 67 | foreach (@{$mw->{analyses}}) { | ||
| 68 | ##-- tag-translation hack: apply BEFORE sending to moot! | ||
| 69 | $_->{tag}=$t if (defined($t=$tagx->{$_->{tag}})); | ||
| 70 | } | ||
| 71 | $mw | ||
| 72 | } @{$s->{tokens}}]; | ||
| 73 | return if (!@$msent); ##-- ignore empty sentences | ||
| 74 | |||
| 75 | $hmm->tag_sentence($msent, $utf8) if (!$notag); | ||
| 76 | |||
| 77 | ##-- language-guesser hack | ||
| 78 | if ($moot_lang && ($lang=$s->{lang}) && $lang ne $moot_lang) { | ||
| 79 | $_->{tag} = "$fmtag.$lang" foreach (grep {$_->{tag} !~ /^\$/} @$msent); | ||
| 80 | } | ||
| 81 | |||
| 82 | foreach (@$msent) { | ||
| 83 | $_->{word}=$_->{text}; | ||
| 84 | delete($_->{text}); | ||
| 85 | #$_->{tag}=$t if (defined($t=$tagx->{$_->{tag}//""})); ##-- do NOT translate output tags (breaks en-wsj, 2016-06-09) | ||
| 86 | $_->{tag}="NE" if ($xpne && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "NE.xp"); | ||
| 87 | $_->{tag}=$fmtag if ($xpfm && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "FM.xp"); | ||
| 88 | if ($prune) { | ||
| 89 | $t = $_->{tag}//""; | ||
| 90 | @{$_->{analyses}} = grep {$_->{tag} eq $t} @{$_->{analyses}}; | ||
| 91 | } | ||
| 92 | } | ||
| 93 | } | ||
| 94 | '; | ||
| 95 | |||
| 96 | |||
| 97 | ##============================================================================== | ||
| 98 | ## Constructors etc. | ||
| 99 | ##============================================================================== | ||
| 100 | |||
| 101 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 102 | ## + object structure: | ||
| 103 | ## ( | ||
| 104 | ## ##-- Filename Options | ||
| 105 | ## hmmFile => $filename, ##-- default: none (REQUIRED) | ||
| 106 | ## tagxFile => $tagxFile, ##-- tag-translation file (hack) | ||
| 107 | ## | ||
| 108 | ## ##-- Analysis Options | ||
| 109 | ## hmmArgs => \%args, ##-- clobber Moot::HMM->new() defaults (default: none) | ||
| 110 | ## hmmUtf8 => $bool, ##-- use hmm utf8 mode? (default=true) | ||
| 111 | ## | ||
| 112 | ## analyzeCode => $code, ##-- pseudo-closure: analyze current sentence $_ | ||
| 113 | ## label => $lab, ##-- destination key (default='moot') | ||
| 114 | ## prune => $bool, ##-- if true (default), prune analyses after tagging | ||
| 115 | ## lctext => $bool, ##-- if true, input text will be bashed to lower-case (default: false) | ||
| 116 | ## notag => $bool, ##-- if true, hmm tagger won't actually be called; read from global analyzer options as "${lab}.notag" | ||
| 117 | ## use_dmoot => $bool, ##-- if true, hmm tagger will try to get text & analyses from token {dmoot} key, otherwise it will be ignored | ||
| 118 | ## xpne => $bool, ##-- if true, force 'NE' tags whenever $w->{xp} =~ /\b(?:pers)Name\b/i (default=true) #NOT 'placeName', the tags are often appositions | ||
| 119 | ## xpfm => $bool, ##-- if true, force 'FM' tags whenever $w->{xp} =~ /\bforeign\b/i (default=true) | ||
| 120 | ## lang => $lang, ##-- only tag sentences marked as language "$lang"; read from global analyzer options as "${lab}.lang" (default='de') | ||
| 121 | ## fmtag => $fmtag, ##-- prefix for "foreign-material" tags; default='FM' | ||
| 122 | ## | ||
| 123 | ## ##-- Analysis Objects | ||
| 124 | ## hmm => $hmm, ##-- a moot::HMM object | ||
| 125 | ## tagx => \%tagx, ##-- tag-translation table (loaded via DTA::CAB::Analyzer::Dict from $tagxFile) | ||
| 126 | ## ) | ||
| 127 | sub new { | ||
| 128 | my $that = shift; | ||
| 129 | my $moot = $that->SUPER::new( | ||
| 130 | ##-- filenames | ||
| 131 | hmmFile => undef, | ||
| 132 | tagxFile => undef, | ||
| 133 | |||
| 134 | ##-- options | ||
| 135 | hmmArgs => { | ||
| 136 | #verbose=>Moot::vlWarnings, | ||
| 137 | #relax => 1, | ||
| 138 | }, | ||
| 139 | hmmUtf8 => 1, | ||
| 140 | |||
| 141 | #prune => 1, | ||
| 142 | #uniqueAnalyses=>0, | ||
| 143 | |||
| 144 | ##-- analysis I/O | ||
| 145 | #analysisClass => 'DTA::CAB::Analyzer::Moot::Analysis', | ||
| 146 | label => 'moot', | ||
| 147 | analyzeCode => $DEFAULT_ANALYZE_CODE, | ||
| 148 | lctext => 0, | ||
| 149 | #notag => undef, | ||
| 150 | #use_dmoot => undef, | ||
| 151 | lang => 'de', | ||
| 152 | xpne => 1, | ||
| 153 | xpfm => 1, | ||
| 154 | fmtag => 'FM', | ||
| 155 | |||
| 156 | #analyzeCostFuncs => {}, | ||
| 157 | #requireAnalyses => 0, | ||
| 158 | #wantTaggedWord => 1, | ||
| 159 | |||
| 160 | ##-- analysis objects | ||
| 161 | #hmm => undef, | ||
| 162 | |||
| 163 | ##-- user args | ||
| 164 | @_ | ||
| 165 | ); | ||
| 166 | return $moot; | ||
| 167 | } | ||
| 168 | |||
| 169 | ## $moot = $moot->clear() | ||
| 170 | sub clear { | ||
| 171 | my $moot = shift; | ||
| 172 | |||
| 173 | ##-- analysis sub(s) | ||
| 174 | $moot->dropClosures(); | ||
| 175 | |||
| 176 | ##-- analysis objects | ||
| 177 | delete($moot->{hmm}); | ||
| 178 | delete($moot->{tagx}); | ||
| 179 | |||
| 180 | return $moot; | ||
| 181 | } | ||
| 182 | |||
| 183 | ## @keys = $anl->typeKeys(\%opts) | ||
| 184 | ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes() | ||
| 185 | ## + override returns empty list | ||
| 186 | sub typeKeys { | ||
| 187 | return qw(); | ||
| 188 | } | ||
| 189 | |||
| 190 | ##============================================================================== | ||
| 191 | ## Methods: Generic | ||
| 192 | ##============================================================================== | ||
| 193 | |||
| 194 | ## $bool = $moot->hmmOk() | ||
| 195 | ## + should return false iff HMM is undefined or "empty" | ||
| 196 | ## + default version checks for non-empty 'lexprobs' and 'n_tags' | ||
| 197 | sub hmmOk { | ||
| 198 | return defined($_[0]{hmm}) && $_[0]{hmm}->n_tags()>1 && $_[0]{hmm}->n_toks()>1; | ||
| 199 | #&& $_[0]{hmm}{lexprobs}->size > 1; | ||
| 200 | } | ||
| 201 | |||
| 202 | ## $class = $moot->hmmClass() | ||
| 203 | ## + returns class for $moot->{hmm} object | ||
| 204 | ## + default just returns 'Moot::HMM' | ||
| 205 | sub hmmClass { return 'Moot::HMM'; } | ||
| 206 | |||
| 207 | ##============================================================================== | ||
| 208 | ## Methods: I/O | ||
| 209 | ##============================================================================== | ||
| 210 | |||
| 211 | ##-------------------------------------------------------------- | ||
| 212 | ## Methods: I/O: Input: all | ||
| 213 | |||
| 214 | ## $bool = $moot->ensureLoaded() | ||
| 215 | ## + ensures model data is loaded from default files (if available) | ||
| 216 | sub ensureLoaded { | ||
| 217 | my $moot = shift; | ||
| 218 | my $rc = 1; ##-- allow empty models | ||
| 219 | |||
| 220 | ##-- ensure: hmm | ||
| 221 | $rc &&= $moot->loadHMM($moot->{hmmFile}) if (defined($moot->{hmmFile}) && !$moot->hmmOk); | ||
| 222 | |||
| 223 | ##-- ensure: dict: tagx | ||
| 224 | $rc &&= $moot->ensureDict('tagx',{}) if (!$moot->{tagx}); | ||
| 225 | |||
| 226 | return $rc; | ||
| 227 | } | ||
| 228 | |||
| 229 | ##-------------------------------------------------------------- | ||
| 230 | ## Methods: I/O: Input: HMM | ||
| 231 | |||
| 232 | ## $moot = $moot->loadHMM($model_file) | ||
| 233 | BEGIN { *loadHMM = *loadHmm = \&loadHMM; } | ||
| 234 | sub loadHMM { | ||
| 235 | my ($moot,$model) = @_; | ||
| 236 | my $hmmClass = $moot->hmmClass; | ||
| 237 | $moot->info("loading HMM model file '$model' using HMM class '$hmmClass'"); | ||
| 238 | if (!defined($moot->{hmm})) { | ||
| 239 | $moot->{hmm} = $hmmClass->new() | ||
| 240 | or $moot->logconfess("could not create HMM object of class '$hmmClass': $!"); | ||
| 241 | $moot->{hmm}->config($moot->{hmmArgs}) if ($moot->{hmmArgs}); | ||
| 242 | } | ||
| 243 | $moot->{hmm}->load($model) | ||
| 244 | or $moot->logconfess("loadHMM(): load failed for '$model': $!"); | ||
| 245 | $moot->dropClosures(); | ||
| 246 | return $moot; | ||
| 247 | } | ||
| 248 | |||
| 249 | ##-------------------------------------------------------------- | ||
| 250 | ## Methods: I/O: Input: Dictionaries: generic | ||
| 251 | |||
| 252 | ## $bool = $a->ensureDict($dictName,\%dictDefault) | ||
| 253 | sub ensureDict { | ||
| 254 | my ($a,$name,$default) = @_; | ||
| 255 | return 1 if ($a->{$name}); ##-- already defined | ||
| 256 | return $a->loadDict($name,$a->{"${name}File"}) if ($a->{"${name}File"}); | ||
| 257 | $a->{$name} = $default ? {%$default} : {}; | ||
| 258 | return 1; | ||
| 259 | } | ||
| 260 | |||
| 261 | ## \%dictHash_or_undef = $a->loadDict($dictName,$dictFile) | ||
| 262 | sub loadDict { | ||
| 263 | my ($a,$name,$dfile) = @_; | ||
| 264 | delete($a->{$name}); | ||
| 265 | my $dclass = 'DTA::CAB::Analyzer::Dict'; | ||
| 266 | $a->info("loading map from '$dfile' as $dclass"); | ||
| 267 | |||
| 268 | ##-- hack: generate a temporary dict object | ||
| 269 | my $dict = $dclass->new(label=>($a->{label}.".dict.$name"), dictFile=>$dfile); | ||
| 270 | $dict->ensureLoaded(); | ||
| 271 | return undef if (!$dict->dictOk); | ||
| 272 | |||
| 273 | ##-- clobber dict | ||
| 274 | $a->{$name} = $dict->dictHash; | ||
| 275 | } | ||
| 276 | |||
| 277 | |||
| 278 | |||
| 279 | ##============================================================================== | ||
| 280 | ## Methods: Persistence | ||
| 281 | ##============================================================================== | ||
| 282 | |||
| 283 | ##====================================================================== | ||
| 284 | ## Methods: Persistence: Perl | ||
| 285 | |||
| 286 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 287 | ## + returns list of keys not to be saved | ||
| 288 | sub noSaveKeys { | ||
| 289 | my $that = shift; | ||
| 290 | return ($that->SUPER::noSaveKeys, qw(hmm)); | ||
| 291 | } | ||
| 292 | |||
| 293 | ## $saveRef = $obj->savePerlRef() | ||
| 294 | ## + inherited from DTA::CAB::Persistent | ||
| 295 | |||
| 296 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 297 | ## + implicitly calls $obj->clear() | ||
| 298 | sub loadPerlRef { | ||
| 299 | my ($that,$ref) = @_; | ||
| 300 | my $obj = $that->SUPER::loadPerlRef($ref); | ||
| 301 | $obj->clear(); | ||
| 302 | return $obj; | ||
| 303 | } | ||
| 304 | |||
| 305 | ##============================================================================== | ||
| 306 | ## Methods: Analysis | ||
| 307 | ##============================================================================== | ||
| 308 | |||
| 309 | ##------------------------------------------------------------------------ | ||
| 310 | ## Methods: Analysis: Generic | ||
| 311 | |||
| 312 | ## $bool = $anl->canAnalyze() | ||
| 313 | ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty) | ||
| 314 | sub canAnalyze { | ||
| 315 | return $_[0]->hmmOk(); | ||
| 316 | } | ||
| 317 | |||
| 318 | |||
| 319 | ##------------------------------------------------------------------------ | ||
| 320 | ## Methods: Analysis: v1.x: API | ||
| 321 | |||
| 322 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 323 | ## + override: only allow analyzeSentences() | ||
| 324 | sub doAnalyze { | ||
| 325 | my $anl = shift; | ||
| 326 | return 0 if (defined($_[1]) && $_[1] ne 'Sentences'); | ||
| 327 | return $anl->SUPER::doAnalyze(@_); | ||
| 328 | } | ||
| 329 | |||
| 330 | ## $doc = $anl->analyzeSentences($doc,\%opts) | ||
| 331 | ## + perform sentence-wise analysis of all sentences $doc->{body}[$si] | ||
| 332 | ## + no default implementation | ||
| 333 | sub analyzeSentences { | ||
| 334 | my ($moot,$doc,$opts) = @_; | ||
| 335 | return undef if (!$moot->ensureLoaded()); ##-- uh-oh... | ||
| 336 | return $doc if (!$moot->canAnalyze); ##-- ok... | ||
| 337 | $doc = toDocument($doc); | ||
| 338 | |||
| 339 | ##-- inherit global options | ||
| 340 | my $notag = $moot->{notag}; | ||
| 341 | my $use_dmoot = $moot->{use_dmoot}; | ||
| 342 | $moot->{notag} //= $opts->{"$moot->{label}.notag"}; | ||
| 343 | $moot->{use_dmoot} //= $opts->{"$moot->{label}.use_dmoot"}; | ||
| 344 | $moot->{use_dmoot} //= 1; | ||
| 345 | $moot->{lang} = $opts->{"$moot->{label}.lang"} if ($opts->{"$moot->{label}.lang"}); | ||
| 346 | |||
| 347 | ##-- setup access closures | ||
| 348 | my $acode_str = $moot->analysisCode(); | ||
| 349 | my $acode_sub = $moot->accessClosure($acode_str); | ||
| 350 | |||
| 351 | ##-- ye olde loope | ||
| 352 | foreach (@{$doc->{body}}) { | ||
| 353 | $acode_sub->(); | ||
| 354 | } | ||
| 355 | |||
| 356 | ##-- restore local options | ||
| 357 | $moot->{notag} = $notag; | ||
| 358 | $moot->{use_dmoot} = $use_dmoot; | ||
| 359 | |||
| 360 | ##-- DEBUG | ||
| 361 | #print STDERR "find_cycle()\n"; | ||
| 362 | #find_cycle([$moot,$doc,$acode_sub]); | ||
| 363 | ##--/DEBUG | ||
| 364 | |||
| 365 | return $doc; | ||
| 366 | } | ||
| 367 | |||
| 368 | ##------------------------------------------------------------------------ | ||
| 369 | ## Methods: Analysis: Closure Utilities | ||
| 370 | |||
| 371 | ## $asub_code = $moot->analysisCode() | ||
| 372 | ## + analysis closure for passing to Analyzer::accessClosure() | ||
| 373 | ## + default just returns $moot->{analyzeCode} || $DEFAULT_ANALYZE_CODE | ||
| 374 | sub analysisCode { | ||
| 375 | my $moot = shift; | ||
| 376 | return | ||
| 377 | #analysisCodeDEBUG($moot) || | ||
| 378 | $moot->{analyzeCode} || $DEFAULT_ANALYZE_CODE; | ||
| 379 | } | ||
| 380 | |||
| 381 | ##------------------------------------------------------------------------ | ||
| 382 | ## Methods: Analysis: DEBUG | ||
| 383 | |||
| 384 | ## analysisCodeDEBUG() : from $DEFAULT_ANALYZE_CODE | ||
| 385 | sub analysisCodeDEBUG { | ||
| 386 | my $anl = shift; | ||
| 387 | |||
| 388 | ## copy+paste from debugger "print $DEFAULT_ANALYZE_CODE" after breaking first e.g. in analysisCode() above | ||
| 389 | package DTA::CAB::Analyzer::Moot; | ||
| 390 | my $moot=$anl; | ||
| 391 | my $lab =$moot->{label}; | ||
| 392 | my $hmm =$moot->{hmm}; | ||
| 393 | my $tagx=$moot->{tagx}; | ||
| 394 | my $utf8=$moot->{hmmUtf8}; | ||
| 395 | my $prune=$moot->{prune}; | ||
| 396 | my $lctext=$moot->{lctext}; | ||
| 397 | my $notag=$moot->{notag}; | ||
| 398 | my $use_dmoot=$moot->{use_dmoot}; | ||
| 399 | my $moot_lang=$moot->{lang}; | ||
| 400 | my $xpne=$moot->{xpne}; | ||
| 401 | my $xpfm=$moot->{xpfm}; | ||
| 402 | my $fmtag=$moot->{fmtag}; | ||
| 403 | my ($s,$msent,$w,$mw,$t,$at,$lang,$val); | ||
| 404 | sub { | ||
| 405 | $s = $_; | ||
| 406 | $msent = [map { | ||
| 407 | $w = $_; | ||
| 408 | $mw = $w->{$lab} = $w->{$lab} ? {%{$w->{$lab}}} : ($w->{$lab}={}); ##-- copy $w->{moot} if present | ||
| 409 | $mw->{text} = (defined($mw->{word}) ? $mw->{word} : (($use_dmoot ? $_->{dmoot} : undef) ? ($use_dmoot ? $_->{dmoot} : undef)->{tag} : ($_->{xlit} ? $_->{xlit}{latin1Text} : $_->{text}) ##== _am_xlit | ||
| 410 | ) ##== _am_tag | ||
| 411 | ) if (!defined($mw->{text})); | ||
| 412 | $mw->{text} = lc($mw->{text}) if ($lctext); | ||
| 413 | $mw->{analyses} = [{tag=>"NE",details=>"NE.xp",prob=>0}] if ($xpne && ($w->{xp}//"") =~ /\b((?:pers)Name)\b/i); #place | ||
| 414 | $mw->{analyses} = [{tag=>$fmtag,details=>"$fmtag.xp",prob=>0}] if ($xpfm && ($w->{xp}//"") =~ /\bforeign\b/i); | ||
| 415 | $val = undef; ##-- temporary for _am_tagh_moota_uniq() | ||
| 416 | $mw->{analyses} = [(map {$val && $val->{details} eq $_->{details} ? qw() : ($val=$_)} sort {($a->{details}//"") cmp ($b->{details}//"") || ($a->{prob}//0) <=> ($b->{prob}//0)} (map {{details=>$_->{hi}, prob=>($_->{w}||0), tag=>($_->{hi} =~ /\[\_?((?:[A-Za-z0-9.]+|\$[^\]]+))\]/ ? $1 : $_->{hi})} ##-- _am_tagh_fst2moota | ||
| 417 | } map {ref($_) ? $_ : {hi=>$_}} map {$_ ? @$_ : qw()} | ||
| 418 | @$w{qw(mlatin tokpp toka)}, | ||
| 419 | ($use_dmoot && $w->{xlit} && !$w->{xlit}{isLatinExt} ? [$fmtag, "XY"] : qw()), | ||
| 420 | ($use_dmoot && $w->{dmoot} ? $w->{dmoot}{morph} | ||
| 421 | : ($w->{morph}, ($w->{rw} ? (map {$_->{morph}} @{$w->{rw}}) : qw())))) ##-- _am_tagh_list2moota | ||
| 422 | ) ##== _am_tagh_moota_uniq | ||
| 423 | |||
| 424 | ] if (!defined($mw->{analyses})); | ||
| 425 | foreach (@{$mw->{analyses}}) { | ||
| 426 | ##-- tag-translation hack: apply BEFORE sending to moot! | ||
| 427 | $_->{tag}=$t if (defined($t=$tagx->{$_->{tag}})); | ||
| 428 | } | ||
| 429 | $mw | ||
| 430 | } @{$s->{tokens}}]; | ||
| 431 | return if (!@$msent); ##-- ignore empty sentences | ||
| 432 | |||
| 433 | $hmm->tag_sentence($msent, $utf8) if (!$notag); | ||
| 434 | |||
| 435 | ##-- language-guesser hack | ||
| 436 | if ($moot_lang && ($lang=$s->{lang}) && $lang ne $moot_lang) { | ||
| 437 | $_->{tag} = "$fmtag.$lang" foreach (grep {$_->{tag} !~ /^\$/} @$msent); | ||
| 438 | } | ||
| 439 | |||
| 440 | foreach (@$msent) { | ||
| 441 | $_->{word}=$_->{text}; | ||
| 442 | delete($_->{text}); | ||
| 443 | #$_->{tag}=$t if (defined($t=$tagx->{$_->{tag}//""})); ##-- do NOT translate output tags (breaks en-wsj, 2016-06-09) | ||
| 444 | $_->{tag}="NE" if ($xpne && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "NE.xp"); | ||
| 445 | $_->{tag}=$fmtag if ($xpfm && $_->{analyses} && $_->{analyses}[0] && $_->{analyses}[0]{details} eq "FM.xp"); | ||
| 446 | if ($prune) { | ||
| 447 | $t = $_->{tag}//""; | ||
| 448 | @{$_->{analyses}} = grep {$_->{tag} eq $t} @{$_->{analyses}}; | ||
| 449 | } | ||
| 450 | } | ||
| 451 | } | ||
| 452 | } | ||
| 453 | |||
| 454 | 1; ##-- be happy | ||
| 455 | |||
| 456 | __END__ |