| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Automaton.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: generic analysis automaton API | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Automaton; | ||
| 8 | use DTA::CAB::Analyzer ':child'; | ||
| 9 | use DTA::CAB::Unify ':all'; | ||
| 10 | use Gfsm; | ||
| 11 | use Encode qw(encode decode); | ||
| 12 | use IO::File; | ||
| 13 | #use File::Basename qw(); | ||
| 14 | use Carp; | ||
| 15 | |||
| 16 | use strict; | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Globals | ||
| 20 | ##============================================================================== | ||
| 21 | |||
| 22 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 23 | |||
| 24 | ## $DEFAULT_ANALYZE_GET | ||
| 25 | ## + default coderef or eval-able string for {analyzeGet} | ||
| 26 | our $DEFAULT_ANALYZE_GET = '$_->{$lab} ? qw() : '._am_xlit; | ||
| 27 | |||
| 28 | ## $DEFAULT_ANALYZE_SET | ||
| 29 | ## + default coderef or eval-able string for {analyzeSet} | ||
| 30 | ## + vars: | ||
| 31 | ## $anl => analyzer (automaton) | ||
| 32 | ## $lab => analyzer label | ||
| 33 | ## $_ => token | ||
| 34 | ## $wa => analyses (array-ref) | ||
| 35 | our $DEFAULT_ANALYZE_SET = '$_->{$lab} = ($wa && @$wa ? [@$wa] : undef);'; | ||
| 36 | |||
| 37 | ##============================================================================== | ||
| 38 | ## Constructors etc. | ||
| 39 | ##============================================================================== | ||
| 40 | |||
| 41 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 42 | ## + object structure: | ||
| 43 | ## ( | ||
| 44 | ## ##-- Filename Options | ||
| 45 | ## fstFile => $filename, ##-- source FST file (default: none) | ||
| 46 | ## labFile => $filename, ##-- source labels file (default: none) | ||
| 47 | ## | ||
| 48 | ## ##-- Exception lexicon options (OBSOLETE for CAB >=v1.15) | ||
| 49 | ## #dictFile => $filename, ##-- source dict file (default: none): clobbers $dict->{dictFile} if defined | ||
| 50 | ## #dict => $dict, ##-- exception lexicon as a DTA::CAB::Analyzer::Dict object or option hash | ||
| 51 | ## # ## + default=undef | ||
| 52 | ## #dictClass => $class, ##-- fallback class for new dict (default='DTA::CAB::Analyzer::Dict') | ||
| 53 | ## | ||
| 54 | ## ##-- Analysis Output | ||
| 55 | ## analyzeGet => $code, ##-- accessor: coderef or string: source text (default=$DEFAULT_ANALYZE_GET; return undef for no analysis) | ||
| 56 | ## analyzeSet => $code, ##-- accessor: coderef or string: set analyses (default=$DEFAULT_ANALYZE_SET) | ||
| 57 | ## analyzePre => $code, ##-- coderef or string: pre-analysis code to call (called as $code->($word)) | ||
| 58 | ## analyzePost => $code, ##-- coderef or string: post-analysis code to call (called as $code->($word)) | ||
| 59 | ## wantAnalysisLo => $bool, ##-- set to true to include 'lo' keys in analyses (default: true) | ||
| 60 | ## #wantAnalysisFst => $bool, ##-- set to true to include 'fst' key in analyses (default: false) | ||
| 61 | ## wantAnalysisLemma => $bool, ##-- set to true to include 'lemma' keys in analyses (default: false) | ||
| 62 | ## | ||
| 63 | ## ##-- Analysis Options | ||
| 64 | ## eow => $sym, ##-- EOW symbol for analysis FST | ||
| 65 | ## check_symbols => $bool, ##-- check for unknown symbols? (default=1) | ||
| 66 | ## labenc => $enc, ##-- encoding of labels file (default='auto' [utf8 if applicable, otherwise latin1]) | ||
| 67 | ## #dictenc => $enc, ##-- dictionary encoding (default='UTF-8') (set $aut->{dict}{encoding} instead) | ||
| 68 | ## auto_connect => $bool, ##-- whether to call $result->_connect() after every lookup (default=0) | ||
| 69 | ## tolower => $bool, ##-- if true, all input words will be bashed to lower-case (default=0) | ||
| 70 | ## tolowerNI => $bool, ##-- if true, all non-initial characters of inputs will be lower-cased (default=0) | ||
| 71 | ## toupperI => $bool, ##-- if true, initial character will be upper-cased (default=0) | ||
| 72 | ## capsFallback => $bool, ##-- if true, all-caps words will also be analyzed as ucfirst(lc($w)) (default=0) | ||
| 73 | ## qsFallback => $bool, ##-- if true, "-'s" suffixes will also be analyzed as "-s" (default=0) | ||
| 74 | ## bashWS => $str, ##-- if defined, input whitespace will be bashed to '$str' (default='_') | ||
| 75 | ## attInput => $bool, ##-- if true, respect AT&T lextools-style escapes in input (default=0) | ||
| 76 | ## attOutput => $bool, ##-- if true, generate AT&T escapes in output (default=1) | ||
| 77 | ## allowTextRegex => $re, ##-- if defined, only tokens with matching 'text' will be analyzed (default: none) | ||
| 78 | ## ## : useful: /^(?:(?:[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+)|(?:[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$/ | ||
| 79 | ## ## : == DTA::CAB::Analyzer::_am_wordlike_regex() | ||
| 80 | ## | ||
| 81 | ## ##-- Analysis objects | ||
| 82 | ## fst => $gfst, ##-- (child classes only) e.g. a Gfsm::Automaton object (default=new) | ||
| 83 | ## lab => $lab, ##-- (child classes only) e.g. a Gfsm::Alphabet object (default=new) | ||
| 84 | ## labh => \%sym2lab, ##-- (?) label hash: $sym2lab{$labSym} = $labId; | ||
| 85 | ## laba => \@lab2sym, ##-- (?) label array: $lab2sym[$labId] = $labSym; | ||
| 86 | ## labc => \@chr2lab, ##-- (?)chr-label array: $chr2lab[ord($chr)] = $labId;, by unicode char number (e.g. unpack('U0U*')) | ||
| 87 | ## warned_symbols => \%sym2undef, ##-- tracks unknown symbols we've already warned about (for check_symbols != 0) | ||
| 88 | ## | ||
| 89 | ## ##-- INHERITED from DTA::CAB::Analyzer | ||
| 90 | ## label => $label, ##-- analyzer label (default: from analyzer class name) | ||
| 91 | ## typeKeys => \@keys, ##-- type-wise keys to expand | ||
| 92 | ## ) | ||
| 93 | sub new { | ||
| 94 | my $that = shift; | ||
| 95 | my $aut = $that->SUPER::new( | ||
| 96 | ##-- filenames | ||
| 97 | fstFile => undef, | ||
| 98 | labFile => undef, | ||
| 99 | |||
| 100 | ##-- analysis objects | ||
| 101 | fst=>undef, | ||
| 102 | lab=>undef, | ||
| 103 | result=>undef, | ||
| 104 | labh=>{}, | ||
| 105 | laba=>[], | ||
| 106 | labc=>[], | ||
| 107 | warned_symbols=>{}, | ||
| 108 | |||
| 109 | ##-- options | ||
| 110 | eow =>'', | ||
| 111 | check_symbols => 1, | ||
| 112 | labenc => 'auto', | ||
| 113 | auto_connect => 0, | ||
| 114 | tolower => 0, | ||
| 115 | tolowerNI => 0, | ||
| 116 | toupperI => 0, | ||
| 117 | capsFallback => 0, | ||
| 118 | qsFallback => 0, | ||
| 119 | bashWS => '_', | ||
| 120 | attInput => 0, | ||
| 121 | attOutput => 1, | ||
| 122 | allowTextRegex => undef, #'(?:^[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+$)|(?:^[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+$)', | ||
| 123 | #'^(?:(?:[[:alpha:]\-\@\x{ac}]*[[:alpha:]]+)|(?:[[:alpha:]]+[[:alpha:]\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$' | ||
| 124 | # == DTA::CAB::Analyzer::_am_wordlike_regex() | ||
| 125 | ##-- analysis I/O | ||
| 126 | analyzeSrc => 'text', | ||
| 127 | wantAnalysisLo => 1, | ||
| 128 | wantAnalysisLemma => 0, | ||
| 129 | |||
| 130 | ##-- user args | ||
| 131 | @_ | ||
| 132 | ); | ||
| 133 | return $aut; | ||
| 134 | } | ||
| 135 | |||
| 136 | ## $aut = $aut->clear() | ||
| 137 | sub clear { | ||
| 138 | my $aut = shift; | ||
| 139 | |||
| 140 | ##-- analysis sub(s) | ||
| 141 | $aut->dropClosures(); | ||
| 142 | |||
| 143 | ##-- analysis objects | ||
| 144 | delete($aut->{fst}); | ||
| 145 | delete($aut->{lab}); | ||
| 146 | delete($aut->{result}); ##-- should be undef anyways | ||
| 147 | %{$aut->{labh}} = qw(); | ||
| 148 | @{$aut->{laba}} = qw(); | ||
| 149 | @{$aut->{labc}} = qw(); | ||
| 150 | |||
| 151 | return $aut; | ||
| 152 | } | ||
| 153 | |||
| 154 | ##============================================================================== | ||
| 155 | ## Methods: Generic | ||
| 156 | ##============================================================================== | ||
| 157 | |||
| 158 | ## $class = $aut->fstClass() | ||
| 159 | ## + default FST class for loadFst() method | ||
| 160 | sub fstClass { return 'Gfsm::Automaton'; } | ||
| 161 | |||
| 162 | ## $class = $aut->labClass() | ||
| 163 | ## + default labels class for loadLabels() method | ||
| 164 | sub labClass { return 'Gfsm::Alphabet'; } | ||
| 165 | |||
| 166 | ## $bool = $aut->fstOk() | ||
| 167 | ## + should return false iff fst is undefined or "empty" | ||
| 168 | sub fstOk { return defined($_[0]{fst}) && $_[0]{fst}->n_states>0; } | ||
| 169 | |||
| 170 | ## $bool = $aut->labOk() | ||
| 171 | ## + should return false iff label-set is undefined or "empty" | ||
| 172 | sub labOk { return defined($_[0]{lab}) && $_[0]{lab}->size>0; } | ||
| 173 | |||
| 174 | ##============================================================================== | ||
| 175 | ## Methods: I/O | ||
| 176 | ##============================================================================== | ||
| 177 | |||
| 178 | ##-------------------------------------------------------------- | ||
| 179 | ## Methods: I/O: Input: all | ||
| 180 | |||
| 181 | ## $bool = $aut->ensureLoaded() | ||
| 182 | ## + ensures automaton data is loaded from default files | ||
| 183 | sub ensureLoaded { | ||
| 184 | my $aut = shift; | ||
| 185 | my $rc = 1; | ||
| 186 | ##-- ensure: fst | ||
| 187 | if ( defined($aut->{fstFile}) && !$aut->fstOk ) { | ||
| 188 | $rc &&= $aut->loadFst($aut->{fstFile}); | ||
| 189 | } | ||
| 190 | ##-- ensure: lab | ||
| 191 | if ( defined($aut->{labFile}) && !$aut->labOk ) { | ||
| 192 | $rc &&= $aut->loadLabels($aut->{labFile}); | ||
| 193 | } | ||
| 194 | return $rc; | ||
| 195 | } | ||
| 196 | |||
| 197 | ## $aut = $aut->load(fst=>$fstFile, lab=>$labFile) | ||
| 198 | sub load { | ||
| 199 | my ($aut,%args) = @_; | ||
| 200 | return 0 if (!grep {defined($_)} @args{qw(fst lab dict)}); | ||
| 201 | my $rc = $aut; | ||
| 202 | $rc &&= $aut->loadFst($args{fst}) if (defined($args{fst})); | ||
| 203 | $rc &&= $aut->loadLabels($args{lab}) if (defined($args{lab})); | ||
| 204 | return $rc; | ||
| 205 | } | ||
| 206 | |||
| 207 | ##-------------------------------------------------------------- | ||
| 208 | ## Methods: I/O: Input: FST | ||
| 209 | |||
| 210 | ## $aut = $aut->loadFst($fstfile) | ||
| 211 | sub loadFst { | ||
| 212 | my ($aut,$fstfile) = @_; | ||
| 213 | $aut->info("loading FST file '$fstfile'"); | ||
| 214 | $aut->{fst} = $aut->fstClass->new() if (!defined($aut->{fst})); | ||
| 215 | $aut->{fst}->load($fstfile) | ||
| 216 | or $aut->logconfess("loadFst(): load failed for '$fstfile': $!"); | ||
| 217 | delete($aut->{result}); | ||
| 218 | #$aut->{result} = $aut->{fst}->shadow; #if (defined($aut->{result}) && $aut->{fst}->can('shadow')); | ||
| 219 | delete($aut->{_analyze}); | ||
| 220 | return $aut; | ||
| 221 | } | ||
| 222 | |||
| 223 | ## $result = $aut->resultFst() | ||
| 224 | ## + returns empty result FST | ||
| 225 | sub resultFst { | ||
| 226 | return $_[0]{fst} ? $_[0]{fst}->shadow : undef; | ||
| 227 | } | ||
| 228 | |||
| 229 | ##-------------------------------------------------------------- | ||
| 230 | ## Methods: I/O: Input: Labels | ||
| 231 | |||
| 232 | ## $aut = $aut->loadLabels($labfile) | ||
| 233 | sub loadLabels { | ||
| 234 | my ($aut,$labfile) = @_; | ||
| 235 | $aut->info("loading labels file '$labfile'"); | ||
| 236 | $aut->{lab} = $aut->labClass->new() if (!defined($aut->{lab})); | ||
| 237 | $aut->{lab}->load($labfile) | ||
| 238 | or $aut->logconfess("loadLabels(): load failed for '$labfile': $!"); | ||
| 239 | if (!$aut->{labenc} || $aut->{labenc} eq 'auto') { | ||
| 240 | ##-- guess label encoding | ||
| 241 | my $buf = join('',@{$aut->{lab}->toArray}); | ||
| 242 | $aut->{labenc} = utf8::decode($buf) ? 'utf8' : 'latin1'; | ||
| 243 | $aut->debug("loadLabels(): guessed label encoding '$aut->{labenc}'"); | ||
| 244 | } | ||
| 245 | $aut->{lab}->utf8(1) | ||
| 246 | if ($aut->{lab}->can('utf8') && (($aut->{labenc}||'') =~ /^utf\-?8$/i)); | ||
| 247 | $aut->parseLabels(); | ||
| 248 | delete($aut->{_analyze}); | ||
| 249 | return $aut; | ||
| 250 | } | ||
| 251 | |||
| 252 | ## $aut = $aut->parseLabels() | ||
| 253 | ## + sets up $aut->{labh}, $aut->{laba}, $aut->{labc} | ||
| 254 | ## + fixes encoding difficulties in $aut->{labh}, $aut->{laba} | ||
| 255 | sub parseLabels { | ||
| 256 | my $aut = shift; | ||
| 257 | my $laba = $aut->{laba}; | ||
| 258 | @$laba = @{$aut->{lab}->asArray}; | ||
| 259 | my ($i); | ||
| 260 | foreach $i (grep { defined($laba->[$_]) } 0..$#$laba) { | ||
| 261 | $laba->[$i] = decode($aut->{labenc}, $laba->[$i]) if ($aut->{labenc}); | ||
| 262 | $aut->{labh}{$laba->[$i]} = $i; | ||
| 263 | } | ||
| 264 | ##-- setup labc: $labId = $labc->[ord($c)]; ##-- single unicode characater | ||
| 265 | ## : @labIds = @$labc[unpack('U0U*',$s)]; ##-- batch lookup for strings (fast) | ||
| 266 | my @csyms = grep {defined($_) && length($_)==1} @$laba; ##-- @csyms = ($sym1, ...) s.t. each sym has len==1 | ||
| 267 | @{$aut->{labc}}[map {ord($_)} @csyms] = @{$aut->{labh}}{@csyms}; | ||
| 268 | ## | ||
| 269 | return $aut; | ||
| 270 | } | ||
| 271 | |||
| 272 | ##============================================================================== | ||
| 273 | ## Methods: Persistence | ||
| 274 | ##============================================================================== | ||
| 275 | |||
| 276 | ##====================================================================== | ||
| 277 | ## Methods: Persistence: Perl | ||
| 278 | |||
| 279 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 280 | ## + returns list of keys not to be saved | ||
| 281 | sub noSaveKeys { | ||
| 282 | my $that = shift; | ||
| 283 | return ($that->SUPER::noSaveKeys, qw(dict fst lab laba labc labh result)); | ||
| 284 | } | ||
| 285 | |||
| 286 | ## $saveRef = $obj->savePerlRef() | ||
| 287 | ## + inherited from DTA::CAB::Persistent | ||
| 288 | |||
| 289 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 290 | ## + implicitly calls $obj->clear() | ||
| 291 | sub loadPerlRef { | ||
| 292 | my ($that,$ref) = @_; | ||
| 293 | my $obj = $that->SUPER::loadPerlRef($ref); | ||
| 294 | $obj->clear(); | ||
| 295 | return $obj; | ||
| 296 | } | ||
| 297 | |||
| 298 | ##============================================================================== | ||
| 299 | ## Methods: Analysis | ||
| 300 | ##============================================================================== | ||
| 301 | |||
| 302 | ##------------------------------------------------------------------------ | ||
| 303 | ## Methods: Analysis: Generic | ||
| 304 | |||
| 305 | ## $bool = $anl->canAnalyze() | ||
| 306 | ## + returns true if analyzer can perform its function (e.g. data is loaded & non-empty) | ||
| 307 | sub canAnalyze { | ||
| 308 | return ($_[0]->labOk && $_[0]->fstOk); | ||
| 309 | } | ||
| 310 | |||
| 311 | |||
| 312 | ##============================================================================== | ||
| 313 | ## Methods: Analysis: v1.x | ||
| 314 | ##============================================================================== | ||
| 315 | |||
| 316 | |||
| 317 | ## $doc = $anl->analyzeTypes($doc,\%types,\%opts) | ||
| 318 | ## + perform type-wise analysis of all (text) types in %types (= %{$doc->{types}}) | ||
| 319 | sub analyzeTypes { | ||
| 320 | my ($aut,$doc,$types,$opts) = @_; | ||
| 321 | return if (!$aut->canAnalyze); | ||
| 322 | $types = $doc->types if (!$types); | ||
| 323 | |||
| 324 | ##-- setup common variables | ||
| 325 | my $fst = $aut->{fst}; | ||
| 326 | my $fst_ok = $aut->fstOk(); | ||
| 327 | my $result = $aut->resultFst(); | ||
| 328 | my $lab = $aut->{lab}; | ||
| 329 | my $labc = $aut->{labc}; | ||
| 330 | my $laba = $aut->{laba}; | ||
| 331 | my $labenc = $aut->{labenc}; | ||
| 332 | my @eowlab = (defined($aut->{eow}) && $aut->{eow} ne '' ? ($aut->{labh}{$aut->{eow}}) : qw()); | ||
| 333 | my $allowTextRegex = defined($aut->{allowTextRegex}) ? qr($aut->{allowTextRegex}) : undef; | ||
| 334 | |||
| 335 | ##-- object analysis options | ||
| 336 | my $check_symbols = $aut->{check_symbols}; | ||
| 337 | my $auto_connect = $aut->{auto_connect}; | ||
| 338 | my $tolower = $aut->{tolower}; | ||
| 339 | my $tolowerNI = $aut->{tolowerNI}; | ||
| 340 | my $toupperI = $aut->{toupperI}; | ||
| 341 | my $capsFallback = $aut->{capsFallback}; | ||
| 342 | my $qsFallback = $aut->{qsFallback}; | ||
| 343 | my $bashWS = $aut->{bashWS}; | ||
| 344 | my $attInput = $aut->{attInput}; | ||
| 345 | my $attOutput = $aut->{attOutput}; | ||
| 346 | my $wantAnalysisLo = $aut->{wantAnalysisLo}; | ||
| 347 | #my $wantAnalysisFst = $aut->{wantAnalysisFst}; | ||
| 348 | my $wantAnalysisLemma = $aut->{wantAnalysisLemma}; | ||
| 349 | |||
| 350 | ##-- closures | ||
| 351 | my (@wa); | ||
| 352 | my $aget_code = defined($aut->{analyzeGet}) ? $aut->{analyzeGet} : $DEFAULT_ANALYZE_GET; | ||
| 353 | my $aset_code = defined($aut->{analyzeSet}) ? $aut->{analyzeSet} : $DEFAULT_ANALYZE_SET; | ||
| 354 | my $ax_pre = join('', | ||
| 355 | map {"my $_;\n"} | ||
| 356 | '$lab=$anl->{label}', | ||
| 357 | '$wa=$opts{wa}', | ||
| 358 | ); | ||
| 359 | my $aget = $aut->accessClosure($aget_code, pre=>$ax_pre, wa=>\@wa, cache=>0); | ||
| 360 | my $aset = $aut->accessClosure($aset_code, pre=>$ax_pre, wa=>\@wa, cache=>0); | ||
| 361 | ## | ||
| 362 | my $apre = $aut->{analyzePre} ? $aut->accessClosure($aut->{analyzePre}) : undef; | ||
| 363 | my $apost = $aut->{analyzePost} ? $aut->accessClosure($aut->{analyzePost}) : undef; | ||
| 364 | |||
| 365 | ##-- ye olde loope | ||
| 366 | my (@w,$w,$uword,$ulword,@wlabs,$ua,$lemma); | ||
| 367 | foreach (values(%$types)) { | ||
| 368 | next if (defined($allowTextRegex) && $_->{text} !~ $allowTextRegex); ##-- text-sensitive regex | ||
| 369 | @w=$aget->(); | ||
| 370 | push(@w, map {ucfirst(lc($_))} grep {/^[[:upper:]]{2,}$/} @w) if ($capsFallback); ##-- handle "FRANKFURT" as "Frankfurt" | ||
| 371 | push(@w, map {m/^(.+)[\'\x{2018}\x{2019}]s$/ ? "${1}s" : qw()} @w) if ($qsFallback); ##-- handle "Goethe's" as "Goethes" | ||
| 372 | #$aut->trace("analyzeTypes(): w=(", join(' ', @w), ")") if ($aut->{label} eq 'eqphox'); ##-- DEBUG | ||
| 373 | |||
| 374 | next if (!@w); ##-- no lookup-inputs for token (e.g. if $_->{$label} was already populated, e.g. from exlex) | ||
| 375 | @wa = qw(); | ||
| 376 | foreach $w (@w) { | ||
| 377 | ##-------- BEGIN analyzeWord | ||
| 378 | #$aut->setLookupOptions($wopts) if ($aut->can('setLookupOptions')); ##-- OBSOLETE: use code-string in 'analyzePre' key instead! | ||
| 379 | $apre->($w) if ($apre); | ||
| 380 | |||
| 381 | ##---- analyze | ||
| 382 | ##-- normalize | ||
| 383 | $uword = $w // ''; | ||
| 384 | if ($tolower) { $uword = lc($uword); } | ||
| 385 | elsif ($tolowerNI) { $uword =~ s/^(.)(.*)$/$1\L$2\E/; } | ||
| 386 | if ($toupperI) { $uword = ucfirst($uword); } | ||
| 387 | if (defined($bashWS)) { $uword =~ s/\s+/$bashWS/g; } | ||
| 388 | |||
| 389 | if ($fst_ok) { | ||
| 390 | ##-- fst lookup (if fst available and no previous analysis was found) | ||
| 391 | |||
| 392 | ##-- fst: get labels | ||
| 393 | if ($attInput) { | ||
| 394 | ##-- fst: labels: att-style (requires gfsm v0.0.10-pre11, gfsm-perl v0.0217) | ||
| 395 | $ulword = $uword; | ||
| 396 | utf8::downgrade($ulword); | ||
| 397 | @wlabs = (@{$lab->string_to_labels($ulword, $check_symbols, 1)}, @eowlab); | ||
| 398 | } | ||
| 399 | elsif ($check_symbols) { | ||
| 400 | ##-- fst: labels: by character: verbose | ||
| 401 | @wlabs = (@$labc[unpack('U0U*',$uword)],@eowlab); | ||
| 402 | foreach (grep { !defined($wlabs[$_]) && !exists($aut->{warned_symbols}{substr($uword,$_,1)}) } (0..$#wlabs)) { | ||
| 403 | $aut->warn("ignoring unknown character '", substr($uword,$_,1), "' in word '$w' (normalized to '$uword').\n"); | ||
| 404 | $aut->{warned_symbols}{substr($uword,$_,1)} = undef; | ||
| 405 | } | ||
| 406 | @wlabs = grep {defined($_)} @wlabs; | ||
| 407 | } | ||
| 408 | else { | ||
| 409 | ##-- fst: labels: by character: quiet | ||
| 410 | @wlabs = grep {defined($_)} (@$labc[unpack('U0U*',$uword)],@eowlab); | ||
| 411 | } | ||
| 412 | |||
| 413 | ##-- fst: lookup | ||
| 414 | $aut->{fst}->lookup(\@wlabs, $result); | ||
| 415 | $result->_connect() if ($auto_connect); | ||
| 416 | #$result->_rmepsilon() if ($wopts->{auto_rmeps}); | ||
| 417 | |||
| 418 | ##-- parse analyses | ||
| 419 | push(@wa, | ||
| 420 | map { | ||
| 421 | {( | ||
| 422 | #($wantAnalysisFst ? (fst=>$result->clone) : qw()), | ||
| 423 | ($wantAnalysisLo ? (lo=>$uword) : qw()), | ||
| 424 | 'hi'=> (defined($labenc) | ||
| 425 | ? decode($labenc,$lab->labels_to_string($_->{hi},0,1)) | ||
| 426 | : $lab->labels_to_string($_->{hi},0,1)), | ||
| 427 | 'w' => $_->{w}, | ||
| 428 | )} | ||
| 429 | } @{$result->paths($Gfsm::LSUpper)} | ||
| 430 | ); | ||
| 431 | |||
| 432 | } | ||
| 433 | }##-- /foreach $w (@w) | ||
| 434 | |||
| 435 | ##-- un-escape output | ||
| 436 | if (!$attOutput) { | ||
| 437 | foreach (@wa) { | ||
| 438 | $_->{hi} =~ s/\\(.)/$1/g; | ||
| 439 | $_->{hi} =~ s/\[([^\]]+)\]/$1/g; | ||
| 440 | } | ||
| 441 | } | ||
| 442 | |||
| 443 | ##-- parse lemmata | ||
| 444 | if ($wantAnalysisLemma) { | ||
| 445 | foreach (@wa) { | ||
| 446 | $lemma = $_->{hi}; | ||
| 447 | if (defined($lemma) && $lemma ne '') { | ||
| 448 | $lemma =~ s/\[.*$//; ##-- trim everything after first non-character symbol | ||
| 449 | $lemma =~ s/(?:\/\w+)|(?:[\\\¬\~\|\=\+\#])//g; | ||
| 450 | substr($lemma,1) = lc(substr($lemma,1)); | ||
| 451 | } else { | ||
| 452 | $lemma = $uword; | ||
| 453 | } | ||
| 454 | $lemma =~ s/^\s*//; | ||
| 455 | $lemma =~ s/\s*$//; | ||
| 456 | $lemma =~ s/\s+/_/g; | ||
| 457 | $_->{lemma} = $lemma; | ||
| 458 | } | ||
| 459 | } | ||
| 460 | ##-------- END analyzeWord | ||
| 461 | |||
| 462 | ##-- set analyses | ||
| 463 | $aset->(); | ||
| 464 | $apost->() if ($apost); | ||
| 465 | } | ||
| 466 | |||
| 467 | return $doc; | ||
| 468 | } | ||
| 469 | |||
| 470 | |||
| 471 | 1; ##-- be happy | ||
| 472 | |||
| 473 | __END__ |