| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: generic analyzer API | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer; | ||
| 8 | use DTA::CAB::Persistent; | ||
| 9 | use DTA::CAB::Logger; | ||
| 10 | use DTA::CAB::Datum ':all'; | ||
| 11 | use DTA::CAB::Utils ':minmax', ':files', ':time'; | ||
| 12 | use File::Basename qw(basename dirname); | ||
| 13 | use Scalar::Util qw(weaken); | ||
| 14 | use Exporter; | ||
| 15 | use Carp; | ||
| 16 | use strict; | ||
| 17 | |||
| 18 | ##============================================================================== | ||
| 19 | ## Globals | ||
| 20 | ##============================================================================== | ||
| 21 | |||
| 22 | our @ISA = qw(Exporter DTA::CAB::Persistent); | ||
| 23 | |||
| 24 | our @EXPORT = qw(); | ||
| 25 | our %EXPORT_TAGS = | ||
| 26 | ( | ||
| 27 | 'access' => [qw(_am_xtext _am_xlit _am_lts _am_rw), | ||
| 28 | qw(_am_tt_list _am_tt_fst), | ||
| 29 | qw(_am_id_fst _am_xlit_fst), | ||
| 30 | qw(_am_fst_wcp _am_fst_wcp_list _am_fst_wcp_listref), | ||
| 31 | qw(_am_tt_fst_list _am_tt_fst_eqlist), | ||
| 32 | qw(_am_fst_sort _am_fst_rsort _am_fst_uniq _am_fst_usort), | ||
| 33 | qw(_am_clean), | ||
| 34 | qw(_am_tag _am_word _am_lemma), | ||
| 35 | qw(_am_tagh_fst2moota _am_tagh_list2moota _am_tagh_moota_uniq _am_tagh_list2moota_uniq), | ||
| 36 | qw(_am_dmoot_fst2moota _am_dmoot_list2moota), | ||
| 37 | qw(_am_wordlike_regex), | ||
| 38 | qw(parseFstString), | ||
| 39 | ], | ||
| 40 | ); | ||
| 41 | our @EXPORT_OK = map {@$_} values %EXPORT_TAGS; | ||
| 42 | $EXPORT_TAGS{all} = [@EXPORT_OK]; | ||
| 43 | $EXPORT_TAGS{child} = [@EXPORT_OK]; | ||
| 44 | |||
| 45 | ## %CLOSURE_CACHE = ("$object" => { "$closureCode" => \&closure, ... }, ... } | ||
| 46 | ## + cache for accessClosure() to avoid unnecessary re-compilation | ||
| 47 | our (%CLOSURE_CACHE); | ||
| 48 | |||
| 49 | ##============================================================================== | ||
| 50 | ## Constructors etc. | ||
| 51 | ##============================================================================== | ||
| 52 | |||
| 53 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 54 | ## + object structure: | ||
| 55 | ## ( | ||
| 56 | ## label => $label, ##-- analyzer label (default: from class name) | ||
| 57 | ## aclass => $class, ##-- analysis class (optional; see $anl->analysisClass() method; default=undef) | ||
| 58 | ## typeKeys => \@keys, ##-- analyzer type keys for $anl->typeKeys() | ||
| 59 | ## enabled => $bool, ##-- set to false, non-undef value to disable this analyzer | ||
| 60 | ## initQuiet => $bool, ##-- if true, initInfo() will not print any output | ||
| 61 | ## traceLevel => $level, ##-- log-level for trace messages (default=undef: none) | ||
| 62 | ## ) | ||
| 63 | sub new { | ||
| 64 | my $that = shift; | ||
| 65 | my $anl = bless({ | ||
| 66 | ##-- user args | ||
| 67 | @_ | ||
| 68 | }, ref($that)||$that); | ||
| 69 | $anl->initialize(); | ||
| 70 | $anl->{label} = $anl->defaultLabel() if (!defined($anl->{label})); ##-- get label | ||
| 71 | return $anl; | ||
| 72 | } | ||
| 73 | |||
| 74 | ## undef = $anl->DESTROY() | ||
| 75 | ## + default destructor deletes any cached closures for $anl | ||
| 76 | sub DESTROY { | ||
| 77 | #print STDERR __PACKAGE__, "::DESTROY( $_[0] ): ", scalar(keys(%{$CLOSURE_CACHE{$_[0]}//{}})), " closure(s)\n"; | ||
| 78 | delete $CLOSURE_CACHE{$_[0]}; | ||
| 79 | } | ||
| 80 | |||
| 81 | ## undef = $anl->initialize(); | ||
| 82 | ## + default implementation does nothing | ||
| 83 | sub initialize { return; } | ||
| 84 | |||
| 85 | ## undef = $anl->dropClosures(); | ||
| 86 | ## + deletes any cached closures for $anl | ||
| 87 | ## + v1.112 : uses %CLOSURE_CACHE instead of "_analyze*" keys | ||
| 88 | sub dropClosures { | ||
| 89 | delete $CLOSURE_CACHE{$_[0]}; | ||
| 90 | } | ||
| 91 | |||
| 92 | ## $label = $anl->defaultLabel() | ||
| 93 | ## + default label for this class | ||
| 94 | ## + default is final component of perl class-name | ||
| 95 | sub defaultLabel { | ||
| 96 | my $anl = shift; | ||
| 97 | my $lab = ref($anl); | ||
| 98 | $lab =~ s/^.*\:\://; | ||
| 99 | return $lab; | ||
| 100 | } | ||
| 101 | |||
| 102 | ## $class = $anl->analysisClass() | ||
| 103 | ## + gets cached $anl->{aclass} if exists, otherwise returns undef | ||
| 104 | ## + really just an ugly wrapper for $anl->{aclass} | ||
| 105 | sub analysisClass { | ||
| 106 | return $_[0]{aclass}; | ||
| 107 | } | ||
| 108 | |||
| 109 | ## @keys = $anl->typeKeys(\%opts) | ||
| 110 | ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes() | ||
| 111 | ## + default returns @{$anl->{typeKeys}} if defined, otherwise ($anl->{label}) | ||
| 112 | sub typeKeys { | ||
| 113 | return $_[0]{typeKeys} ? @{$_[0]{typeKeys}} : (defined($_[0]{label}) ? ($_[0]{label}) : qw()); | ||
| 114 | } | ||
| 115 | |||
| 116 | ##============================================================================== | ||
| 117 | ## Methods: version | ||
| 118 | ##============================================================================== | ||
| 119 | |||
| 120 | ## $timestamp_str_or_undef = $anl->timestamp(%opts) | ||
| 121 | ## + gets local ($opts{deep}=0) or recursive timestamp ($opts{deep}=1) | ||
| 122 | sub timestamp { | ||
| 123 | my ($anl,%opts) = @_; | ||
| 124 | return $opts{deep} ? $anl->versionInfo(%opts)->{timestamp} : $anl->timestampLocal; | ||
| 125 | } | ||
| 126 | |||
| 127 | ## $timestamp_str_or_undef = $anl->timestampLocal() | ||
| 128 | ## + gets local analyzer timestamp | ||
| 129 | ## + default implementation returns $anl->{timestampLocal} or newest mtime among all $anl->timestampFiles() | ||
| 130 | sub timestampLocal { | ||
| 131 | my $anl = shift; | ||
| 132 | return $anl->{timestampLocal} if (defined($anl->{timestampLocal})); | ||
| 133 | my @tsfiles = grep {defined($_) && -e $_} $anl->timestampFiles; | ||
| 134 | my $mtime = @tsfiles ? (sort {$b<=>$a} map {(stat($_))[9]} @tsfiles)[0] : undef; | ||
| 135 | return $anl->{timestampLocal} = defined($mtime) ? timestamp_str($mtime) : undef; | ||
| 136 | } | ||
| 137 | |||
| 138 | ## @files = $anl->timestampFiles() | ||
| 139 | ## + resource files for determining this analyzer's local timestamp | ||
| 140 | ## + default checks for analyzer keys matching m/file$/i | ||
| 141 | sub timestampFiles { | ||
| 142 | my $anl = shift; | ||
| 143 | return @$anl{grep {m/file$/i} keys %$anl}; | ||
| 144 | } | ||
| 145 | |||
| 146 | ## $version_or_undef = $anl->version() | ||
| 147 | ## + gets local analyzer version string | ||
| 148 | ## + default implementation returns $anl->{version} if defined, otherwise caches from first available file from $anl->versionFiles() | ||
| 149 | sub version { | ||
| 150 | my $anl = shift; | ||
| 151 | return $anl->{version} if (defined($anl->{version})); | ||
| 152 | my $vfile = (grep {defined($_) && -e $_} $anl->versionFiles())[0]; | ||
| 153 | return undef if (!defined($vfile)); | ||
| 154 | open(my $vfh,"<$vfile") | ||
| 155 | or $anl->logconfess("version(): open failed for version file '$vfile': $!"); | ||
| 156 | my ($version); | ||
| 157 | { | ||
| 158 | local $/=undef; | ||
| 159 | $version = <$vfh>; | ||
| 160 | } | ||
| 161 | chomp($version); | ||
| 162 | close($vfh); | ||
| 163 | return $anl->{version} = $version; | ||
| 164 | } | ||
| 165 | |||
| 166 | ## @files = $anl->versionFiles() | ||
| 167 | ## + resource files for determining this analyzer's local version | ||
| 168 | ## + default searches $_.ver, noext($_).ver for all $anl->timestampFiles(), finally dirname($_)/version.txt | ||
| 169 | sub versionFiles { | ||
| 170 | my $anl = shift; | ||
| 171 | my @tsfiles = grep {defined($_)} $anl->timestampFiles(); | ||
| 172 | my ($base); | ||
| 173 | return ( | ||
| 174 | (map {$base=$_; $base =~ s/\.[^\.]*$//; ("$_.ver","$base.ver")} @tsfiles), | ||
| 175 | (map {dirname($_)."/version.txt"} @tsfiles), | ||
| 176 | ); | ||
| 177 | } | ||
| 178 | |||
| 179 | ## \%vinfo_or_undef = $anl->versionInfo(%opts) | ||
| 180 | ## + gets analyzer version info, including sub-analyzers | ||
| 181 | ## + options %opts as for analyzeDocument() | ||
| 182 | ## + returned HASH %vinfo = | ||
| 183 | ## ( | ||
| 184 | ## rcfile => $rcfile, ##-- from $anl->{rcfile} if available | ||
| 185 | ## class => $class, | ||
| 186 | ## label => $label, | ||
| 187 | ## version => $version, | ||
| 188 | ## timestampLocal => $timestampLocal, | ||
| 189 | ## timestamp => $timestampDeep, ##-- youngest local or sub-analyzer timestamp | ||
| 190 | ## subs => \@subAnalyzerVersionInfo, | ||
| 191 | ## ) | ||
| 192 | sub versionInfo { | ||
| 193 | my ($anl,%opts) = @_; | ||
| 194 | return undef if (!$anl->enabled(\%opts)); ##-- no version information for disabled analyzer | ||
| 195 | my $subs = $anl->can('chain') ? $anl->chain(\%opts) : $anl->subAnalyzers(\%opts); | ||
| 196 | my $vinfo = { | ||
| 197 | class => ref($anl), | ||
| 198 | label => $anl->{label}, | ||
| 199 | rcfile => $anl->{rcfile}, | ||
| 200 | version => $anl->version(%opts), | ||
| 201 | timestampLocal => $anl->timestampLocal(%opts), | ||
| 202 | ($subs && @$subs ? (subs=>[grep {defined($_)} map {$_->versionInfo(%opts)} @$subs]) : qw()), | ||
| 203 | }; | ||
| 204 | $vinfo->{timestamp} = (sort {($b||'') cmp ($a||'')} ($vinfo->{timestampLocal}, map {$_->{timestamp}} @{$vinfo->{subs}||[]}))[0]; | ||
| 205 | delete @$vinfo{grep {!defined($vinfo->{$_}) || $vinfo->{$_} eq ''} keys %$vinfo}; | ||
| 206 | return $vinfo; | ||
| 207 | } | ||
| 208 | |||
| 209 | ##============================================================================== | ||
| 210 | ## Methods: I/O | ||
| 211 | ##============================================================================== | ||
| 212 | |||
| 213 | ##-------------------------------------------------------------- | ||
| 214 | ## Methods: I/O: Input: all | ||
| 215 | |||
| 216 | ## $bool = $anl->ensureLoaded() | ||
| 217 | ## $bool = $anl->ensureLoaded(\%opts) | ||
| 218 | ## + ensures analysis data is loaded from default files, or that | ||
| 219 | ## no data is available to be loaded | ||
| 220 | ## + should return false only if user has requested data to be loaded | ||
| 221 | ## and that data cannot be loaded. "empty" analyzers should return | ||
| 222 | ## true here. | ||
| 223 | ## + default version always returns true | ||
| 224 | ## + see canAnalyze(), autoDisable() for alternatives | ||
| 225 | sub ensureLoaded { return 1; } | ||
| 226 | |||
| 227 | ## $bool = $anl->prepare() | ||
| 228 | ## $bool = $anl->prepare(\%opts) | ||
| 229 | ## + wrapper for ensureLoaded(), autoEnable(), initInfo() | ||
| 230 | sub prepare { | ||
| 231 | my $anl = shift; | ||
| 232 | $anl->ensureLoaded(@_) | ||
| 233 | or $anl->logdie("ensureLoaded() failed: $!"); | ||
| 234 | $anl->autoEnable(@_); | ||
| 235 | $anl->initInfo(@_); | ||
| 236 | $anl->canAnalyze(@_) | ||
| 237 | or $anl->logdie("canAnalyze() failed"); | ||
| 238 | return 1; | ||
| 239 | } | ||
| 240 | |||
| 241 | ##============================================================================== | ||
| 242 | ## Methods: Persistence | ||
| 243 | ##============================================================================== | ||
| 244 | |||
| 245 | ##====================================================================== | ||
| 246 | ## Methods: Persistence: Perl | ||
| 247 | |||
| 248 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 249 | ## + returns list of keys not to be saved | ||
| 250 | ## + default just greps for CODE-refs | ||
| 251 | sub noSaveKeys { | ||
| 252 | return grep {UNIVERSAL::isa($_[0]{$_},'CODE')} keys(%{$_[0]}); | ||
| 253 | } | ||
| 254 | |||
| 255 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 256 | ## + default implementation just clobbers $CLASS_OR_OBJ with $ref and blesses | ||
| 257 | sub loadPerlRef { | ||
| 258 | my $that = shift; | ||
| 259 | my $obj = $that->SUPER::loadPerlRef(@_); | ||
| 260 | $obj->dropClosures(); | ||
| 261 | return $obj; | ||
| 262 | } | ||
| 263 | |||
| 264 | ##====================================================================== | ||
| 265 | ## Methods: Persistence: Bin | ||
| 266 | |||
| 267 | ## @keys = $class_or_obj->noSaveBinKeys() | ||
| 268 | ## + returns list of keys not to be saved for binary mode | ||
| 269 | ## + default just greps for CODE-refs | ||
| 270 | sub noSaveBinKeys { | ||
| 271 | grep {UNIVERSAL::isa($_[0]{$_},'CODE')} keys(%{$_[0]}); | ||
| 272 | } | ||
| 273 | |||
| 274 | ## $loadedObj = $CLASS_OR_OBJ->loadBinRef($ref) | ||
| 275 | ## + drops closures | ||
| 276 | sub loadBinRef { | ||
| 277 | my $that = shift; | ||
| 278 | $that->dropClosures() if (ref($that)); | ||
| 279 | return $that->SUPER::loadBinRef(@_); | ||
| 280 | } | ||
| 281 | |||
| 282 | |||
| 283 | ##============================================================================== | ||
| 284 | ## Methods: Analysis: v1.x | ||
| 285 | |||
| 286 | ##------------------------------------------------------------------------ | ||
| 287 | ## Methods: Analysis: Utils | ||
| 288 | |||
| 289 | ## $bool = $anl->canAnalyze(); | ||
| 290 | ## $bool = $anl->canAnalyze(\%opts); | ||
| 291 | ## + returns true iff analyzer can perform its function (e.g. data is loaded & non-empty) | ||
| 292 | ## + default implementation always returns true | ||
| 293 | sub canAnalyze { return 1; } | ||
| 294 | |||
| 295 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 296 | ## + alias for $anl->can("analyze${name}") && (!exists($opts{"doAnalyze${name}"}) || $opts{"doAnalyze${name}"}) | ||
| 297 | sub doAnalyze { | ||
| 298 | my ($anl,$opts,$name) = @_; | ||
| 299 | return $anl->can("analyze${name}") && (!$opts || !exists($opts->{"doAnalyze${name}"}) || $opts->{"doAnalyze${name}"}); | ||
| 300 | } | ||
| 301 | |||
| 302 | ## $bool = $anl->enabled(\%opts) | ||
| 303 | ## + returns true if analyzer SHOULD operate, acording to %opts | ||
| 304 | ## + default returns: | ||
| 305 | ## (!defined($anl->{enabled}) || $anl->{enabled}) ##-- globally enabled | ||
| 306 | ## && | ||
| 307 | ## (!$opts || !defined($opts{"${lab}_enabled"} || $opts{"${lab}_enabled"}) ##-- ... and locally enabled | ||
| 308 | sub enabled { | ||
| 309 | return ( | ||
| 310 | (!defined($_[0]{enabled}) || $_[0]{enabled}) | ||
| 311 | && | ||
| 312 | (!$_[1] || !defined($_[1]{"$_[0]{label}_enabled"}) || $_[1]{"$_[0]{label}_enabled"}) | ||
| 313 | ); | ||
| 314 | } | ||
| 315 | |||
| 316 | ## $bool = $anl->autoEnable() | ||
| 317 | ## $bool = $anl->autoEnable(\%opts) | ||
| 318 | ## + sets $anl->{enabled} flag if not already defined | ||
| 319 | ## + calls $anl->canAnalyze(\%opts) | ||
| 320 | ## + returns new value of $anl->{enabled} | ||
| 321 | ## + implicitly calls autoEnable() on all sub-analyzers | ||
| 322 | sub autoEnable { | ||
| 323 | my $anl = shift; | ||
| 324 | foreach (@{$anl->subAnalyzers(@_)}) { | ||
| 325 | $_->autoEnable(@_); | ||
| 326 | } | ||
| 327 | return $anl->{enabled} if (defined($anl->{enabled})); | ||
| 328 | return $anl->{enabled} = $anl->canAnalyze(@_) ? 1 : 0; | ||
| 329 | } | ||
| 330 | sub autoDisable { return $_[0]->autoEnable(@_[1..$#_]); } | ||
| 331 | |||
| 332 | ## undef = $anl->initInfo() | ||
| 333 | ## + logs initialization info | ||
| 334 | ## + default method reports values of {label}, enabled() | ||
| 335 | ## + sets $anl->{initQuiet}=1 (don't report multiple times) | ||
| 336 | sub initInfo { | ||
| 337 | my $anl = shift; | ||
| 338 | $anl->info("initInfo($anl->{label}): enabled=", ($anl->enabled(@_) ? 1 : 0)) if (!$anl->{initQuiet}); | ||
| 339 | $anl->{initQuiet}=1; | ||
| 340 | } | ||
| 341 | |||
| 342 | ## \@analyzers = $anl->subAnalyzers() | ||
| 343 | ## \@analyzers = $anl->subAnalyzers(\%opts) | ||
| 344 | ## + returns a list of all sub-analyzers | ||
| 345 | ## + default returns all DTA::CAB::Analyzer subclass instances in values(%$anl) | ||
| 346 | sub subAnalyzers { | ||
| 347 | my $anl = shift; | ||
| 348 | return [] if (!ref($anl)); | ||
| 349 | return [grep {ref($_) && UNIVERSAL::isa($_,'DTA::CAB::Analyzer')} values(%$anl)]; | ||
| 350 | } | ||
| 351 | |||
| 352 | |||
| 353 | ##------------------------------------------------------------------------ | ||
| 354 | ## Methods: Analysis: API | ||
| 355 | |||
| 356 | ## $doc = $anl->analyzeDocument($doc,\%opts) | ||
| 357 | ## + analyze a DTA::CAB::Document $doc | ||
| 358 | ## + top-level API routine | ||
| 359 | ## + default implementation just calls: | ||
| 360 | ## #$anl->ensureLoaded(); | ||
| 361 | ## $doc = toDocument($doc); | ||
| 362 | ## if ($anl->doAnalyze('Types')) { | ||
| 363 | ## $types = $anl->getTypes($doc); | ||
| 364 | ## $anl->analyzeTypes($doc,$types,\%opts); | ||
| 365 | ## $anl->expandTypes($doc,$types,\%opts); | ||
| 366 | ## $anl->clearTypes($doc); | ||
| 367 | ## } | ||
| 368 | ## $anl->analyzeTokens($doc,\%opts) if ($anl->doAnalyze(\%opts,'Tokens')); | ||
| 369 | ## $anl->analyzeSentences($doc,\%opts) if ($anl->doAnalyze(\%opts,'Sentences')); | ||
| 370 | ## $anl->analyzeLocal($doc,\%opts) if ($anl->doAnalyze(\%opts,'Local')); | ||
| 371 | ## $anl->analyzeClean($doc,\%opts) if ($anl->doAnalyze(\%opts,'Clean')); | ||
| 372 | sub analyzeDocument { | ||
| 373 | my ($anl,$doc,$opts) = @_; | ||
| 374 | return $doc if (!$anl->enabled($opts)); ##-- disabled analyzer | ||
| 375 | #return undef if (!$anl->ensureLoaded()); ##-- uh-oh... | ||
| 376 | return $doc if (!$anl->canAnalyze); ##-- ok... (?) | ||
| 377 | $doc = toDocument($doc); | ||
| 378 | my ($types); | ||
| 379 | if ($anl->doAnalyze($opts,'Types')) { | ||
| 380 | $types = $anl->getTypes($doc); | ||
| 381 | $anl->analyzeTypes($doc,$types,$opts); | ||
| 382 | $anl->expandTypes($doc,$types,$opts); | ||
| 383 | $anl->clearTypes($doc); | ||
| 384 | } | ||
| 385 | $anl->analyzeTokens($doc,$opts) if ($anl->doAnalyze($opts,'Tokens')); | ||
| 386 | $anl->analyzeSentences($doc,$opts) if ($anl->doAnalyze($opts,'Sentences')); | ||
| 387 | $anl->analyzeLocal($doc,$opts) if ($anl->doAnalyze($opts,'Local')); | ||
| 388 | $anl->analyzeClean($doc,$opts) if ($anl->doAnalyze($opts,'Clean')); | ||
| 389 | return $doc; | ||
| 390 | } | ||
| 391 | |||
| 392 | ## $doc = $anl->analyzeTypes($doc,\%types,\%opts) | ||
| 393 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 394 | ## + default implementation does nothing | ||
| 395 | sub analyzeTypes { return $_[1]; } | ||
| 396 | |||
| 397 | ## $doc = $anl->analyzeTokens($doc,\%opts) | ||
| 398 | ## + perform token-wise analysis of all tokens $doc->{body}[$si]{tokens}[$wi] | ||
| 399 | ## + no default implementation | ||
| 400 | sub analyzeTokens { return $_[1]; } | ||
| 401 | |||
| 402 | ## $doc = $anl->analyzeSentences($doc,\%opts) | ||
| 403 | ## + perform sentence-wise analysis of all sentences $doc->{body}[$si] | ||
| 404 | ## + no default implementation | ||
| 405 | sub analyzeSentences { return $_[1]; } | ||
| 406 | |||
| 407 | ## $doc = $anl->analyzeLocal($doc,\%opts) | ||
| 408 | ## + perform analyzer-local document-level analysis of $doc | ||
| 409 | ## + no default implementation | ||
| 410 | sub analyzeLocal { return $_[1]; } | ||
| 411 | |||
| 412 | ## $doc = $anl->analyzeClean($doc,\%opts) | ||
| 413 | ## + cleanup any temporary data associated with $doc | ||
| 414 | ## + no default implementation | ||
| 415 | sub analyzeClean { return $_[1]; } | ||
| 416 | |||
| 417 | ## $doc = $anl->analyzeClean_rm_undef($doc,\%opts) | ||
| 418 | ## + cleanup any temporary data associated with $doc | ||
| 419 | ## + removes keys with undef values from all tokens | ||
| 420 | sub analyzeClean_rm_undef { | ||
| 421 | my ($anl,$doc,$opts) = @_; | ||
| 422 | my ($tok); | ||
| 423 | foreach (@{$doc->{body}}) { | ||
| 424 | foreach $tok (@{$_->{tokens}}) { | ||
| 425 | delete @$tok{grep {!defined($tok->{$_})} keys %$tok}; | ||
| 426 | } | ||
| 427 | } | ||
| 428 | return $doc; | ||
| 429 | } | ||
| 430 | |||
| 431 | ##------------------------------------------------------------------------ | ||
| 432 | ## Methods: Analysis: API: Type-wise | ||
| 433 | |||
| 434 | ## \%types = $anl->getTypes($doc) | ||
| 435 | ## + returns a hash \%types = ($typeText => $typeToken, ...) mapping token text to | ||
| 436 | ## basic token objects (with only 'text' key defined) | ||
| 437 | ## + default just calls $doc->types() | ||
| 438 | sub getTypes { | ||
| 439 | return $_[1]->types; | ||
| 440 | } | ||
| 441 | |||
| 442 | ## $doc = $anl->expandTypes($doc,\%types,\%opts) | ||
| 443 | ## + expands \%types into $doc->{body} tokens | ||
| 444 | ## + default just calls $doc->expandTypeKeys(\@typeKeys,\%types,\%opts) | ||
| 445 | sub expandTypes { | ||
| 446 | my ($anl,$doc,$types,$opts) = @_; | ||
| 447 | my %typeKeys = map {($_=>undef)} $anl->typeKeys($opts); | ||
| 448 | return $doc->expandTypeKeys([keys %typeKeys],$types,$opts); | ||
| 449 | } | ||
| 450 | |||
| 451 | ## $doc = $anl->clearTypes($doc) | ||
| 452 | ## + clears cached type->object map in $doc->{types} | ||
| 453 | ## + default just calls $doc->clearTypes() | ||
| 454 | sub clearTypes { | ||
| 455 | return $_[1]->clearTypes(); | ||
| 456 | } | ||
| 457 | |||
| 458 | |||
| 459 | ##------------------------------------------------------------------------ | ||
| 460 | ## Methods: Analysis: Wrappers | ||
| 461 | |||
| 462 | ## $tok = $anl->analyzeToken($tok_or_string,\%opts) | ||
| 463 | ## + perform type- and token- analyses on $tok_or_string | ||
| 464 | ## + wrapper for $anl->analyzeDocument() | ||
| 465 | sub analyzeToken { | ||
| 466 | my ($anl,$tok,$opts) = @_; | ||
| 467 | my $doc = toDocument([toSentence([toToken($tok)])]); | ||
| 468 | $anl->analyzeDocument($doc, { ($opts ? %$opts : qw()) , doAnalyzeSentences=>0,doAnalyzeLocal=>0}); | ||
| 469 | return $doc->{body}[0]{tokens}[0]; | ||
| 470 | } | ||
| 471 | |||
| 472 | ## $tok = $anl->analyzeSentence($sent_or_array,\%opts) | ||
| 473 | ## + perform type- and token-, and sentence- analyses on $sent_or_array | ||
| 474 | ## + wrapper for $anl->analyzeDocument() | ||
| 475 | sub analyzeSentence { | ||
| 476 | my ($anl,$sent,$opts) = @_; | ||
| 477 | $sent = [$sent] if (!UNIVERSAL::isa($sent,'ARRAY')); | ||
| 478 | @$sent = map {toToken($_)} @$sent; | ||
| 479 | my $doc = toDocument([toSentence($sent)]); | ||
| 480 | $anl->analyzeDocument($doc, { ($opts ? %$opts : qw()), doAnalyzeLocal=>0}); | ||
| 481 | return $doc->{body}[0]; | ||
| 482 | } | ||
| 483 | |||
| 484 | ## $rpc_xml_base64 = $anl->analyzeData($data_str,\%opts) | ||
| 485 | ## + analyze a raw (formatted) data string $data_str with internal parsing & formatting | ||
| 486 | ## + wrapper for $anl->analyzeDocument() | ||
| 487 | sub analyzeData { | ||
| 488 | require RPC::XML; | ||
| 489 | my ($anl,$doc0,$opts) = @_; | ||
| 490 | |||
| 491 | ##-- parsing & formatting options | ||
| 492 | my $reader = $opts && $opts->{reader} ? $opts->{reader} : {}; ##-- reader options | ||
| 493 | my $writer = $opts && $opts->{writer} ? $opts->{writer} : {}; ##-- writer options | ||
| 494 | |||
| 495 | ##-- get format reader,writer | ||
| 496 | my $ifmt = DTA::CAB::Format->newReader(%$reader); | ||
| 497 | my $ofmt = DTA::CAB::Format->newWriter(class=>ref($ifmt), %$writer); | ||
| 498 | |||
| 499 | ##-- parse, analyze, format | ||
| 500 | my $doc = $ifmt->parseString($doc0); | ||
| 501 | #$doc = DTA::CAB::Utils::deep_decode('UTF-8', $doc); ##-- this should NOT be necessary! | ||
| 502 | $doc = $anl->analyzeDocument($doc,$opts); | ||
| 503 | my $str = $ofmt->flush->putDocument($doc)->toString; | ||
| 504 | $ofmt->flush; | ||
| 505 | |||
| 506 | return RPC::XML::base64->new($str); | ||
| 507 | } | ||
| 508 | |||
| 509 | ##------------------------------------------------------------------------ | ||
| 510 | ## Methods: Analysis: Closure Utilities (optional) | ||
| 511 | |||
| 512 | ## \&closure = $anl->analyzeClosure($which) | ||
| 513 | ## + returns cached $anl->{"_analyze${which}"} if present | ||
| 514 | ## + otherwise calls $anl->getAnalyzeClosure($which) & caches result | ||
| 515 | ## + optional utility for closure-based analysis | ||
| 516 | sub analyzeClosure { | ||
| 517 | my ($anl,$which) = @_; | ||
| 518 | return $anl->{"_analyze${which}"} if (defined($anl->{"_analyze${which}"})); | ||
| 519 | return $anl->{"_analyze${which}"} = $anl->getAnalyzeClosure($which); | ||
| 520 | } | ||
| 521 | |||
| 522 | ## \&closure = $anl->getAnalyzeClosure($which) | ||
| 523 | ## + returns closure \&closure for analyzing data of type "$which" | ||
| 524 | ## (e.g. Word, Type, Token, Sentence, Document, ...) | ||
| 525 | ## + default implementation calls $anl->getAnalyze"${which}"Closure() if | ||
| 526 | ## available, otherwise croak()s | ||
| 527 | sub getAnalyzeClosure { | ||
| 528 | my ($anl,$which) = @_; | ||
| 529 | my $getsub = $anl->can("getAnalyze${which}Closure"); | ||
| 530 | weaken($anl); | ||
| 531 | return $getsub->($anl) if ($getsub); | ||
| 532 | $anl->logconfess("getAnalyzeClosure('$which'): no getAnalyze${which}Closure() method defined!"); | ||
| 533 | } | ||
| 534 | |||
| 535 | ##------------------------------------------------------------------------ | ||
| 536 | ## Methods: Analysis: (Token-)Accessor Closures | ||
| 537 | |||
| 538 | ## $closure = $anl->accessClosure( $methodName, %opts); | ||
| 539 | ## $closure = $anl->accessClosure(\&codeRef, %opts); | ||
| 540 | ## $closure = $anl->accessClosure( $codeString, %opts); | ||
| 541 | ## $closure = $anl->accessClosure(\%opts ); | ||
| 542 | ## + returns accessor-closure $closure for $anl | ||
| 543 | ## + passed argument can be one of the following: | ||
| 544 | ## - a CODE ref resolves to itself | ||
| 545 | ## - a method name resolves to $anl->can($methodName) | ||
| 546 | ## - anything else resolves to a string passed to eval() | ||
| 547 | ## + if the string contains no /\bsub\b/, it will be wrapped | ||
| 548 | ## as "sub {$codeString}" | ||
| 549 | ## + $codeString may reference the closure variable $anl | ||
| 550 | ## (and maybe others; see 'pre' and 'vars' options) | ||
| 551 | ## + %opts | ||
| 552 | ## code => $codeRefOrMethodNameOrCodeString, ##-- clobbers first argument | ||
| 553 | ## pre => $code_str, ##-- for $codeString accessors, prefix for eval (e.g. 'my ($lexVar);') | ||
| 554 | ## vars => \@vars, ##-- adds lexical vars 'my ('.join(',',@varNames).');' | ||
| 555 | ## cache => $bool, ##-- enable/disable use of %CLOSURE_CACHE (default=enabled) | ||
| 556 | sub accessClosure { | ||
| 557 | my ($anl,$code,%opts) = @_; | ||
| 558 | if (UNIVERSAL::isa($code,'HASH')) { | ||
| 559 | %opts = (%$code,%opts); | ||
| 560 | $code = undef; | ||
| 561 | } | ||
| 562 | $code = $opts{code} if (defined($opts{code})); | ||
| 563 | $code = ';' if (!defined($code)); | ||
| 564 | return $code if (UNIVERSAL::isa($code,'CODE')); | ||
| 565 | return $anl->can($code) if ($anl->can($code)); | ||
| 566 | $code = ('' | ||
| 567 | .($opts{pre} ? "$opts{pre}; " : '') | ||
| 568 | .($opts{vars} ? ('my ('.join(',',@{$opts{vars}}).'); ') : '') | ||
| 569 | .($code =~ /\bsub\b/ ? $code : "sub { $code }") | ||
| 570 | ); | ||
| 571 | |||
| 572 | print STDERR | ||
| 573 | ((ref($anl)||$anl), "->accessClosure():\n$code\n") if (0 || (ref($anl) && $anl->{debugAccessClosure})); | ||
| 574 | |||
| 575 | my $do_cache = !exists($opts{cache}) || $opts{cache}; | ||
| 576 | my $sub = ($do_cache ? $CLOSURE_CACHE{$anl}{$code} : undef); | ||
| 577 | my $cached = $sub ? 1 : 0; | ||
| 578 | |||
| 579 | weaken($anl); | ||
| 580 | $sub ||= eval $code; | ||
| 581 | $anl->logcluck("accessClosure(): could not compile closure {$code}: $@") if (!$sub); | ||
| 582 | $CLOSURE_CACHE{$anl}{$code} = $sub if ($do_cache && !$cached); | ||
| 583 | |||
| 584 | return $sub; | ||
| 585 | } | ||
| 586 | |||
| 587 | ## PACKAGE::_am_xlit($tokvar='$_') | ||
| 588 | ## + access-closure macro (EXPR): get text (xlit.latin1Text << text) for token $$tokvar | ||
| 589 | ## + evaluates to a string: | ||
| 590 | ## ($$tokvar->{xlit} ? $$tokvar->{xlit}{latin1Text} : $$tokvar->{text}) | ||
| 591 | sub _am_xlit { | ||
| 592 | my $tokvar = shift || '$_'; | ||
| 593 | return "($tokvar\->{xlit} ? $tokvar\->{xlit}{latin1Text} : $tokvar\->{text}) ##== _am_xlit\n"; | ||
| 594 | } | ||
| 595 | |||
| 596 | ## PACKAGE::_am_xtext($tokvar='$_') | ||
| 597 | ## + access-closure macro (EXPR): get (exlex << xlit.latin1Text << text) for token $$tokvar | ||
| 598 | ## + evaluates to a string: | ||
| 599 | ## (defined($$tokvar->{exlex}) ? $$tokvar->{exlex} : $${am_xlit($tokvar)}) | ||
| 600 | sub _am_xtext { | ||
| 601 | my $tokvar = shift || '$_'; | ||
| 602 | return ("(defined($tokvar\->{exlex})" | ||
| 603 | ." ? $tokvar\->{exlex}" | ||
| 604 | ." : ($tokvar\->{xlit} ? $tokvar\->{xlit}{latin1Text} : $tokvar\->{text})" | ||
| 605 | .") ##== _am_xtext\n" | ||
| 606 | ); | ||
| 607 | } | ||
| 608 | |||
| 609 | ## PACKAGE::_am_lts($tokvar='$_') | ||
| 610 | ## + access-closure macro (EXPR) for first LTS analysis of token $$tokvar | ||
| 611 | ## + evaluates to string: | ||
| 612 | ## ($$tokvar->{lts} && @{$$tokvar->{lts}} ? $$tokvar->{lts}[0]{hi} : $$tokvar->{text}) | ||
| 613 | sub _am_lts { | ||
| 614 | my $tokvar = shift || '$_'; | ||
| 615 | return "($tokvar\->{lts} && \@{$tokvar\->{lts}} ? $tokvar\->{lts}[0]{hi} : $tokvar\->{text}) ##== _am_lts\n"; | ||
| 616 | } | ||
| 617 | |||
| 618 | ## PACKAGE::_am_rw($tokvar='$_') | ||
| 619 | ## + access-closure macro (EXPR) for rw output(s) for token $$tokvar | ||
| 620 | ## + evaluates to string: | ||
| 621 | ## ($$tokvar->{rw} ? (map {$_->{hi}} @{$$tokvar->{rw}}) : qw()) | ||
| 622 | sub _am_rw { | ||
| 623 | my $tokvar = shift || '$_'; | ||
| 624 | return "($tokvar\->{rw} ? (map {\$_->{hi}} \@{$tokvar\->{rw}}) : qw()) ##== _am_rw\n"; | ||
| 625 | } | ||
| 626 | |||
| 627 | ## PACKAGE::_am_tt_list($ttvar='$_') | ||
| 628 | ## + access-closure macro (EXPR) for a TT-style list of strings $$ttvar | ||
| 629 | ## + evaluatees to a list: "split(/\\t/,$$ttvar)" | ||
| 630 | sub _am_tt_list { | ||
| 631 | my $ttvar = shift || '$_'; | ||
| 632 | return "split(/\\t/,$ttvar) ##== _am_tt_list\n"; | ||
| 633 | } | ||
| 634 | |||
| 635 | ## PACKAGE::_am_tt_fst($ttvar='$_') | ||
| 636 | ## + access-closure macro (EXPR) for a single TT-style FST analysis $$ttvar | ||
| 637 | ## + formerly mutliply defined in sub-packages as SUBPACKAGE::parseFstString() | ||
| 638 | ## + evaluates to a FST-analysis hash {hi=>$hi,w=>$w,lo=>$lo,lemma=>$lemma}: | ||
| 639 | ## ( | ||
| 640 | ## $$ttvar =~ /^(?:(.*?) \: )?(?:(.*?) \@ )?(.*?)(?: \<([\d\.\+\-eE]+)\>)?$/ | ||
| 641 | ## ? {(defined($1) ? (lo=>$1) : qw()), (defined($2) ? (lemma=>$2) : qw()), hi=>$3, w=>($4||0)} | ||
| 642 | ## : {hi=>$$ttvar} | ||
| 643 | ## ) | ||
| 644 | sub _am_tt_fst { | ||
| 645 | my $ttvar = shift || '$_'; | ||
| 646 | return ("($ttvar".' =~ /^(?:(.*?) \: )?(?:(.*?) \@ )?(.*?)(?: \<([\d\.\+\-eE]+)\>)?$/' | ||
| 647 | .' ? {(defined($1) ? (lo=>$1) : qw()), (defined($2) ? (lemma=>$2) : qw()), hi=>$3, w=>($4||0)}' | ||
| 648 | ." : {hi=>$ttvar})" | ||
| 649 | ." ##== _am_tt_fst\n"); | ||
| 650 | } | ||
| 651 | |||
| 652 | ## PACKAGE::_parseFstString($ttstr) | ||
| 653 | ## + actual subroutine for parsing an fst string | ||
| 654 | sub parseFstString { | ||
| 655 | return ($_[0] =~ /^(?:(.*?) \: )?(?:(.*?) \@ )?(.*?)(?: \<([\d\.\+\-eE]+)\>)?$/ | ||
| 656 | ? {(defined($1) ? (lo=>$1) : qw()), (defined($2) ? (lemma=>$2) : qw()), hi=>$3, w=>($4||0)} | ||
| 657 | : {hi=>$_[0]}); | ||
| 658 | } | ||
| 659 | |||
| 660 | ## PACKAGE::_am_id_fst($tokvar='$_', $wvar='0') | ||
| 661 | ## + access-closure macro (EXPR) for a identity FST analysis | ||
| 662 | ## + really just a wrapper for _am_xlit_fst() | ||
| 663 | sub _am_id_fst { | ||
| 664 | return _am_xlit_fst(@_); | ||
| 665 | } | ||
| 666 | |||
| 667 | ## PACKAGE::_am_xlit_fst($tokvar='$_', $wvar='0') | ||
| 668 | ## + access-closure macro (EXPR) for a xlit-FST analysis | ||
| 669 | ## + evaluates to a single fst analysis hash: | ||
| 670 | ## {hi=>_am_xlit($tokvar), w=>$$wvar} | ||
| 671 | sub _am_xlit_fst { | ||
| 672 | my $tokvar = shift || '$_'; | ||
| 673 | my $wvar = shift || '0'; | ||
| 674 | return '{hi=>'._am_xlit($tokvar).', w=>'.$wvar.'}'." ##== _am_id_fst\n"; | ||
| 675 | } | ||
| 676 | |||
| 677 | ## PACKAGE::_am_fst_wcp($fstvar='$_', $wexpr=$fstvar.'->{w}') | ||
| 678 | ## + access-closure macro (EXPR) for a re-weighted copy of FST analysis $$fstvar | ||
| 679 | ## + evaluates to a copy of $$fstvar with $$wexpr replacing $$fstvar->{w}: | ||
| 680 | ## {%$$fstvar,w=>$$wexpr} | ||
| 681 | sub _am_fst_wcp { | ||
| 682 | my $fstvar = shift || '$_'; | ||
| 683 | my $wexpr = shift || $fstvar.'->{w}'; | ||
| 684 | return "{ %{$fstvar}, w=>$wexpr } ##-- _am_fst_wcp\n"; | ||
| 685 | } | ||
| 686 | |||
| 687 | ## PACKAGE::_am_fst_wcp_list($listvar='@_', $wexpr='$_->{w}') | ||
| 688 | ## + access-closure macro (EXPR) for a list of weighted copies of FST analysis-list $$listvar | ||
| 689 | ## + evaluates to a list of copies of $$listvar analyses with $$wexpr replacing $_->{w}: | ||
| 690 | ## (map { $${_am_fst_wcp('$_',$wexpr)} } $$listvar) | ||
| 691 | sub _am_fst_wcp_list { | ||
| 692 | my $listvar = shift || '@_'; | ||
| 693 | my $wexpr = shift || '$_->{w}'; | ||
| 694 | return "(map {"._am_fst_wcp('$_',$wexpr)."} $listvar) ##-- _am_fst_wcp_list\n"; | ||
| 695 | } | ||
| 696 | |||
| 697 | ## PACKAGE::_am_fst_wcp_listref($listrefvar='$_->{rw}', $wexpr='$_->{w}') | ||
| 698 | ## + access-closure macro (EXPR) for a list of weighted copies of FST analysis-listref $$listrefvar | ||
| 699 | ## + accepts undef $$listrefvar | ||
| 700 | ## + evaluates to a list of copies of $$listvar analyses with $$wexpr replacing $_->{w}: | ||
| 701 | ## ($$listvar ? (map { $${_am_fst_wcp('$_',$wexpr)} } @{$$listvar}) : qw()) | ||
| 702 | sub _am_fst_wcp_listref { | ||
| 703 | my $listrefvar = shift || '$_->{rw}'; | ||
| 704 | my $wexpr = shift; | ||
| 705 | return "($listrefvar ? "._am_fst_wcp_list("\@{$listrefvar}",$wexpr)." : qw()) ##-- _am_fst_wcp_listref\n"; | ||
| 706 | } | ||
| 707 | |||
| 708 | ## PACKAGE::_am_tt_fst_list($ttvar='$_') | ||
| 709 | ## + access-closure macro (EXPR) for a list of TT-style FST analyses $$ttvar | ||
| 710 | ## + evaluates to a list of fst analysis hashes: | ||
| 711 | ## (map {_am_tt_fst('$_')} split(/\t/,$$ttvar)) | ||
| 712 | sub _am_tt_fst_list { | ||
| 713 | my $ttvar = shift || '$_'; | ||
| 714 | return '(map {'._am_tt_fst('$_').'} split(/\t/,'.$ttvar.'))'." ##== _am_tt_fst_list\n"; | ||
| 715 | } | ||
| 716 | |||
| 717 | ## PACKAGE::_am_tt_fst_eqlist($ttvar='$tt', $tokvar='$_', $wvar='0') | ||
| 718 | ## + access-closure macro (EXPR) for a list of TT-style FST analyses $$ttvar | ||
| 719 | ## + evaluates to a list of fst analysis hashes: | ||
| 720 | ## (_am_id_fst($tokvar,$wvar), _am_tt_fst_list($ttvar)) | ||
| 721 | sub _am_tt_fst_eqlist { | ||
| 722 | my $ttvar = shift || '$tt'; | ||
| 723 | my $tokvar = shift || '$_'; | ||
| 724 | my $wvar = shift || '0'; | ||
| 725 | return "("._am_id_fst($tokvar,$wvar).', '._am_tt_fst_list($ttvar).')'." ##== _am_tt_fst_eqlist\n"; | ||
| 726 | } | ||
| 727 | |||
| 728 | ## PACKAGE::_am_fst_sort($listvar='@_') | ||
| 729 | ## + access-closure macro (EXPR) to sort a list of FST analyses $$listvar by weight | ||
| 730 | ## + evaluates to a sorted list of fst analysis hashes: | ||
| 731 | ## (sort {($a->{w}||0) <=> ($b->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} $$listvar) | ||
| 732 | sub _am_fst_sort { | ||
| 733 | my $listvar = shift || '@_'; | ||
| 734 | return '(sort {($a->{w}||0) <=> ($b->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} '.$listvar.')'." ##== _am_fst_sort\n"; | ||
| 735 | } | ||
| 736 | |||
| 737 | ## PACKAGE::_am_fst_rsort($listvar='@_') | ||
| 738 | ## + access-closure macro (EXPR) to reverse-sort a list of FST analyses $$listvar by weight | ||
| 739 | ## + evaluates to a sorted list of fst analysis hashes: | ||
| 740 | ## (sort {($b->{w}||0) <=> ($a->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} $$listvar) | ||
| 741 | sub _am_fst_rsort { | ||
| 742 | my $listvar = shift || '@_'; | ||
| 743 | return '(sort {($b->{w}||0) <=> ($a->{w}||0) || ($a->{hi}||"") cmp ($b->{hi}||"")} '.$listvar.')'." ##== _am_fst_rsort\n"; | ||
| 744 | } | ||
| 745 | |||
| 746 | ## PACKAGE::_am_fst_uniq($listvar='@_', $tmpvar='$val') | ||
| 747 | ## + access-closure macro (EXPR) for a unique list of TT-style FST analyses $$listvar | ||
| 748 | ## + only the weight-minimal analysis is kept | ||
| 749 | ## + evaluates to a list of upper-unique fst analysis hashes: | ||
| 750 | ## (map {$$val && $$val->{hi} eq $_->{hi} ? qw() : ($$val=$_)} sort {$a->{hi} cmp $b->{hi} || ($a->{w}||0) <=> ($b->{w}||0)} $$listvar) | ||
| 751 | ## + assumes $$tmpvar is initialized to undef at start of evaluation | ||
| 752 | sub _am_fst_uniq { | ||
| 753 | my $listvar = shift || '@_'; | ||
| 754 | my $tmpvar = shift || '$val'; | ||
| 755 | return ("(map {$tmpvar && $tmpvar\->{hi} eq \$_->{hi} ? qw() : ($tmpvar=\$_)}" | ||
| 756 | .' sort {($a->{hi}//"") cmp ($b->{hi}//"") || ($a->{w}//0) <=> ($b->{w}//0)}' | ||
| 757 | ." $listvar)" | ||
| 758 | ." ##== _am_fst_uniq\n" | ||
| 759 | ); | ||
| 760 | } | ||
| 761 | |||
| 762 | ## PACKAGE::_am_fst_usort($listvar='@_', $tmpvar='$val') | ||
| 763 | ## + wrapper for _am_fst_sort(_am_fst_uniq($listvar,$tmpvar)) | ||
| 764 | sub _am_fst_usort { | ||
| 765 | return _am_fst_sort(_am_fst_uniq(@_)); | ||
| 766 | } | ||
| 767 | |||
| 768 | ## PACKAGE::_am_clean($hashvar='$_->{$lab}') | ||
| 769 | ## + access-closure macro (STMT) to delete a hash entry if undefined; evaluates to | ||
| 770 | ## delete($$hashvar) if (!defined($$hashvar)); | ||
| 771 | sub _am_clean { | ||
| 772 | my $hashvar = shift || '$_->{$lab}'; | ||
| 773 | return "delete($hashvar) if (!defined($hashvar)); ##== _am_clean\n" | ||
| 774 | } | ||
| 775 | |||
| 776 | ## PACKAGE::_am_tag($mootvar='$_->{moot}', $defaultvar='undef') | ||
| 777 | ## + access-closure macro (EXPR) for a moot or dmoot tag; evaluates to | ||
| 778 | ## ($$mootvar ? $$mootvar->{tag} : $defaultvar) | ||
| 779 | sub _am_tag { | ||
| 780 | my $mootvar = shift||'$_->{moot}'; | ||
| 781 | my $default = shift||'undef'; | ||
| 782 | return "($mootvar ? $mootvar\->{tag} : $default) ##== _am_tag\n"; | ||
| 783 | } | ||
| 784 | |||
| 785 | ## PACKAGE::_am_word($mootvar='$_->{moot}', $defaultvar='undef') | ||
| 786 | ## + access-closure macro (EXPR) for a moot or dmoot tag; evaluates to | ||
| 787 | ## ($$mootvar ? $$mootvar->{word} : $defaultvar) | ||
| 788 | sub _am_word { | ||
| 789 | my $mootvar = shift||'$_->{moot}'; | ||
| 790 | my $default = shift||'undef'; | ||
| 791 | return "($mootvar ? $mootvar\->{word} : $default) ##== _am_word\n"; | ||
| 792 | } | ||
| 793 | |||
| 794 | ## PACKAGE::_am_lemma($mootvar='$_->{moot}', $defaultvar='undef') | ||
| 795 | ## + access-closure macro (EXPR) for a moot lemma; evaluates to | ||
| 796 | ## ($$mootvar ? $$mootvar->{word} : $defaultvar) | ||
| 797 | sub _am_lemma { | ||
| 798 | my $mootvar = shift||'$_->{moot}'; | ||
| 799 | my $default = shift||'undef'; | ||
| 800 | return "($mootvar ? $mootvar\->{lemma} : $default) ##== _am_lemma\n"; | ||
| 801 | } | ||
| 802 | |||
| 803 | ## PACKAGE::_am_tagh_fst2moota($taghvar='$_') | ||
| 804 | ## + access-closure macro (EXPR): single moot token analysis from TAGH-style fst analysis | ||
| 805 | ## + requires: $$taghvar->{hi}; evaluates to: | ||
| 806 | ## {details=>$taghvar->{hi}, prob=>($$taghvar->{w}||0), tag=>($$taghvar->{hi} =~ /\[\_?([A-Z0-9]+)\]/ ? \$1 : $$taghvar->{hi})} | ||
| 807 | sub _am_tagh_fst2moota { | ||
| 808 | my $taghvar = shift||'$_'; | ||
| 809 | return ("{details=>$taghvar\->{hi}," | ||
| 810 | ." prob=>($taghvar\->{w}||0)," | ||
| 811 | ." tag=>($taghvar\->{hi} =~ /\\[\\_?((?:[A-Za-z0-9\.]+|\\\$[^\\]]+))\\]/ ? \$1 : $taghvar\->{hi})" ##-- allow e.g. [$(] tags from tokenizer! | ||
| 812 | ."} ##-- _am_tagh_fst2moota\n"); | ||
| 813 | } | ||
| 814 | |||
| 815 | ## PACKAGE::_am_tagh_list2moota($listvar='@{$_->{morph}}') | ||
| 816 | ## + access-closure macro (EXPR): moot token analysis-list from TAGH-style fst analysis-list | ||
| 817 | ## + evaluates to (something like): | ||
| 818 | ## (map { $${_am_tagh_fst2moota('$_')} } $$listvar) | ||
| 819 | sub _am_tagh_list2moota { | ||
| 820 | my $listvar = shift||'@{$_->{morph}}'; | ||
| 821 | #return "(map {ref(\$_) ? "._am_tagh_fst2moota('$_')." : {details=>\$_,tag=>\$_,prob=>0}} $listvar) ##-- _am_tagh_list2moota\n"; | ||
| 822 | return "(map {"._am_tagh_fst2moota('$_')."} map {ref(\$_) ? \$_ : {hi=>\$_}} $listvar) ##-- _am_tagh_list2moota\n"; | ||
| 823 | } | ||
| 824 | |||
| 825 | ## PACKAGE::_am_tagh_moota_uniq($listvar='@_', $tmpvar='$val') | ||
| 826 | ## + access-closure macro (EXPR) for a unique (by details) list of moot analyses $$listvar | ||
| 827 | ## + only the prob-minimal analysis is kept for each 'details' | ||
| 828 | ## + evaluates to a list of details-unique fst analysis hashes: | ||
| 829 | ## (map {$$val && $$val->{details} eq $_->{details} ? qw() : ($$val=$_)} sort {$a->{details} cmp $b->{details} || ($a->{prob}//0) <=> ($b->{prob}//0)} $$listvar) | ||
| 830 | ## + assumes $$tmpvar is initialized to undef at start of evaluation | ||
| 831 | sub _am_tagh_moota_uniq { | ||
| 832 | my $listvar = shift || '@_'; | ||
| 833 | my $tmpvar = shift || '$val'; | ||
| 834 | return ("(map {$tmpvar && $tmpvar\->{details} eq \$_->{details} ? qw() : ($tmpvar=\$_)}" | ||
| 835 | .' sort {($a->{details}//"") cmp ($b->{details}//"") || ($a->{prob}//0) <=> ($b->{prob}//0)}' | ||
| 836 | ." $listvar)" | ||
| 837 | ." ##== _am_tagh_moota_uniq\n" | ||
| 838 | ); | ||
| 839 | } | ||
| 840 | |||
| 841 | ## PACKAGE::_am_tagh_list2moota_uniq($listvar, $tmpvar) | ||
| 842 | ## + wrapper for _am_tagh_moota_uniq( _am_tagh_list2moota($listvar), $tmpvar ) | ||
| 843 | sub _am_tagh_list2moota_uniq { | ||
| 844 | return _am_tagh_moota_uniq( _am_tagh_list2moota($_[0]), $_[1] ); | ||
| 845 | } | ||
| 846 | |||
| 847 | ## PACKAGE::_am_dmoot_fst2moota($fstvar='$_') | ||
| 848 | ## + access-closure macro (EXPR): single dmoot token analysis from fst analysis | ||
| 849 | ## + requires: $$fstvar->{hi}; evaluates to: | ||
| 850 | ## {tag=>$$fstvar->{hi}, prob=>($$fstvar->{w}||0)} | ||
| 851 | sub _am_dmoot_fst2moota { | ||
| 852 | my $fstvar = shift||'$_'; | ||
| 853 | return "{tag=>$fstvar\->{hi}, prob=>($fstvar\->{w}||0)} ##-- _am_dmoot_fst2moota\n"; | ||
| 854 | } | ||
| 855 | |||
| 856 | ## PACKAGE::_am_dmoot_list2moota($listvar='@_') | ||
| 857 | ## + access-closure macro (EXPR): dmoot token analysis-list from fst analysis-list | ||
| 858 | ## + evaluates to: | ||
| 859 | ## (map { $${_am_dmoot_fst2moota('$_')} } $$listvar) | ||
| 860 | sub _am_dmoot_list2moota { | ||
| 861 | my $listvar = shift||'@_'; | ||
| 862 | return "(map {"._am_dmoot_fst2moota('$_')."} $listvar) ##-- _am_dmoot_list2moota\n"; | ||
| 863 | } | ||
| 864 | |||
| 865 | ## $regex_str = PACKAGE::_am_wordlike_regex() | ||
| 866 | ## + for use e.g. by Analyzer::Automaton subclass {allowTextRegex} property defaults | ||
| 867 | sub _am_wordlike_regex { | ||
| 868 | return '^(?:(?:[[:alpha:]\p{Combining_Diacritical_Marks}\-\@\x{ac}]*[[:alpha:]\p{CombiningDiacriticalMarks}]+)|(?:[[:alpha:]\p{CombiningDiacriticalMarks}]+[[:alpha:]\p{CombiningDiacriticalMarks}\-\@\x{ac}]+))(?:[\'\x{2018}\x{2019}]s)?$'; | ||
| 869 | } | ||
| 870 | |||
| 871 | |||
| 872 | ##============================================================================== | ||
| 873 | ## Methods: XML-RPC | ||
| 874 | ##============================================================================== | ||
| 875 | |||
| 876 | ## \@sigs = $anl->xmlRpcSignatures() | ||
| 877 | ## + returns an array-ref of valid XML-RPC signatures: | ||
| 878 | ## [ "$returnType1 $argType1_1 $argType1_2 ...", ..., "$returnTypeN ..." ] | ||
| 879 | ## + known types (see http://www.xmlrpc.com/spec): | ||
| 880 | ## Tag Type Example | ||
| 881 | ## "i4" or "int" four-byte signed integer 42 | ||
| 882 | ## "boolean" 0 (false) or 1 (true) 1 | ||
| 883 | ## "string" string hello world | ||
| 884 | ## "double" double-precision signed floating point number -24.7 | ||
| 885 | ## "dateTime.iso8601" date/time 19980717T14:08:55 | ||
| 886 | ## "base64" base64-encoded binary eW91IGNhbid0IHJlYWQgdGhpcyE= | ||
| 887 | ## "struct" complex structure { x=>42, y=>24 } | ||
| 888 | ## + Default returns "string string struct" | ||
| 889 | #sub xmlRpcSignature { return ['string string']; } | ||
| 890 | |||
| 891 | ## $str = $anl->xmlRpcHelp() | ||
| 892 | ## + returns help string for default XML-RPC procedure | ||
| 893 | #sub xmlRpcHelp { return '?'; } | ||
| 894 | |||
| 895 | ## \%opts = $anl->mergeOptions(\%defaultOptions,\%userOptions) | ||
| 896 | ## + returns options hash like (%defaultOptions,%userOptions) [user clobbers default] | ||
| 897 | sub mergeOptions { | ||
| 898 | my ($anl,$defaults,$user) = @_; | ||
| 899 | return { ($defaults ? %$defaults : qw()), ($user ? %$user : qw()) }; | ||
| 900 | } | ||
| 901 | |||
| 902 | |||
| 903 | ## @procedures = $anl->xmlRpcMethods() | ||
| 904 | ## @procedures = $anl->xmlRpcMethods($prefix,\%opts) | ||
| 905 | ## + returns a list of procedures suitable for passing to RPC::XML::Server::add_proc() | ||
| 906 | ## + additional keys recognized in procedure specs: see DTA::CAB::Server::XmlRpc::prepareLocal() | ||
| 907 | ## + "${prefix}." is appended to procedure 'name' key if $prefix is specified | ||
| 908 | ## + \%opts are passed to analyze methods if defined | ||
| 909 | sub xmlRpcMethods { | ||
| 910 | my ($anl,$prefix,$opts) = @_; | ||
| 911 | $prefix = $prefix ? "${prefix}." : ''; | ||
| 912 | return ( | ||
| 913 | { | ||
| 914 | ##-- Analyze: Type (v1.x) | ||
| 915 | name => "${prefix}analyzeType", | ||
| 916 | code => sub { $anl->analyzeType($_[0],$anl->mergeOptions($opts,$_[1])) }, | ||
| 917 | signature => [ 'struct string', 'struct string struct', ## string ?opts -> struct | ||
| 918 | 'struct struct', 'struct struct struct', ## struct ?opts -> struct | ||
| 919 | ], | ||
| 920 | help => 'Analyze a single token (text string or struct with "text" string field)', | ||
| 921 | wrapEncoding => 1, | ||
| 922 | }, | ||
| 923 | { | ||
| 924 | ##-- Analyze: Token (v1.x) | ||
| 925 | name => "${prefix}analyzeToken", | ||
| 926 | code => sub { | ||
| 927 | #$anl->trace("analyzeToken(", (ref($_[0]) ? $_[0]{text} : $_[0]), ")"); ##-- DEBUG | ||
| 928 | $anl->analyzeToken($_[0],$anl->mergeOptions($opts,$_[1])); | ||
| 929 | }, | ||
| 930 | signature => [ 'struct string', 'struct string struct', ## string ?opts -> struct | ||
| 931 | 'struct struct', 'struct struct struct', ## struct ?opts -> struct | ||
| 932 | ], | ||
| 933 | help => 'Analyze a single token (text string or struct with "text" string field)', | ||
| 934 | wrapEncoding => 1, | ||
| 935 | }, | ||
| 936 | { | ||
| 937 | ##-- Analyze: Sentence (v1.x) | ||
| 938 | name => "${prefix}analyzeSentence", | ||
| 939 | code => sub { $anl->analyzeSentence($_[0],$anl->mergeOptions($opts,$_[1])) }, | ||
| 940 | signature => [ 'struct array', 'struct array struct', ## array ?opts -> struct | ||
| 941 | 'struct struct', 'struct struct struct', ## struct ?opts -> struct | ||
| 942 | ], | ||
| 943 | help => 'Analyze a single sentence (array of tokens or struct with "tokens" array field)', | ||
| 944 | wrapEncoding => 1, | ||
| 945 | }, | ||
| 946 | { | ||
| 947 | ##-- Analyze: Document (v1.x) | ||
| 948 | name => "${prefix}analyzeDocument", | ||
| 949 | code => sub { $anl->analyzeDocument($_[0],$anl->mergeOptions($opts,$_[1])) }, | ||
| 950 | signature => [ | ||
| 951 | 'struct array', 'struct array struct', ## array ?opts -> struct | ||
| 952 | 'struct struct', 'struct struct struct', ## struct ?opts -> struct | ||
| 953 | ], | ||
| 954 | help => 'Analyze a whole document (array of sentences or struct with "body" array field)', | ||
| 955 | wrapEncoding => 1, | ||
| 956 | }, | ||
| 957 | ##-- Analyze: raw data (v1.x) | ||
| 958 | { | ||
| 959 | name => "${prefix}analyzeData", | ||
| 960 | code => sub { $anl->analyzeData($_[0],$anl->mergeOptions($opts,$_[1])) }, | ||
| 961 | signature => [ | ||
| 962 | #'string string', ## string -> string | ||
| 963 | #'string string struct', ## string ?opts -> string | ||
| 964 | ##-- | ||
| 965 | 'base64 base64', ## base64 -> base64 | ||
| 966 | 'base64 base64 struct', ## base64 ?opts -> base64 | ||
| 967 | ], | ||
| 968 | help => 'Analyze raw document data with server-side parsing & formatting', | ||
| 969 | wrapEncoding => 0, | ||
| 970 | }, | ||
| 971 | ); | ||
| 972 | } | ||
| 973 | |||
| 974 | |||
| 975 | 1; ##-- be happy | ||
| 976 | |||
| 977 | __END__ |