| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::MootSub.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: post-processing for moot PoS tagger in DTA chain | ||
| 6 | ## + tweaks $tok->{moot}{word}, instantiates $tok->{moot}{lemma} | ||
| 7 | |||
| 8 | package DTA::CAB::Analyzer::MootSub; | ||
| 9 | use DTA::CAB::Analyzer ':child'; | ||
| 10 | use DTA::CAB::Analyzer::Lemmatizer; | ||
| 11 | use Text::LevenshteinXS qw(); | ||
| 12 | use Carp; | ||
| 13 | use strict; | ||
| 14 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 15 | |||
| 16 | ##====================================================================== | ||
| 17 | ## Methods | ||
| 18 | |||
| 19 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 20 | ## + object structure, %args: | ||
| 21 | ## mootLabel => $label, ##-- label for Moot tagger object (default='moot') | ||
| 22 | ## lz => $lemmatizer, ##-- DTA::CAB::Analyzer::Lemmatizer sub-object | ||
| 23 | ## bytoken => $bool, ##-- type-wise expand $mootLabel if true; depends on global option "${label}.bytoken"; see typeKeys() method | ||
| 24 | ## xyTags => $xytags, ##-- use literal text (not dmoot) for these tags (string, array, or HASH-ref; default=[qw(XY FM)]) | ||
| 25 | ## ucTags => $uctags, ##-- implicitly upper-case lemmata for these tags (string, array, or HASH-ref; default=[qw(NN NE)]) | ||
| 26 | ## stts => $bool, ##-- implicitly use STTS-specific heuristics? (default=1) | ||
| 27 | ## wMorph => $bool, ##-- morph-cost coefficient for lemma-selection heuristics (default=1000) | ||
| 28 | sub new { | ||
| 29 | my $that = shift; | ||
| 30 | my $asub = $that->SUPER::new( | ||
| 31 | ##-- analysis selection | ||
| 32 | label => 'mootsub', | ||
| 33 | mootLabel => 'moot', | ||
| 34 | bytoken => 1, | ||
| 35 | lz => DTA::CAB::Analyzer::Lemmatizer->new(analyzeGet =>$DTA::CAB::Analyzer::Lemmatizer::GET_MOOT_ANALYSES, | ||
| 36 | analyzeGetText=>$DTA::CAB::Analyzer::Lemmatizer::GET_MOOT_TEXT, | ||
| 37 | analyzeWhich =>'Sentences', | ||
| 38 | segmentLabel =>'segs', | ||
| 39 | ), | ||
| 40 | xyTags => 'XY FM', #CARD NE ##-- use literal text (not dmoot) for these tags | ||
| 41 | ucTags => 'NN NE', ##-- implicitly upper-case lemmata for these tags | ||
| 42 | stts => 1, | ||
| 43 | wMorph => 1000, | ||
| 44 | |||
| 45 | ##-- user args | ||
| 46 | @_ | ||
| 47 | ); | ||
| 48 | |||
| 49 | $asub->{lz}{label} = $asub->{label}."_lz"; | ||
| 50 | return $asub; | ||
| 51 | } | ||
| 52 | |||
| 53 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 54 | ## + override: only allow analyzeSentences() | ||
| 55 | sub doAnalyze { | ||
| 56 | my $anl = shift; | ||
| 57 | return 0 if (defined($_[1]) && $_[1] ne 'Sentences'); | ||
| 58 | return $anl->SUPER::doAnalyze(@_); | ||
| 59 | } | ||
| 60 | |||
| 61 | ## $doc = $anl->Sentences($doc,\%opts) | ||
| 62 | ## + post-processing for 'moot' object | ||
| 63 | sub analyzeSentences { | ||
| 64 | my ($asub,$doc,$opts) = @_; | ||
| 65 | return $doc if (!$asub->enabled($opts)); | ||
| 66 | |||
| 67 | ##-- tweak list-tags | ||
| 68 | foreach my $lkey (grep {defined($asub->{$_})} qw(xyTags ucTags)) { | ||
| 69 | $asub->{$lkey} = [split(' ',$asub->{$lkey})] if (!ref($asub->{$lkey})); | ||
| 70 | $asub->{$lkey} = {map {($_=>undef)} @{$asub->{$lkey}}} if (!UNIVERSAL::isa($asub->{$lkey},'HASH')); | ||
| 71 | #$asub->trace("$lkey = ".join(' ', sort keys %{$asub->{$lkey}})); ##-- DEBUG | ||
| 72 | } | ||
| 73 | |||
| 74 | ##-- common variables | ||
| 75 | my $mlabel = $asub->{mootLabel}; | ||
| 76 | my $lz = $asub->{lz}; | ||
| 77 | my $xytags = $asub->{xyTags} // {}; | ||
| 78 | my $uctags = $asub->{ucTags} // {}; | ||
| 79 | my $stts = $asub->{stts}; | ||
| 80 | my $wmorph = $asub->{wMorph}; | ||
| 81 | my $toks = [map {@{$_->{tokens}}} @{$doc->{body}}]; | ||
| 82 | |||
| 83 | ##-- Step 1: ensure $tok->{moot}, $tok->{moot}{tag} are defined (should be obsolete!), apply tag hacks | ||
| 84 | my ($tok,$m,$w); | ||
| 85 | foreach $tok (@$toks) { | ||
| 86 | $m = $tok->{$mlabel} = {} if (!defined($m=$tok->{$mlabel})); | ||
| 87 | $m->{tag} = '@UNKNOWN' if (!defined($m->{tag})); | ||
| 88 | $w = $m->{word} // $tok->{text}; | ||
| 89 | |||
| 90 | ##-- tag hacks: stts | ||
| 91 | if ($stts) { | ||
| 92 | if ($m->{tag} eq 'TRUNC' && $w !~ m/\w/) { | ||
| 93 | ##-- tag-hack: avoid TRUNC tags for non-wordlike tokens | ||
| 94 | $m->{tag} = ($w =~ /[^[:punct:]]$/ ? 'XY' : '$('); | ||
| 95 | } | ||
| 96 | elsif ($w !~ m/[^[:punct:]\p{MathematicalOperators}]/ && $m->{tag} !~ /^(?:\$|XY)/) { | ||
| 97 | ##-- tag-hack: avoid "normal" tags for punctuation-only tokens | ||
| 98 | $m->{tag} = '$('; | ||
| 99 | } | ||
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | ##-- Step 2: run lemmatizer (populates $tok->{moot}{analyses}[$i]{lemma} | ||
| 104 | $lz->_analyzeGuts($toks,$opts) if ($lz->enabled($opts)); | ||
| 105 | |||
| 106 | ##-- Step 3: lemma-extraction & tag-sensitive lemmatization hacks | ||
| 107 | my %cache = qw(); ##-- $cache{"$tok_text/$moot_word/$moot_tag"} = $best_analysis | ||
| 108 | my ($u,$t,$la,$l,$key,$ma,$maa,%l2d, $ld, $a0,$ld0); | ||
| 109 | foreach $tok (@$toks) { | ||
| 110 | $u = $tok->{text}; | ||
| 111 | $m = $tok->{$mlabel}; | ||
| 112 | ($w,$t) = @$m{qw(word tag)}; | ||
| 113 | $key = "$u/$w/$t"; | ||
| 114 | if (defined($la=$cache{$key})) { | ||
| 115 | ##-- cached value | ||
| 116 | @$m{qw(details lemma)} = ($la,$la->{lemma}); | ||
| 117 | next; | ||
| 118 | } | ||
| 119 | |||
| 120 | ##-- get analyses | ||
| 121 | $ma = $m->{analyses} || []; ##-- all | ||
| 122 | $maa = @$ma ? [grep {$_->{tag} eq $t} @$ma] : $ma; ##-- ... with matching tag? | ||
| 123 | $ma = $maa if ($maa ne $ma && @$maa); ##-- ... or without! | ||
| 124 | |||
| 125 | if (!@$ma | ||
| 126 | || exists($xytags->{$t}) | ||
| 127 | || $t =~ /^FM\./ | ||
| 128 | #|| ($t eq 'NE' && !$tok->{msafe}) | ||
| 129 | ) | ||
| 130 | { | ||
| 131 | ##-- hack: bash XY-tagged elements to raw (possibly transliterated) text | ||
| 132 | $l = $m->{word} = (defined($tok->{exlex}) ? $tok->{exlex} : (defined($tok->{xlit}) && $tok->{xlit}{isLatinExt} ? $tok->{xlit}{latin1Text} : $tok->{text})); | ||
| 133 | $l =~ s/\s+/_/g; | ||
| 134 | #$l =~ s/^(.)(.*)$/$1\L$2\E/ ;#if (length($l) > 3 || $l =~ /[[:lower:]]/); | ||
| 135 | $l =~ s/[\x{ac}]//g; | ||
| 136 | $l = lc($l); | ||
| 137 | $l =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g if (exists($uctags->{$t})); ##-- implicitly upper-case NN, NE (in case e.g. 'NE' \in $xytags) | ||
| 138 | $m->{details} = $cache{$key} = {lemma=>$l,tag=>$t,details=>"*",prob=>0}; | ||
| 139 | } | ||
| 140 | else | ||
| 141 | { | ||
| 142 | ##-- extract lemma from "best" analysis | ||
| 143 | %l2d = qw(); | ||
| 144 | |||
| 145 | $a0 = $ld0 = undef; | ||
| 146 | foreach (sort {($a->{cost}||$a->{prob}||0)<=>($b->{cost}||$b->{prob}||0)} @$ma) { | ||
| 147 | ##-- get lemma distance | ||
| 148 | $l = $_->{lemma}; | ||
| 149 | $ld = Text::LevenshteinXS::distance($w, $l) ; | ||
| 150 | $ld += $wmorph*($_->{cost}||$_->{prob}||0); ##-- hack: morph cost clobbers edit-distance | ||
| 151 | $ld += $wmorph*(10) if (($_->{hi}||$_->{details}||'') =~ /\[orgname\]/); ##-- hack: punish orgname targets | ||
| 152 | $l2d{$l} = $ld if (!defined($l2d{$l}) || $l2d{$l} > $ld); | ||
| 153 | next if (defined($ld0) && $ld0 <= $ld); | ||
| 154 | $ld0 = $ld; | ||
| 155 | $a0 = $_; | ||
| 156 | } | ||
| 157 | #print STDERR "$tok->{text}:\n", map {"\t$_ : $l2d{$_}\n"} sort keys %l2d; ##-- DEBUG | ||
| 158 | |||
| 159 | $a0->{lemma} =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g if (exists($uctags->{$t})); ##-- implicitly upper-case lemmata NN, NE (in case e.g. 'NE') | ||
| 160 | $m->{details} = $cache{$key} = $a0; | ||
| 161 | } | ||
| 162 | $m->{lemma} = $m->{details}{lemma}; | ||
| 163 | } | ||
| 164 | |||
| 165 | ##-- return | ||
| 166 | return $doc; | ||
| 167 | } | ||
| 168 | |||
| 169 | ## @keys = $anl->typeKeys(\%opts) | ||
| 170 | ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes() | ||
| 171 | ## + override returns $anl->{mootLabel} or empty list, depending on $anl->{bytoken} | ||
| 172 | sub typeKeys { | ||
| 173 | my ($anl,$opts) = @_; | ||
| 174 | return ($opts->{"$anl->{label}.bytoken"} // $anl->{bytoken} ? qw() : $_[0]{mootLabel}); | ||
| 175 | } | ||
| 176 | |||
| 177 | 1; ##-- be happy | ||
| 178 | |||
| 179 | __END__ |