CAB/Analyzer/MootSub.pm untested

filter: Statementstmt 0.0% 0 / 96 Branchbran 0.0% 0 / 46 Conditioncond 0.0% 0 / 59 MC/DCmcdc 0.0% 0 / 32 Subroutinesub 0.0% 0 / 9 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 246 CC 24 SCAR 58.6
CC
24
Cov
0%
analyzeSentences
506.0
doAnalyze
6.0
typeKeys
6.0
CRAP 349.0
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::MootSub.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: post-processing for moot PoS tagger in DTA chain
6 ## + tweaks $tok->{moot}{word}, instantiates $tok->{moot}{lemma}
7  
8 package DTA::CAB::Analyzer::MootSub;
9 use DTA::CAB::Analyzer ':child';
10 use DTA::CAB::Analyzer::Lemmatizer;
11 use Text::LevenshteinXS qw();
12 use Carp;
13 use strict;
14 our @ISA = qw(DTA::CAB::Analyzer);
15  
16 ##======================================================================
17 ## Methods
18  
19 ## $obj = CLASS_OR_OBJ->new(%args)
20 ## + object structure, %args:
21 ## mootLabel => $label, ##-- label for Moot tagger object (default='moot')
22 ## lz => $lemmatizer, ##-- DTA::CAB::Analyzer::Lemmatizer sub-object
23 ## bytoken => $bool, ##-- type-wise expand $mootLabel if true; depends on global option "${label}.bytoken"; see typeKeys() method
24 ## xyTags => $xytags, ##-- use literal text (not dmoot) for these tags (string, array, or HASH-ref; default=[qw(XY FM)])
25 ## ucTags => $uctags, ##-- implicitly upper-case lemmata for these tags (string, array, or HASH-ref; default=[qw(NN NE)])
26 ## stts => $bool, ##-- implicitly use STTS-specific heuristics? (default=1)
27 ## wMorph => $bool, ##-- morph-cost coefficient for lemma-selection heuristics (default=1000)
28 sub new {
29 my $that = shift;
30 my $asub = $that->SUPER::new(
31 ##-- analysis selection
32 label => 'mootsub',
33 mootLabel => 'moot',
34 bytoken => 1,
35 lz => DTA::CAB::Analyzer::Lemmatizer->new(analyzeGet =>$DTA::CAB::Analyzer::Lemmatizer::GET_MOOT_ANALYSES,
36 analyzeGetText=>$DTA::CAB::Analyzer::Lemmatizer::GET_MOOT_TEXT,
37 analyzeWhich =>'Sentences',
38 segmentLabel =>'segs',
39 ),
40 xyTags => 'XY FM', #CARD NE ##-- use literal text (not dmoot) for these tags
41 ucTags => 'NN NE', ##-- implicitly upper-case lemmata for these tags
42 stts => 1,
43 wMorph => 1000,
44  
45 ##-- user args
46 @_
47 );
48  
49 $asub->{lz}{label} = $asub->{label}."_lz";
50 return $asub;
51 }
52  
53 ## $bool = $anl->doAnalyze(\%opts, $name)
54 ## + override: only allow analyzeSentences()
55 sub doAnalyze {
56 my $anl = shift;
57 return 0 if (defined($_[1]) && $_[1] ne 'Sentences');
58 return $anl->SUPER::doAnalyze(@_);
59 }
60  
61 ## $doc = $anl->Sentences($doc,\%opts)
62 ## + post-processing for 'moot' object
63 sub analyzeSentences {
64 my ($asub,$doc,$opts) = @_;
65 return $doc if (!$asub->enabled($opts));
66  
67 ##-- tweak list-tags
68 foreach my $lkey (grep {defined($asub->{$_})} qw(xyTags ucTags)) {
69 $asub->{$lkey} = [split(' ',$asub->{$lkey})] if (!ref($asub->{$lkey}));
70 $asub->{$lkey} = {map {($_=>undef)} @{$asub->{$lkey}}} if (!UNIVERSAL::isa($asub->{$lkey},'HASH'));
71 #$asub->trace("$lkey = ".join(' ', sort keys %{$asub->{$lkey}})); ##-- DEBUG
72 }
73  
74 ##-- common variables
75 my $mlabel = $asub->{mootLabel};
76 my $lz = $asub->{lz};
77 my $xytags = $asub->{xyTags} // {};
78 my $uctags = $asub->{ucTags} // {};
79 my $stts = $asub->{stts};
80 my $wmorph = $asub->{wMorph};
81 my $toks = [map {@{$_->{tokens}}} @{$doc->{body}}];
82  
83 ##-- Step 1: ensure $tok->{moot}, $tok->{moot}{tag} are defined (should be obsolete!), apply tag hacks
84 my ($tok,$m,$w);
85 foreach $tok (@$toks) {
86 $m = $tok->{$mlabel} = {} if (!defined($m=$tok->{$mlabel}));
87 $m->{tag} = '@UNKNOWN' if (!defined($m->{tag}));
88 $w = $m->{word} // $tok->{text};
89  
90 ##-- tag hacks: stts
91 if ($stts) {
92 if ($m->{tag} eq 'TRUNC' && $w !~ m/\w/) {
93 ##-- tag-hack: avoid TRUNC tags for non-wordlike tokens
94 $m->{tag} = ($w =~ /[^[:punct:]]$/ ? 'XY' : '$(');
95 }
96 elsif ($w !~ m/[^[:punct:]\p{MathematicalOperators}]/ && $m->{tag} !~ /^(?:\$|XY)/) {
97 ##-- tag-hack: avoid "normal" tags for punctuation-only tokens
98 $m->{tag} = '$(';
99 }
100 }
101 }
102  
103 ##-- Step 2: run lemmatizer (populates $tok->{moot}{analyses}[$i]{lemma}
104 $lz->_analyzeGuts($toks,$opts) if ($lz->enabled($opts));
105  
106 ##-- Step 3: lemma-extraction & tag-sensitive lemmatization hacks
107 my %cache = qw(); ##-- $cache{"$tok_text/$moot_word/$moot_tag"} = $best_analysis
108 my ($u,$t,$la,$l,$key,$ma,$maa,%l2d, $ld, $a0,$ld0);
109 foreach $tok (@$toks) {
110 $u = $tok->{text};
111 $m = $tok->{$mlabel};
112 ($w,$t) = @$m{qw(word tag)};
113 $key = "$u/$w/$t";
114 if (defined($la=$cache{$key})) {
115 ##-- cached value
116 @$m{qw(details lemma)} = ($la,$la->{lemma});
117 next;
118 }
119  
120 ##-- get analyses
121 $ma = $m->{analyses} || []; ##-- all
122 $maa = @$ma ? [grep {$_->{tag} eq $t} @$ma] : $ma; ##-- ... with matching tag?
123 $ma = $maa if ($maa ne $ma && @$maa); ##-- ... or without!
124  
125 if (!@$ma
126 || exists($xytags->{$t})
127 || $t =~ /^FM\./
128 #|| ($t eq 'NE' && !$tok->{msafe})
129 )
130 {
131 ##-- hack: bash XY-tagged elements to raw (possibly transliterated) text
132 $l = $m->{word} = (defined($tok->{exlex}) ? $tok->{exlex} : (defined($tok->{xlit}) && $tok->{xlit}{isLatinExt} ? $tok->{xlit}{latin1Text} : $tok->{text}));
133 $l =~ s/\s+/_/g;
134 #$l =~ s/^(.)(.*)$/$1\L$2\E/ ;#if (length($l) > 3 || $l =~ /[[:lower:]]/);
135 $l =~ s/[\x{ac}]//g;
136 $l = lc($l);
137 $l =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g if (exists($uctags->{$t})); ##-- implicitly upper-case NN, NE (in case e.g. 'NE' \in $xytags)
138 $m->{details} = $cache{$key} = {lemma=>$l,tag=>$t,details=>"*",prob=>0};
139 }
140 else
141 {
142 ##-- extract lemma from "best" analysis
143 %l2d = qw();
144  
145 $a0 = $ld0 = undef;
146 foreach (sort {($a->{cost}||$a->{prob}||0)<=>($b->{cost}||$b->{prob}||0)} @$ma) {
147 ##-- get lemma distance
148 $l = $_->{lemma};
149 $ld = Text::LevenshteinXS::distance($w, $l) ;
150 $ld += $wmorph*($_->{cost}||$_->{prob}||0); ##-- hack: morph cost clobbers edit-distance
151 $ld += $wmorph*(10) if (($_->{hi}||$_->{details}||'') =~ /\[orgname\]/); ##-- hack: punish orgname targets
152 $l2d{$l} = $ld if (!defined($l2d{$l}) || $l2d{$l} > $ld);
153 next if (defined($ld0) && $ld0 <= $ld);
154 $ld0 = $ld;
155 $a0 = $_;
156 }
157 #print STDERR "$tok->{text}:\n", map {"\t$_ : $l2d{$_}\n"} sort keys %l2d; ##-- DEBUG
158  
159 $a0->{lemma} =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g if (exists($uctags->{$t})); ##-- implicitly upper-case lemmata NN, NE (in case e.g. 'NE')
160 $m->{details} = $cache{$key} = $a0;
161 }
162 $m->{lemma} = $m->{details}{lemma};
163 }
164  
165 ##-- return
166 return $doc;
167 }
168  
169 ## @keys = $anl->typeKeys(\%opts)
170 ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes()
171 ## + override returns $anl->{mootLabel} or empty list, depending on $anl->{bytoken}
172 sub typeKeys {
173 my ($anl,$opts) = @_;
174 return ($opts->{"$anl->{label}.bytoken"} // $anl->{bytoken} ? qw() : $_[0]{mootLabel});
175 }
176  
177 1; ##-- be happy
178  
179 __END__