CAB/Analyzer/Lemmatizer.pm untested

filter: Statementstmt 0.0% 0 / 80 Branchbran 0.0% 0 / 26 Conditioncond 0.0% 0 / 27 MC/DCmcdc 0.0% 0 / 17 Subroutinesub 0.0% 0 / 13 Podpod 0.0% 0 / 10 totaltotal 0.0% 0 / 173 CC 14 SCAR 38.0
CC
14
Cov
0%
_analyzeGuts
110.0
typeKeys
6.0
doAnalyze
6.0
CRAP 44.8
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Lemmatizer.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: lemma extractor for TAGH analyses or bare text
6  
7 package DTA::CAB::Analyzer::Lemmatizer;
8 use DTA::CAB::Analyzer;
9 use Carp;
10 use strict;
11 our @ISA = qw(DTA::CAB::Analyzer);
12  
13 ##==============================================================================
14 ## Globals
15  
16 ## $GET_MORPH
17 ## + code string: \@morph_analyses = "$GET_MORPH"->()
18 ## + available vars: $tok, $lz
19 our $GET_MORPH = '$tok->{morph}';
20  
21 ## $GET_DMOOT_MORPH
22 ## + code string: \@morph_analyses = "$GET_DMOOT_MORPH"->()
23 ## + available vars: $tok, $lz
24 our $GET_DMOOT_MORPH = '$tok->{dmoot} ? $tok->{dmoot}{morph} : undef';
25  
26 ## $GET_TEXT
27 ## + code string: get text for analysis $_
28 ## + available vars: $tok, $tokm (array of analyses), $ma (current analysis), $lz (analyzer obj),
29 our $GET_TEXT = '$tok->{xlit} ? $tok->{xlit}{latin1Text} : $tok->{text}';
30  
31 ## $GET_MOOT_ANALYSES
32 ## + code string: \@morph_analyses = "$GET_DMOOT_MORPH"->()
33 ## + available vars: $tok, $lz
34 our $GET_MOOT_ANALYSES = '$tok->{moot} ? $tok->{moot}{analyses} : undef';
35  
36 ## $GET_MOOT_TEXT
37 ## + code string: get text for analysis $_
38 ## + available vars: $tok, $tokm (array of analyses), $ma (current analysis), $lz (analyzer obj),
39 our $GET_MOOT_TEXT = '$tok->{moot} ? $tok->{moot}{word} : ('.$GET_TEXT.')';
40  
41 ##==============================================================================
42 ## Methods
43  
44 ## $obj = CLASS_OR_OBJ->new(%args)
45 ## + object structure, %args:
46 ## analyzeGet => $code, ##-- pseudo-accessor: @morph_analyses = "$code"->(\@toks)
47 ## analyzeWhich => $which, ##-- e.g. 'Types','Tokens','Sentences','Local': default=Types
48 ## ## + the underlying analysis is always performed by the analyzeTypes() method! (default='Types')
49 ## analyzeLabel => $label, ##-- ouput label (default='lemma')
50 sub new {
51 my $that = shift;
52 my $lz = $that->SUPER::new(
53 ##-- analysis selection
54 label => 'lemma',
55 analyzeLabel => 'lemma',
56 analyzeGet => $GET_MORPH,
57 analyzeGetText => $GET_TEXT,
58 analyzeWhich => 'Types',
59 #typeKeys => undef,
60  
61 ##-- user args
62 @_
63 );
64 return $lz;
65 }
66  
67 ## @keys = $anl->typeKeys()
68 ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes()
69 ## + override returns @{$lt->{typeKeys}}
70 sub typeKeys {
71 return @{$_[0]{typeKeys}} if ($_[0]{typeKeys} && @{$_[0]{typeKeys}});
72 return qw();
73 }
74  
75 ## $bool = $anl->doAnalyze(\%opts, $name)
76 ## + override: only allow analyzeSentences()
77 sub doAnalyze {
78 my $anl = shift;
79 return 0 if (defined($_[1]) && $_[1] ne $anl->{analyzeWhich});
80 return $anl->SUPER::doAnalyze(@_);
81 }
82  
83 ## \@toks = $anl->_analyzeGuts(\@toks,\%opts)
84 ## + guts: analyze all tokens in \@toks
85 sub _analyzeGuts {
86 my ($lz,$toks,$opts) = @_;
87  
88 ##-- common vars
89 my $lab = $lz->{label};
90 my $alab = defined($lz->{analyzeLabel}) ? $lz->{analyzeLabel} : $lab;
91 my $lab_txt = $lab."_text";
92 my $lab_key = $lab."_key";
93  
94 ##-- prepare map $key2a = { "$text\t$hi" => $analysis, ... }
95 my $key2a = {};
96 my ($tok,$tokm,$ma,$txt,$key);
97 my $prep_code =
98 'foreach $tok (@$toks) {
99 next if (!($tokm='.$lz->{analyzeGet}.'));
100 foreach (grep {defined($_)} @$tokm) {
101 $txt = $_->{$lab_txt} = '.$lz->{analyzeGetText}.';
102 $key = $_->{$lab_key} = $txt."\t".(defined($_->{hi}) ? $_->{hi} : $_->{details});
103 next if (exists($key2a->{$key}));
104 $key2a->{$key} = $_;
105 }
106 }';
107 my $prep_sub = eval "sub { $prep_code }";
108 $lz->logcluck("_analyzeGuts(): could not compile preprocessing sub {$prep_code}: $@") if (!$prep_sub);
109 $prep_sub->();
110  
111 ##-- lemmatize, type-wise by (text+analysis)-pair
112 my ($lemma,$tag,$lemmaFromMorph);
113 foreach (values %$key2a) {
114 $lemma = defined($_->{hi}) ? $_->{hi} : $_->{details};
115 if (defined($lemma) && $lemma ne '' && $lemma =~ /^[^\]]+\[/) { ##-- tagh analysis (vs. tokenizer-supplied analysis)
116 #$lemma =~ s/\~e?t(?=\W)/\~en/g; ##-- rename verb inflection morphs [BUG: l("Christ/N\en~tum[_NN]") = Christenenum]
117 $lemma =~ s/\[<([^\>\]]*)>\]/<$1>/g; ##-- unquote taghm-2.5 "diamond-tags", e.g "kurz<A>\@Stiel<N>~ig[_ADJD]<none>"
118 $lemma =~ s/\[.*$//; ##-- trim everything after first non-character symbol
119 #$lemma =~ s/(?:\/[A-Za-z]{1,2})|(?:\bge\\\|)|(?:[\\\~\|\=\+\#\x{ac}])//g; ##-- hack: remove "ge\|" prefixes too (but not e.g. "ver\|", "be\|", etc.)
120 $lemma =~ s/(?:\/[A-Za-z]{1,2})|(?:\[?<[^>]+>\]?)|(?:[\@\\\~\|\=\+\#\x{ac}])//g; ##-- unhack: don't remove "ge\|" prefixes (for consistency e.g. with dwds-kc20)
121 $lemmaFromMorph = 1;
122 } else {
123 $lemma = $_->{$lab_txt};
124 $lemma =~ s/\x{ac}//g;
125 $lemmaFromMorph = 0;
126 }
127 ##-- extract tag if available
128 $tag = $_->{tag} || ($_->{hi} && $_->{hi} =~ m/\[_([^\]]+)\]/ ? $1 : '');
129 ##
130 ##-- normalization
131 $lemma =~ s/(?:^\s+|\s+\z)//g;
132 $lemma =~ s/\s+/_/g;
133 if ($lemmaFromMorph && $tag =~ /^(?:NE|XY)$/) {
134 ##-- retain morphology-supplied original case for names and symbols
135 ;
136 }
137 elsif ($_->{tag} ne 'XY') {
138 ##-- lower-case all lemmata here, otherwise Anschlußstelle->AnschlußStelle (mantis #23127)
139 $lemma = lc($lemma);
140 $lemma =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g
141 if (#$_->{tag} ? ($_->{tag}=~m/^N/) : ##-- disabled 2016-06-01 for Helsinki-style english morphology; see ucTags key in Analyzer::MootSub
142 ($_->{hi} && $_->{hi}=~m/\[_N/)
143 );
144 }
145 $_->{$alab} = $lemma;
146 }
147  
148 ##-- postprocessing: re-expand types
149 my $postp_code =
150 'foreach $tok (@$toks) {
151 next if (!($tokm='.$lz->{analyzeGet}.'));
152 foreach (grep {defined($_)} @$tokm) {
153 $_->{$alab} = $key2a->{$_->{$lab_key}}{$alab};
154 delete(@$_{$lab_key,$lab_txt});
155 }
156 }';
157 my $postp_sub = eval "sub { $postp_code }";
158 $lz->logcluck("_analyzeGuts(): could not compile postprocessing sub {$postp_code}: $@") if (!$postp_sub);
159 $postp_sub->();
160  
161 return $toks;
162 }
163  
164  
165 ## $doc = $anl->analyzeTypes($doc,\%types,\%opts)
166 ## + perform type-wise analysis of all (text) types in $doc->{types}
167 sub analyzeTypes {
168 my ($lz,$doc,$types,$opts) = @_;
169 return $doc if ($lz->{analyzeWhich} ne 'Types');
170 $lz->_analyzeGuts([values %$types],$opts);
171 return $doc;
172 }
173  
174 ## $doc = $anl->analyzeOther($which, $doc,\%opts)
175 ## + analyze all tokens in $doc
176 sub analyzeOther {
177 my ($anl,$which,$doc,$opts) = @_;
178 return $doc if (defined($which) && $which ne $anl->{analyzeWhich});
179 $anl->_analyzeGuts([map {@{$_->{tokens}}} @{$doc->{body}}],$opts);
180 return $doc;
181 }
182  
183 sub analyzeTokens { return $_[0]->analyzeOther('Tokens',@_[1..$#_]); }
184 sub analyzeSentences { return $_[0]->analyzeOther('Sentences',@_[1..$#_]); }
185 sub analyzeLocal { return $_[0]->analyzeOther('Local',@_[1..$#_]); }
186 sub analyzeClean { return $_[0]->analyzeOther('Clean',@_[1..$#_]); }
187  
188  
189  
190 1; ##-- be happy
191  
192 __END__