CAB/Chain/DTA.pm untested

filter: Statementstmt 0.0% 0 / 223 Branchbran 0.0% 0 / 20 Conditioncond 0.0% 0 / 13 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 0.0% 0 / 36 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 305 CC 11 SCAR 26.9
CC
11
Cov
0%
setupChains
72.0
ensureLoaded
6.0
doAnalyze
6.0
CRAP 14.7
1 ## -*- Mode: CPerl -*-
2 ## File: DTA::CAB::Chain::DTA.pm
3 ## Author: Bryan Jurish <moocow@cpan.org>
4 ## Description: robust analysis: default chain
5  
6 package DTA::CAB::Chain::DTA;
7 use DTA::CAB::Datum ':all';
8 use DTA::CAB::Chain::Multi;
9  
10 ##-- sub-analyzers
11 use DTA::CAB::Analyzer::ExLex;
12 use DTA::CAB::Analyzer::Cache::Static;
13 use DTA::CAB::Analyzer::TokPP;
14 use DTA::CAB::Analyzer::LTS;
15 use DTA::CAB::Analyzer::Morph;
16 use DTA::CAB::Analyzer::Morph::Latin;
17 use DTA::CAB::Analyzer::Morph::Extra::OrtLexHessen; ##-- auxilliary DB: Siedlungsplaetze_Hessen, from HLGL Marburg (optional)
18 use DTA::CAB::Analyzer::MorphSafe;
19 use DTA::CAB::Analyzer::Rewrite;
20 use DTA::CAB::Analyzer::Moot;
21 use DTA::CAB::Analyzer::Moot::Boltzmann;
22 use DTA::CAB::Analyzer::EqPho;
23 use DTA::CAB::Analyzer::EqPhoX;
24 use DTA::CAB::Analyzer::EqRW;
25 use DTA::CAB::Analyzer::RewriteSub;
26 use DTA::CAB::Analyzer::DmootSub;
27 use DTA::CAB::Analyzer::LangId::Simple;
28 use DTA::CAB::Analyzer::MootSub;
29 use DTA::CAB::Analyzer::EqLemma;
30 use DTA::CAB::Analyzer::DTAMapClass;
31 use DTA::CAB::Analyzer::SynCoPe::NER;
32 use DTA::CAB::Analyzer::GermaNet::Hypernyms;
33 use DTA::CAB::Analyzer::GermaNet::Hyponyms;
34 use DTA::CAB::Analyzer::GermaNet::Synonyms;
35 use DTA::CAB::Analyzer::DTAClean;
36 use DTA::CAB::Analyzer::Null;
37  
38 use IO::File;
39 use Carp;
40  
41 use strict;
42  
43 ##==============================================================================
44 ## Constants
45 ##==============================================================================
46 our @ISA = qw(DTA::CAB::Chain::Multi);
47  
48 ## supported date-optimized rewrite-ranges
49 our @RW_RANGES = qw(1600-1700 1700-1800 1800-1900);
50  
51 ##==============================================================================
52 ## Constructors etc.
53 ##==============================================================================
54  
55 ## $obj = CLASS_OR_OBJ->new(%args)
56 ## + object structure: HASH
57 sub new {
58 my $that = shift;
59 return $that->SUPER::new
60 (
61 ##-- analyzers
62 static => DTA::CAB::Analyzer::Cache::Static->new(typeKeys=>[qw(eqphox errid exlex f lts mlatin morph msafe pnd rw xlit lang)]),
63 exlex => DTA::CAB::Analyzer::ExLex->new(),
64 tokpp => DTA::CAB::Analyzer::TokPP->new(),
65 xlit => DTA::CAB::Analyzer::Unicruft->new(),
66 ##
67 lts => DTA::CAB::Analyzer::LTS->new(),
68 ##
69 morph => DTA::CAB::Analyzer::Morph->new(),
70 mlatin=> DTA::CAB::Analyzer::Morph::Latin->new(),
71 mhessen => DTA::CAB::Analyzer::Morph::Extra::OrtLexHessen->new(),
72 mhessengeo => DTA::CAB::Analyzer::Morph::Extra::GeoLexHessen->new(),
73 msafe => DTA::CAB::Analyzer::MorphSafe->new(),
74 rw => DTA::CAB::Analyzer::Rewrite->new(),
75 (map {("rw.$_" => DTA::CAB::Analyzer::Rewrite->new())} @RW_RANGES),
76 rwsub => DTA::CAB::Analyzer::RewriteSub->new(),
77 ##
78 eqphox => DTA::CAB::Analyzer::EqPhoX->new(),
79 eqpho => DTA::CAB::Analyzer::EqPho->new(),
80 eqrw => DTA::CAB::Analyzer::EqRW->new(),
81 ##
82 ##
83 dmoot => DTA::CAB::Analyzer::Moot::Boltzmann->new(), ##-- moot n-gram disambiguator ((n>=1)-grams)
84 dmoot1 => DTA::CAB::Analyzer::Moot::Boltzmann->new(), ##-- moot n-gram disambiguator (1-grams only)
85 dmootsub => DTA::CAB::Analyzer::DmootSub->new(), ##-- moot n-gram disambiguator: sub-morph
86 moot => DTA::CAB::Analyzer::Moot->new(), ##-- moot tagger (on dmoot output; (n>1)-grams)
87 moot1 => DTA::CAB::Analyzer::Moot->new(), ##-- moot tagger (on dmoot output; 1-grams only)
88 mootsub => DTA::CAB::Analyzer::MootSub->new(), ##-- moot tagger, post-processing hacks
89 mapclass => DTA::CAB::Analyzer::DTAMapClass->new(), ##-- mapping class (post-moot)
90 ##
91 langid => DTA::CAB::Analyzer::LangId::Simple->new(), ##-- language-guesser (stopword-based; between msafe and rw)
92 ##
93 ner => DTA::CAB::Analyzer::SynCoPe::NER->new(), ##-- ne-recognizer (post-moot)
94 ##
95 eqlemma => DTA::CAB::Analyzer::EqLemma->new(), ##-- eqlemma (best only)
96 ##
97 'gn-syn' => DTA::CAB::Analyzer::GermaNet::Synonyms->new(), ##-- GermaNet synonyms
98 'gn-isa' => DTA::CAB::Analyzer::GermaNet::Hypernyms->new(), ##-- GermaNet hyperyms (superclasses)
99 'gn-asi' => DTA::CAB::Analyzer::GermaNet::Hyponyms->new(), ##-- GermaNet hyponyms (subclasses)
100 ##
101 'ot-syn' => DTA::CAB::Analyzer::GermaNet::Synonyms->new(label=>'ot-syn'), ##-- OpenThesaurus synonyms
102 'ot-isa' => DTA::CAB::Analyzer::GermaNet::Hypernyms->new(label=>'ot-isa'), ##-- OpenThesaurus hypernyms (superclasses)
103 'ot-asi' => DTA::CAB::Analyzer::GermaNet::Hyponyms->new(label=>'ot-asi'), ##-- OpenThesaurus hyponyms (subclasses)
104 ##
105 clean => DTA::CAB::Analyzer::DTAClean->new(),
106 ##
107 null => DTA::CAB::Analyzer::Null->new(), ##-- null analyzer (for 'null' chain)
108  
109 ##-- security
110 autoClean => 0, ##-- always run 'clean' analyzer regardless of options; checked in both doAnalyze(), analyzeClean()
111 defaultChain => 'default',
112  
113 ##-- user args
114 @_,
115  
116 ##-- overrides
117 chains => undef, ##-- see setupChains() method
118 chain => undef, ##-- see setupChains() method
119 );
120 }
121  
122 ##==============================================================================
123 ## Methods: Chain selection
124 ##==============================================================================
125  
126 ## $ach = $ach->setupChains()
127 ## + setup default named sub-chains in $ach->{chains}
128 ## + override
129 ## + dta ddc and django cab demo both call 'all' chain:
130 ## - ddc web search: (client = 194.95.188.36 = services.dwds.de)
131 ## 2010-11-19 13:15:42 [24194] (DEBUG) DTA.CAB.Server.XmlRpc.Procedure: dta.cab.all.analyzeToken(): client=194.95.188.36
132 ## - django cab demo (client = 194.95.188.22 = www.deutschestextchiv.de)
133 ## 2010-11-19 13:15:09 [24194] (DEBUG) DTA.CAB.Server.XmlRpc.Procedure: dta.cab.all.analyzeToken(): client=194.95.188.22
134 sub setupChains {
135 my $ach = shift;
136 $ach->{rwsub}{chain} = [@$ach{qw(lts morph)}];
137 $ach->{dmootsub}{chain} = [@$ach{qw(morph mlatin)}]; #mhessen mhessengeo
138 my @akeys = grep {UNIVERSAL::isa($ach->{$_},'DTA::CAB::Analyzer')} keys(%$ach);
139 my $chains = $ach->{chains} =
140 {
141 (map {("sub.$_"=>[$ach->{$_}])} @akeys), ##-- sub.xlit, sub.lts, ...
142 #(map {("$_"=>[$ach->{$_}])} @akeys), ##-- xlit, lts, ...
143 ##
144 'sub.expand' =>[@$ach{qw(eqpho eqrw eqlemma)}],
145 'sub.sent' =>[@$ach{qw(dmoot dmootsub moot mootsub)}],
146 'sub.sent1' =>[@$ach{qw(dmoot1 dmootsub moot1 mootsub)}],
147 'sub.gn' =>[@$ach{qw(gn-syn gn-isa gn-asi)}],
148 'sub.ot' =>[@$ach{qw(ot-syn ot-isa ot-asi)}],
149 ##
150 'default.static' =>[@$ach{qw(static)}],
151 'default.exlex' =>[@$ach{qw(exlex)}],
152 'default.tokpp' =>[@$ach{qw(tokpp)}],
153 'default.xlit' =>[@$ach{qw(xlit)}],
154 'default.lts' =>[@$ach{qw(xlit lts)}],
155 'default.eqphox' =>[@$ach{qw(tokpp xlit lts eqphox)}],
156 'default.morph' =>[@$ach{qw(tokpp xlit morph)}],
157 'default.mlatin' =>[@$ach{qw(tokpp xlit mlatin)}],
158 'default.mhessen'=>[@$ach{qw(tokpp xlit mhessen)}],
159 'default.mhessengeo'=>[@$ach{qw(tokpp xlit mhessengeo)}],
160 'default.msafe' =>[@$ach{qw(tokpp xlit morph mlatin msafe)}],
161 'default.langid' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid)}],
162 'default.rw' =>[@$ach{qw(tokpp xlit rw)}],
163 'default.rw.safe'=>[@$ach{qw(tokpp xlit morph mlatin msafe langid rw)}],
164 'default.dmoot' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot)}],
165 'default.dmoot1' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot1)}],
166 'default.moot' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot dmootsub moot)}],
167 'default.moot1' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot1 dmootsub moot1)}],
168 'default.lemma' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot1 dmootsub moot mootsub)}],
169 'default.lemma1' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot1 dmootsub moot1 mootsub)}],
170 'default.ner' =>[@$ach{qw(tokpp xlit lts eqphox morph mlatin msafe langid rw dmoot dmootsub moot mootsub ner)}],
171 'default.base' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid)}],
172 'default.type' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw rwsub)}],
173 ##
174 'expand.old' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqpho eqrw)}],
175 'expand.ext' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqpho eqrw eqphox)}],
176 'expand.all' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqpho eqrw eqphox dmoot1 dmootsub moot1 mootsub eqlemma)}],
177 'expand.eqpho' =>[@$ach{qw(static exlex xlit lts eqpho)}],
178 'expand.eqrw' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqrw)}],
179 'expand.eqlemma' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub eqlemma)}],
180 'expand.gn-syn' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub gn-syn)}],
181 'expand.gn-isa' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub gn-isa)}],
182 'expand.gn-asi' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub gn-asi)}],
183 'expand.gn' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub gn-syn gn-isa gn-asi)}],
184 'expand.ot-syn' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub ot-syn)}],
185 'expand.ot-isa' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub ot-isa)}],
186 'expand.ot-asi' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub ot-asi)}],
187 'expand.ot' =>[@$ach{qw(static exlex xlit lts morph mlatin msafe rw eqphox dmoot1 dmootsub moot1 mootsub ot-syn ot-isa ot-asi)}],
188 ##
189 'norm' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw eqphox dmoot dmootsub moot mootsub)}],
190 'norm1' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw eqphox dmoot1 dmootsub moot1 mootsub)}],
191 'ner' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw eqphox dmoot dmootsub moot mootsub ner)}],
192 'caberr' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw eqphox dmoot dmootsub moot mootsub mapclass)}],
193 'caberr1' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw eqphox dmoot1 dmootsub moot1 mootsub mapclass)}],
194 'all' =>[@$ach{qw(static exlex tokpp xlit lts morph mlatin msafe langid rw rwsub eqpho eqrw eqphox dmoot dmootsub moot mootsub eqlemma)}], ##-- old dta clients use 'all'!
195 'clean' =>[@$ach{qw(clean)}],
196 'null' =>[$ach->{null}],
197 };
198 #$chains->{'default'} = [map {@{$chains->{$_}}} qw(default.type sub.sent)];
199  
200 ##-- chain aliases
201 $chains->{'default'} = $chains->{lemma} = $chains->{'norm'};
202 $chains->{'default1'} = $chains->{lemma1} = $chains->{'norm1'};
203 $chains->{'expand'} = $chains->{'expand.all'};
204  
205 ##-- BEGIN TEMPORARY custom chain(s): "hlgl": place-name recognition Hessen
206 $chains->{"norm.hlgl"} = [map {($_,$_ eq $ach->{mlatin} ? $ach->{mhessen} : qw())} @{$chains->{norm}}];
207 $chains->{"norm1.hlgl"} = [map {($_,$_ eq $ach->{mlatin} ? $ach->{mhessen} : qw())} @{$chains->{norm1}}];
208 ##
209 $chains->{"norm.hlgl.geo"} = [map {($_,$_ eq $ach->{mlatin} ? $ach->{mhessengeo} : qw())} @{$chains->{norm}}];
210 $chains->{"norm1.hlgl.geo"} = [map {($_,$_ eq $ach->{mlatin} ? $ach->{mhessengeo} : qw())} @{$chains->{norm1}}];
211 ##-- END TEMPORARY custom chain(s)
212  
213 ##-- date-dependent chains
214 foreach my $rng (@RW_RANGES) {
215 if ($ach->{"rw.$rng"} && ($ach->{"rw.$rng"}{enabled}//1)) {
216 foreach my $key (qw(norm norm1 lemma lemma1 default default1 expand)) {
217 $chains->{"$key.$rng"} = [map {$_ eq $ach->{rw} ? $ach->{"rw.$rng"} : $_} @{$chains->{$key}}];
218 }
219 } else {
220 $ach->warn("optimized rewrite cascade rw.$rng not available: disabling derived chains for range $rng");
221 delete $ach->{"rw.$rng"};
222 delete $chains->{"sub.rw.$rng"};
223 }
224 }
225  
226 ##-- sanitize chains
227 foreach (values %{$ach->{chains}}) {
228 @$_ = grep {ref($_)} @$_;
229 }
230  
231 ##-- set default chain
232 $ach->{chain} = $ach->{chains}{$ach->{defaultChain}};
233  
234 ##-- force default labels
235 foreach (grep {UNIVERSAL::isa($ach->{$_},'DTA::CAB::Analyzer')} keys(%$ach)) {
236 next if ($_ =~ /^(?:langid|rw\.[0-9\-]+|mhessen(?:geo)?)$/); ##-- keep labels for these analyzers
237 ($ach->{$_}{label} = $_) =~ s/1$//; ##-- truncate '1' suffix for label (e.g. dmoot1, moot1)
238 }
239 return $ach;
240 }
241  
242 ## \@analyzers = $ach->chain()
243 ## \@analyzers = $ach->chain(\%opts)
244 ## + get selected analyzer chain
245 ## + inherited from DTA::CAB::Chain::Multi
246 ## - calls setupChains() if $ach->{chain} is empty
247 ## - checks for $opts{chain} and returns $ach->{chains}{ $opts{chain} } if available
248  
249 ## $ach = $ach->ensureChain()
250 ## + checks for $ach->{chain}, calls $ach->setupChains() if needed
251 ## + inherited from DTA::CAB::Chain::Multi
252  
253 ##==============================================================================
254 ## Methods: I/O
255 ##==============================================================================
256  
257 ## $bool = $ach->ensureLoaded()
258 ## + ensures analysis data is loaded from default files
259 ## + inherited DTA::CAB::Chain::Multi override calls ensureChain() before inherited method
260 sub ensureLoaded {
261 my $ach = shift;
262 $ach->SUPER::ensureLoaded(@_) || return 0;
263  
264 ##-- hack: copy chain members AFTER loading for sub-analyzers, setting 'enabled' if appropriate
265 my ($subkey);
266 foreach $subkey (qw(rwsub dmootsub)) {
267 if (ref($ach->{$subkey})) {
268 foreach (grep {!$_->{"_${subkey}"}} @{$ach->{$subkey}{chain}}) {
269 $_ = bless( {%$_}, ref($_) );
270 $_->{label} = $subkey.'_'.$_->{label};
271 $_->{enabled} = $ach->{$subkey}{enabled};
272 $_->{"_$subkey"} = 1;
273 }
274 }
275 }
276  
277 return 1;
278 }
279  
280 ##==============================================================================
281 ## Methods: Persistence
282 ##==============================================================================
283  
284 ##======================================================================
285 ## Methods: Persistence: Perl
286  
287 ## @keys = $class_or_obj->noSaveKeys()
288 ## + returns list of keys not to be saved
289 ## + default just greps for CODE-refs
290 ## + inherited from DTA::CAB::Chain::Multi: override appends {chain},{chains}
291  
292 ## $saveRef = $obj->savePerlRef()
293 ## + return reference to be saved (top-level objects only)
294 ## + inherited from DTA::CAB::Persistent
295  
296 ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref)
297 ## + default implementation just clobbers $CLASS_OR_OBJ with $ref and blesses
298 ## + inherited from DTA::CAB::Persistent
299  
300 ##==============================================================================
301 ## Methods: Analysis
302 ##==============================================================================
303  
304 ##------------------------------------------------------------------------
305 ## Methods: Analysis: v1.x: Utils
306  
307 ## $bool = $anl->doAnalyze(\%opts, $name)
308 ## + alias for $anl->can("analyze${name}") && (!exists($opts{"doAnalyze${name}"}) || $opts{"doAnalyze${name}"})
309 ## + override checks $anl->{autoClean} flag
310 sub doAnalyze {
311 my ($anl,$opts,$name) = @_;
312 return 1 if ($anl->{autoClean} && $name eq 'Clean');
313 return $anl->SUPER::doAnalyze($opts,$name);
314 }
315  
316  
317 ##------------------------------------------------------------------------
318 ## Methods: Analysis: v1.x: API
319  
320 ## $doc = $ach->analyzeDocument($doc,\%opts)
321 ## + analyze a DTA::CAB::Document $doc
322 ## + top-level API routine
323 ## + INHERITED from DTA::CAB::Analyzer
324  
325 ## $doc = $ach->analyzeTypes($doc,$types,\%opts)
326 ## + perform type-wise analysis of all (text) types in $doc->{types}
327 ## + Chain default calls $a->analyzeTypes for each analyzer $a in the chain
328 ## + INHERITED from DTA::CAB::Chain
329  
330 ## $doc = $ach->analyzeTokens($doc,\%opts)
331 ## + perform token-wise analysis of all tokens $doc->{body}[$si]{tokens}[$wi]
332 ## + default implementation just shallow copies tokens in $doc->{types}
333 ## + INHERITED from DTA::CAB::Analyzer
334  
335 ## $doc = $ach->analyzeSentences($doc,\%opts)
336 ## + perform sentence-wise analysis of all sentences $doc->{body}[$si]
337 ## + Chain default calls $a->analyzeSentences for each analyzer $a in the chain
338 ## + INHERITED from DTA::CAB::Chain
339  
340 ## $doc = $ach->analyzeLocal($doc,\%opts)
341 ## + perform local document-level analysis of $doc
342 ## + Chain default calls $a->analyzeLocal for each analyzer $a in the chain
343 ## + INHERITED from DTA::CAB::Chain
344  
345 ## $doc = $ach->analyzeClean($doc,\%opts)
346 ## + cleanup any temporary data associated with $doc
347 ## + Chain default calls $a->analyzeClean for each analyzer $a in the chain,
348 ## then superclass Analyzer->analyzeClean
349 sub analyzeClean {
350 my ($ach,$doc,$opts) = @_;
351 $ach->SUPER::analyzeClean($doc,$opts); ##-- inherited from DTA::CAB::Chain (chain-local cleanup)
352 $ach->analyzeClean_rm_undef($doc,$opts); ##-- remove keys with undef values from tokens
353 return $doc if (!$ach->{autoClean} && !exists($opts->{doAnalyzeClean})); ##-- don't "clean" by default
354 return $ach->{clean}->analyzeClean($doc,$opts);
355 }
356  
357  
358 ##------------------------------------------------------------------------
359 ## Methods: Analysis: v1.x: Wrappers
360  
361 ## $tok = $ach->analyzeToken($tok_or_string,\%opts)
362 ## + perform type- and token-analyses on $tok_or_string
363 ## + wrapper for $ach->analyzeDocument()
364 ## + INHERITED from DTA::CAB::Analyzer
365  
366 ## $tok = $ach->analyzeSentence($sent_or_array,\%opts)
367 ## + perform type-, token-, and sentence-analyses on $sent_or_array
368 ## + wrapper for $ach->analyzeDocument()
369 ## + INHERITED from DTA::CAB::Analyzer
370  
371 ## $rpc_xml_base64 = $anl->analyzeData($data_str,\%opts)
372 ## + analyze a raw (formatted) data string $data_str with internal parsing & formatting
373 ## + wrapper for $anl->analyzeDocument()
374 ## + INHERITED from DTA::CAB::Analyzer
375  
376 ##==============================================================================
377 ## Methods: XML-RPC
378 ## + INHERITED from DTA::CAB::Chain::Multi
379  
380 1; ##-- be happy
381  
382 __END__