CAB/Analyzer/MorphSafe.pm untested

filter: Statementstmt 0.0% 0 / 79 Branchbran 0.0% 0 / 30 Conditioncond 0.0% 0 / 57 MC/DCmcdc 0.0% 0 / 32 Subroutinesub 0.0% 0 / 12 Podpod 0.0% 0 / 5 totaltotal 0.0% 0 / 215 CC 16 SCAR 36.1
CC
16
Cov
0%
analyzeTypes
90.0
ensureLoaded
20.0
ensureDict
20.0
CRAP 37.0
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::MorphSafe.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: safety checker for analyses output by DTA::CAB::Analyzer::Morph (TAGH)
6  
7 package DTA::CAB::Analyzer::MorphSafe;
8  
9 use DTA::CAB::Analyzer;
10 use DTA::CAB::Analyzer::Dict;
11 use DTA::CAB::Unify ':all';
12  
13 use Encode qw(encode decode);
14 use IO::File;
15 use Carp;
16  
17 use strict;
18  
19 ##==============================================================================
20 ## Globals
21 ##==============================================================================
22  
23 our @ISA = qw(DTA::CAB::Analyzer);
24  
25 ## %badTypes = ($text=>$isGood, ...)
26 ## + default hash of bad text types
27 our %badTypes = (
28 #Andre=>0, ##-- bad: type: Andre[_NE][firstname][none][none][sg][nom_acc_dat]
29 );
30  
31 ## %badMorphs = ($taghMorph=>$isBad, ...)
32 ## + default hash of bad TAGH morphs
33 our %badMorphs = (
34 #'Thür'=>1, ##-- bad: stem
35 #'/ON'=>1, ##-- bad: stem class: organization name
36 );
37  
38 ## %badTags = ($taghTag=>$isBad, ...)
39 ## + default hash of bad TAGH tags
40 our %badTags = (
41 'FM'=>1, ##-- bad: FM (e.g. That:That[_FM][en])
42 'XY'=>1, ##-- bad: XY
43 'ITJ'=>1, ##-- bad: ITJ
44 );
45  
46 ##==============================================================================
47 ## Constructors etc.
48 ##==============================================================================
49  
50 ## $obj = CLASS_OR_OBJ->new(%args)
51 ## + object structure, new:
52 ## ##-- analysis selection
53 ## allowTokenizerAnalyses => $bool, ##-- if true, tokenizer-analyzed tokens (as determined by $tok->{toka}, $tok->{tokpp}) are "safe"; (default=true)
54 ## allowExlexAnalyses => $bool, ##-- if true, exlex-analyzed tokens (as determined by $tok->{exlex}) are "safe"; (default=false)
55 ## allowApostropheS => $bool, ##-- if true, "-'s" tokens with a morph analysis are "safe" (default=true)
56 ## tokMorphKey => $key, ##-- key for token 'morph' property (default='morph')
57 ## morphHiKey => $key, ##-- key for morph analysis 'hi' property (default='hi')
58 ##
59 ## ##-- Exception lexicon options
60 ## #dict => $dict, ##-- exception lexicon as a DTA::CAB::Analyzer::Dict object or option hash
61 ## # ## + default=undef
62 ## #dictClass => $class, ##-- fallback class for new dict (default='DTA::CAB::Analyzer::Dict')
63 ##
64 sub new {
65 my $that = shift;
66 return $that->SUPER::new(
67 ##-- options
68 label => 'msafe',
69 allowTokenizerAnalyses => 1,
70 allowExlexAnalyses => 0,
71 allowApostropheS => 1,
72 tokMorphKey => 'morph',
73 morphHiKey => 'hi',
74  
75 ##-- dictionary stuff
76 #badTypesFile => undef, ##-- filename of ($text "\t" $isGoodBool) mapping for raw utf8 text types
77 #badTypes => {%badTypes}, ##-- hash of bad utf8 text types ($text=>$isGoodBool)
78 ##
79 #badMorphsFile => undef, ##-- filename of ($taghMorph "\t" $isBadBool) mapping for TAGH morph components
80 #badMorphs => {%badMorphs}, ##-- hash of bad TAGH morphs ($taghMorph=>$isBadBool)
81 ##
82 #badTagsFile => undef, ##-- filename of ($taghTag "\t" $isBadBool) mapping for TAGH tags (without '[_', ']')
83 #badTags => {%badTags}, ##-- hash of bad TAGH tags ($taghTag=>$isBadBool)
84  
85 ##-- user args
86 @_
87 );
88 }
89  
90 ##==============================================================================
91 ## Methods: I/O
92 ##==============================================================================
93  
94 ## $bool = $msafe->ensureLoaded()
95 ## + ensures analysis data is loaded
96 sub ensureLoaded {
97 my $msafe = shift;
98 my $rc = 1;
99  
100 ##-- ensure: dict: badTypes
101 $rc &&= $msafe->ensureDict('badTypes',\%badTypes) if (!$msafe->{badTypes});
102  
103 ##-- ensure: dict: badMorphs
104 $rc &&= $msafe->ensureDict('badMorphs',\%badMorphs) if (!$msafe->{badMorphs});
105  
106 ##-- ensure: dict: badTags
107 $rc &&= $msafe->ensureDict('badTags',\%badTags) if (!$msafe->{badTags});
108  
109 return $rc;
110 }
111  
112 ##--------------------------------------------------------------
113 ## Methods: I/O: Input: Dictionaries: generic
114  
115 ## $bool = $msafe->ensureDict($dictName,\%dictDefault)
116 sub ensureDict {
117 my ($ms,$name,$default) = @_;
118 return 1 if ($ms->{$name}); ##-- already defined
119 return $ms->loadDict($name,$ms->{"${name}File"}) if ($ms->{"${name}File"});
120 $ms->{$name} = $default ? {%$default} : {};
121 return 1;
122 }
123  
124 ## \%dictHash_or_undef = $msafe->loadDict($dictName,$dictFile)
125 sub loadDict {
126 my ($ms,$name,$dfile) = @_;
127 delete($ms->{$name});
128 $ms->info("loading exception lexicon from '$dfile'");
129  
130 ##-- hack: generate a temporary dict object
131 my $dict = DTA::CAB::Analyzer::Dict->new(label=>($ms->{label}.".dict.$name"), dictFile=>$dfile);
132 $dict->ensureLoaded();
133 return undef if (!$dict->dictOk);
134  
135 ##-- clobber dict
136 $ms->{$name} = $dict->dictHash;
137 }
138  
139  
140 ##==============================================================================
141 ## Methods: Analysis: v1.x
142 ##==============================================================================
143  
144 ## $doc = $msafe->analyzeTypes($doc,\%types,\%opts)
145 ## + perform type-wise analysis of all (text) types in %types (= %{$doc->{types}})
146 ## + checks for "safe" analyses in $tok->{morph} for each $tok in $doc->{types}
147 ## + sets $tok->{ $anl->{label} } = $bool
148 sub analyzeTypes {
149 my ($ms,$doc,$types,$opts) = @_;
150 $types = $doc->types if (!$types);
151 #$types = $doc->extendTypes($types,'morph') if (!grep {$_->{morph}} values(%$types)); ##-- DEBUG
152  
153 my $label = $ms->{label};
154 my $want_toka = $ms->{allowTokenizerAnalyses};
155 my $want_exlex= $ms->{allowExlexAnalyses};
156 my $want_apos_s = $ms->{allowApostropheS};
157 my $tok_morph = $ms->{tokMorphKey};
158 my $morph_hi = $ms->{morphHiKey};
159 my $badTypes = $ms->{badTypes}||{};
160 my $badTags = $ms->{badTags}||{};
161 my $badMorphs = $ms->{badMorphs}||{};
162  
163 my ($tok,$safe,$nsafe,@m,$ma);
164 foreach $tok (values %$types) {
165 next if (defined($tok->{$label})); ##-- avoid re-analysis (e.g. of global exlex-provided analyses)
166  
167 ##-- no dict entry: use morph heuristics
168 $safe =
169 (($want_exlex && defined($tok->{exlex})) ##-- exception-lexicon analyses are considered "safe"
170 || (
171 $want_toka
172 && (
173 ($tok->{toka} && @{$tok->{toka}}) ##-- tokenizer-analyzed words are considered "safe"
174 ||
175 ($tok->{tokpp} && @{$tok->{tokpp}})
176 )
177 )
178 || (
179 $tok->{text} !~ m/^[\p{Letter}\#\@]/ ##-- non-alphabetic tokens are (usually) "safe"
180 ## ... unless they contain a placeholder for unrecognized material ('#' or '@')
181 ## [replaces /[[:digit:][:punct:]]/ heuristic; Tue, 28 Feb 2012 11:21:29 +0100]
182 )
183 || $tok->{mlatin} ##-- latin words are "safe" [NEW Fri, 01 Apr 2011 11:38:45 +0200]
184 || (
185 $want_apos_s ##-- "-'s" words with a morph analysis may be safe
186 && $tok->{$tok_morph}
187 && $tok->{text} =~ m/^(.+)[\'\x{2018}\x{2019}]s$/
188 )
189 );
190  
191 ##-- are we still unsafe? then check for some "safe" morph analysis: if found, set $safe=1 & bug out
192 if (!$safe
193 && !defined($safe=$badTypes->{$tok->{text}}) ##-- ... only if it's not a known type
194 && $tok->{$tok_morph} ##-- ... and it has morph analyses (empty $tok->{morph} will still be "unsafe")
195 )
196 {
197 MORPHA:
198 foreach (@{$tok->{$tok_morph}}) {
199 @m = $_->{$morph_hi} =~ m{\G
200 (?:[^\~\#\/\[\=\|\-\+\\\ ]+) ##-- morph: stem
201 |(?:\/[A-Z]{1,2}) ##-- morph: stem class
202 |(?:[\~\#\=\|\-\+\\\ ]+) ##-- morph: separator
203 |(?:\[.*$) ##-- morph: syntax (tag+features)
204 }gx;
205 $ma = pop @m;
206  
207 ##-- check for bad tags (unsafe)
208 next if (
209 $badTags->{$ma =~ /^\[_([A-Z0-9]+)\]/ ? $1 : $ma}
210 ||
211 $ma =~ m{
212 ^\[_NE\]\[
213 (?:
214 #geoname|
215 #firstname|
216 lastname|
217 orgname|
218 productname
219 )
220 \]
221 }x
222 );
223  
224 ##-- check for unsafe roots
225 foreach (@m) {
226 next MORPHA if ($badMorphs->{$_});
227 }
228  
229 ##-- check for suspicious composites, e.g. "Mittheilung:Mitte/N#heil/V~ung", "Abtheilung:Abt/N#heil/V~ung"
230 next if ($_->{$morph_hi} =~ m{te?\/[A-Z]{1,2}\#[Hh]});
231  
232 ##-- this analysis is safe: update flag & break out of morph-analysis loop
233 $safe=1;
234 last;
235 }
236 }
237  
238 ##-- output
239 $tok->{$label} = $safe ? 1 : 0;
240 }
241  
242 return $doc;
243 }
244  
245  
246 1; ##-- be happy
247  
248 __END__