| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::MorphSafe.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: safety checker for analyses output by DTA::CAB::Analyzer::Morph (TAGH) | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::MorphSafe; | ||
| 8 | |||
| 9 | use DTA::CAB::Analyzer; | ||
| 10 | use DTA::CAB::Analyzer::Dict; | ||
| 11 | use DTA::CAB::Unify ':all'; | ||
| 12 | |||
| 13 | use Encode qw(encode decode); | ||
| 14 | use IO::File; | ||
| 15 | use Carp; | ||
| 16 | |||
| 17 | use strict; | ||
| 18 | |||
| 19 | ##============================================================================== | ||
| 20 | ## Globals | ||
| 21 | ##============================================================================== | ||
| 22 | |||
| 23 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 24 | |||
| 25 | ## %badTypes = ($text=>$isGood, ...) | ||
| 26 | ## + default hash of bad text types | ||
| 27 | our %badTypes = ( | ||
| 28 | #Andre=>0, ##-- bad: type: Andre[_NE][firstname][none][none][sg][nom_acc_dat] | ||
| 29 | ); | ||
| 30 | |||
| 31 | ## %badMorphs = ($taghMorph=>$isBad, ...) | ||
| 32 | ## + default hash of bad TAGH morphs | ||
| 33 | our %badMorphs = ( | ||
| 34 | #'Thür'=>1, ##-- bad: stem | ||
| 35 | #'/ON'=>1, ##-- bad: stem class: organization name | ||
| 36 | ); | ||
| 37 | |||
| 38 | ## %badTags = ($taghTag=>$isBad, ...) | ||
| 39 | ## + default hash of bad TAGH tags | ||
| 40 | our %badTags = ( | ||
| 41 | 'FM'=>1, ##-- bad: FM (e.g. That:That[_FM][en]) | ||
| 42 | 'XY'=>1, ##-- bad: XY | ||
| 43 | 'ITJ'=>1, ##-- bad: ITJ | ||
| 44 | ); | ||
| 45 | |||
| 46 | ##============================================================================== | ||
| 47 | ## Constructors etc. | ||
| 48 | ##============================================================================== | ||
| 49 | |||
| 50 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 51 | ## + object structure, new: | ||
| 52 | ## ##-- analysis selection | ||
| 53 | ## allowTokenizerAnalyses => $bool, ##-- if true, tokenizer-analyzed tokens (as determined by $tok->{toka}, $tok->{tokpp}) are "safe"; (default=true) | ||
| 54 | ## allowExlexAnalyses => $bool, ##-- if true, exlex-analyzed tokens (as determined by $tok->{exlex}) are "safe"; (default=false) | ||
| 55 | ## allowApostropheS => $bool, ##-- if true, "-'s" tokens with a morph analysis are "safe" (default=true) | ||
| 56 | ## tokMorphKey => $key, ##-- key for token 'morph' property (default='morph') | ||
| 57 | ## morphHiKey => $key, ##-- key for morph analysis 'hi' property (default='hi') | ||
| 58 | ## | ||
| 59 | ## ##-- Exception lexicon options | ||
| 60 | ## #dict => $dict, ##-- exception lexicon as a DTA::CAB::Analyzer::Dict object or option hash | ||
| 61 | ## # ## + default=undef | ||
| 62 | ## #dictClass => $class, ##-- fallback class for new dict (default='DTA::CAB::Analyzer::Dict') | ||
| 63 | ## | ||
| 64 | sub new { | ||
| 65 | my $that = shift; | ||
| 66 | return $that->SUPER::new( | ||
| 67 | ##-- options | ||
| 68 | label => 'msafe', | ||
| 69 | allowTokenizerAnalyses => 1, | ||
| 70 | allowExlexAnalyses => 0, | ||
| 71 | allowApostropheS => 1, | ||
| 72 | tokMorphKey => 'morph', | ||
| 73 | morphHiKey => 'hi', | ||
| 74 | |||
| 75 | ##-- dictionary stuff | ||
| 76 | #badTypesFile => undef, ##-- filename of ($text "\t" $isGoodBool) mapping for raw utf8 text types | ||
| 77 | #badTypes => {%badTypes}, ##-- hash of bad utf8 text types ($text=>$isGoodBool) | ||
| 78 | ## | ||
| 79 | #badMorphsFile => undef, ##-- filename of ($taghMorph "\t" $isBadBool) mapping for TAGH morph components | ||
| 80 | #badMorphs => {%badMorphs}, ##-- hash of bad TAGH morphs ($taghMorph=>$isBadBool) | ||
| 81 | ## | ||
| 82 | #badTagsFile => undef, ##-- filename of ($taghTag "\t" $isBadBool) mapping for TAGH tags (without '[_', ']') | ||
| 83 | #badTags => {%badTags}, ##-- hash of bad TAGH tags ($taghTag=>$isBadBool) | ||
| 84 | |||
| 85 | ##-- user args | ||
| 86 | @_ | ||
| 87 | ); | ||
| 88 | } | ||
| 89 | |||
| 90 | ##============================================================================== | ||
| 91 | ## Methods: I/O | ||
| 92 | ##============================================================================== | ||
| 93 | |||
| 94 | ## $bool = $msafe->ensureLoaded() | ||
| 95 | ## + ensures analysis data is loaded | ||
| 96 | sub ensureLoaded { | ||
| 97 | my $msafe = shift; | ||
| 98 | my $rc = 1; | ||
| 99 | |||
| 100 | ##-- ensure: dict: badTypes | ||
| 101 | $rc &&= $msafe->ensureDict('badTypes',\%badTypes) if (!$msafe->{badTypes}); | ||
| 102 | |||
| 103 | ##-- ensure: dict: badMorphs | ||
| 104 | $rc &&= $msafe->ensureDict('badMorphs',\%badMorphs) if (!$msafe->{badMorphs}); | ||
| 105 | |||
| 106 | ##-- ensure: dict: badTags | ||
| 107 | $rc &&= $msafe->ensureDict('badTags',\%badTags) if (!$msafe->{badTags}); | ||
| 108 | |||
| 109 | return $rc; | ||
| 110 | } | ||
| 111 | |||
| 112 | ##-------------------------------------------------------------- | ||
| 113 | ## Methods: I/O: Input: Dictionaries: generic | ||
| 114 | |||
| 115 | ## $bool = $msafe->ensureDict($dictName,\%dictDefault) | ||
| 116 | sub ensureDict { | ||
| 117 | my ($ms,$name,$default) = @_; | ||
| 118 | return 1 if ($ms->{$name}); ##-- already defined | ||
| 119 | return $ms->loadDict($name,$ms->{"${name}File"}) if ($ms->{"${name}File"}); | ||
| 120 | $ms->{$name} = $default ? {%$default} : {}; | ||
| 121 | return 1; | ||
| 122 | } | ||
| 123 | |||
| 124 | ## \%dictHash_or_undef = $msafe->loadDict($dictName,$dictFile) | ||
| 125 | sub loadDict { | ||
| 126 | my ($ms,$name,$dfile) = @_; | ||
| 127 | delete($ms->{$name}); | ||
| 128 | $ms->info("loading exception lexicon from '$dfile'"); | ||
| 129 | |||
| 130 | ##-- hack: generate a temporary dict object | ||
| 131 | my $dict = DTA::CAB::Analyzer::Dict->new(label=>($ms->{label}.".dict.$name"), dictFile=>$dfile); | ||
| 132 | $dict->ensureLoaded(); | ||
| 133 | return undef if (!$dict->dictOk); | ||
| 134 | |||
| 135 | ##-- clobber dict | ||
| 136 | $ms->{$name} = $dict->dictHash; | ||
| 137 | } | ||
| 138 | |||
| 139 | |||
| 140 | ##============================================================================== | ||
| 141 | ## Methods: Analysis: v1.x | ||
| 142 | ##============================================================================== | ||
| 143 | |||
| 144 | ## $doc = $msafe->analyzeTypes($doc,\%types,\%opts) | ||
| 145 | ## + perform type-wise analysis of all (text) types in %types (= %{$doc->{types}}) | ||
| 146 | ## + checks for "safe" analyses in $tok->{morph} for each $tok in $doc->{types} | ||
| 147 | ## + sets $tok->{ $anl->{label} } = $bool | ||
| 148 | sub analyzeTypes { | ||
| 149 | my ($ms,$doc,$types,$opts) = @_; | ||
| 150 | $types = $doc->types if (!$types); | ||
| 151 | #$types = $doc->extendTypes($types,'morph') if (!grep {$_->{morph}} values(%$types)); ##-- DEBUG | ||
| 152 | |||
| 153 | my $label = $ms->{label}; | ||
| 154 | my $want_toka = $ms->{allowTokenizerAnalyses}; | ||
| 155 | my $want_exlex= $ms->{allowExlexAnalyses}; | ||
| 156 | my $want_apos_s = $ms->{allowApostropheS}; | ||
| 157 | my $tok_morph = $ms->{tokMorphKey}; | ||
| 158 | my $morph_hi = $ms->{morphHiKey}; | ||
| 159 | my $badTypes = $ms->{badTypes}||{}; | ||
| 160 | my $badTags = $ms->{badTags}||{}; | ||
| 161 | my $badMorphs = $ms->{badMorphs}||{}; | ||
| 162 | |||
| 163 | my ($tok,$safe,$nsafe,@m,$ma); | ||
| 164 | foreach $tok (values %$types) { | ||
| 165 | next if (defined($tok->{$label})); ##-- avoid re-analysis (e.g. of global exlex-provided analyses) | ||
| 166 | |||
| 167 | ##-- no dict entry: use morph heuristics | ||
| 168 | $safe = | ||
| 169 | (($want_exlex && defined($tok->{exlex})) ##-- exception-lexicon analyses are considered "safe" | ||
| 170 | || ( | ||
| 171 | $want_toka | ||
| 172 | && ( | ||
| 173 | ($tok->{toka} && @{$tok->{toka}}) ##-- tokenizer-analyzed words are considered "safe" | ||
| 174 | || | ||
| 175 | ($tok->{tokpp} && @{$tok->{tokpp}}) | ||
| 176 | ) | ||
| 177 | ) | ||
| 178 | || ( | ||
| 179 | $tok->{text} !~ m/^[\p{Letter}\#\@]/ ##-- non-alphabetic tokens are (usually) "safe" | ||
| 180 | ## ... unless they contain a placeholder for unrecognized material ('#' or '@') | ||
| 181 | ## [replaces /[[:digit:][:punct:]]/ heuristic; Tue, 28 Feb 2012 11:21:29 +0100] | ||
| 182 | ) | ||
| 183 | || $tok->{mlatin} ##-- latin words are "safe" [NEW Fri, 01 Apr 2011 11:38:45 +0200] | ||
| 184 | || ( | ||
| 185 | $want_apos_s ##-- "-'s" words with a morph analysis may be safe | ||
| 186 | && $tok->{$tok_morph} | ||
| 187 | && $tok->{text} =~ m/^(.+)[\'\x{2018}\x{2019}]s$/ | ||
| 188 | ) | ||
| 189 | ); | ||
| 190 | |||
| 191 | ##-- are we still unsafe? then check for some "safe" morph analysis: if found, set $safe=1 & bug out | ||
| 192 | if (!$safe | ||
| 193 | && !defined($safe=$badTypes->{$tok->{text}}) ##-- ... only if it's not a known type | ||
| 194 | && $tok->{$tok_morph} ##-- ... and it has morph analyses (empty $tok->{morph} will still be "unsafe") | ||
| 195 | ) | ||
| 196 | { | ||
| 197 | MORPHA: | ||
| 198 | foreach (@{$tok->{$tok_morph}}) { | ||
| 199 | @m = $_->{$morph_hi} =~ m{\G | ||
| 200 | (?:[^\~\#\/\[\=\|\-\+\\\ ]+) ##-- morph: stem | ||
| 201 | |(?:\/[A-Z]{1,2}) ##-- morph: stem class | ||
| 202 | |(?:[\~\#\=\|\-\+\\\ ]+) ##-- morph: separator | ||
| 203 | |(?:\[.*$) ##-- morph: syntax (tag+features) | ||
| 204 | }gx; | ||
| 205 | $ma = pop @m; | ||
| 206 | |||
| 207 | ##-- check for bad tags (unsafe) | ||
| 208 | next if ( | ||
| 209 | $badTags->{$ma =~ /^\[_([A-Z0-9]+)\]/ ? $1 : $ma} | ||
| 210 | || | ||
| 211 | $ma =~ m{ | ||
| 212 | ^\[_NE\]\[ | ||
| 213 | (?: | ||
| 214 | #geoname| | ||
| 215 | #firstname| | ||
| 216 | lastname| | ||
| 217 | orgname| | ||
| 218 | productname | ||
| 219 | ) | ||
| 220 | \] | ||
| 221 | }x | ||
| 222 | ); | ||
| 223 | |||
| 224 | ##-- check for unsafe roots | ||
| 225 | foreach (@m) { | ||
| 226 | next MORPHA if ($badMorphs->{$_}); | ||
| 227 | } | ||
| 228 | |||
| 229 | ##-- check for suspicious composites, e.g. "Mittheilung:Mitte/N#heil/V~ung", "Abtheilung:Abt/N#heil/V~ung" | ||
| 230 | next if ($_->{$morph_hi} =~ m{te?\/[A-Z]{1,2}\#[Hh]}); | ||
| 231 | |||
| 232 | ##-- this analysis is safe: update flag & break out of morph-analysis loop | ||
| 233 | $safe=1; | ||
| 234 | last; | ||
| 235 | } | ||
| 236 | } | ||
| 237 | |||
| 238 | ##-- output | ||
| 239 | $tok->{$label} = $safe ? 1 : 0; | ||
| 240 | } | ||
| 241 | |||
| 242 | return $doc; | ||
| 243 | } | ||
| 244 | |||
| 245 | |||
| 246 | 1; ##-- be happy | ||
| 247 | |||
| 248 | __END__ |