| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## File: DTA::CAB::Chain::DE_free.pm | ||
| 3 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 4 | ## Description: robust analysis: default chain (German, using only redistributable resources) | ||
| 5 | |||
| 6 | package DTA::CAB::Chain::DE_free; | ||
| 7 | use DTA::CAB::Datum ':all'; | ||
| 8 | use DTA::CAB::Chain::Multi; | ||
| 9 | |||
| 10 | ##-- sub-analyzers | ||
| 11 | use DTA::CAB::Analyzer::TokPP; | ||
| 12 | use DTA::CAB::Analyzer::Morph::Helsinki::DE; | ||
| 13 | use DTA::CAB::Analyzer::Morph::Latin; | ||
| 14 | use DTA::CAB::Analyzer::MorphSafe; | ||
| 15 | use DTA::CAB::Analyzer::Moot; | ||
| 16 | use DTA::CAB::Analyzer::MootSub; | ||
| 17 | use DTA::CAB::Analyzer::LangId::Simple; | ||
| 18 | use DTA::CAB::Analyzer::DTAClean; | ||
| 19 | use DTA::CAB::Analyzer::Null; | ||
| 20 | |||
| 21 | use IO::File; | ||
| 22 | use Carp; | ||
| 23 | |||
| 24 | use strict; | ||
| 25 | |||
| 26 | ##============================================================================== | ||
| 27 | ## Constants | ||
| 28 | ##============================================================================== | ||
| 29 | our @ISA = qw(DTA::CAB::Chain::Multi); | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Constructors etc. | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 36 | ## + object structure: HASH | ||
| 37 | sub new { | ||
| 38 | my $that = shift; | ||
| 39 | return $that->SUPER::new | ||
| 40 | ( | ||
| 41 | ##-- analyzers | ||
| 42 | tokpp => DTA::CAB::Analyzer::TokPP->new(), | ||
| 43 | xlit => DTA::CAB::Analyzer::Unicruft->new(), | ||
| 44 | ## | ||
| 45 | morph => DTA::CAB::Analyzer::Morph::Helsinki::DE->new(), | ||
| 46 | mlatin=> DTA::CAB::Analyzer::Morph::Latin->new(), | ||
| 47 | msafe => DTA::CAB::Analyzer::MorphSafe->new(), ##-- remove this for DE_free chain? | ||
| 48 | ## | ||
| 49 | moot => DTA::CAB::Analyzer::Moot->new(lang=>'de'), ##-- moot tagger (on dmoot output; (n>1)-grams) | ||
| 50 | moot1 => DTA::CAB::Analyzer::Moot->new(lang=>'de'), ##-- moot tagger (on dmoot output; 1-grams only) | ||
| 51 | mootsub => DTA::CAB::Analyzer::MootSub->new(ucTags=>[qw(NN NE)],stts=>1,wMorph=>.2), ##-- moot tagger, post-processing hacks | ||
| 52 | ## | ||
| 53 | langid => DTA::CAB::Analyzer::LangId::Simple->new(defaultLang=>'de'), ##-- language-guesser (stopword-based; between msafe and rw) | ||
| 54 | ## | ||
| 55 | clean => DTA::CAB::Analyzer::DTAClean->new(), | ||
| 56 | ## | ||
| 57 | null => DTA::CAB::Analyzer::Null->new(), ##-- null analyzer (for 'null' chain) | ||
| 58 | |||
| 59 | ##-- security | ||
| 60 | autoClean => 0, ##-- always run 'clean' analyzer regardless of options; checked in both doAnalyze(), analyzeClean() | ||
| 61 | defaultChain => 'default', | ||
| 62 | |||
| 63 | ##-- user args | ||
| 64 | @_, | ||
| 65 | |||
| 66 | ##-- overrides | ||
| 67 | chains => undef, ##-- see setupChains() method | ||
| 68 | chain => undef, ##-- see setupChains() method | ||
| 69 | ); | ||
| 70 | } | ||
| 71 | |||
| 72 | ##============================================================================== | ||
| 73 | ## Methods: Chain selection | ||
| 74 | ##============================================================================== | ||
| 75 | |||
| 76 | ## $ach = $ach->setupChains() | ||
| 77 | ## + setup default named sub-chains in $ach->{chains} | ||
| 78 | ## + override | ||
| 79 | ## + adapted from Chain::DTA | ||
| 80 | sub setupChains { | ||
| 81 | my $ach = shift; | ||
| 82 | my @akeys = grep {UNIVERSAL::isa($ach->{$_},'DTA::CAB::Analyzer')} keys(%$ach); | ||
| 83 | my $chains = $ach->{chains} = | ||
| 84 | { | ||
| 85 | (map {("sub.$_"=>[$ach->{$_}])} @akeys), ##-- sub.xlit, sub.lts, ... | ||
| 86 | #(map {("$_"=>[$ach->{$_}])} @akeys), ##-- xlit, lts, ... | ||
| 87 | ## | ||
| 88 | 'sub.sent' =>[@$ach{qw(moot mootsub)}], | ||
| 89 | 'sub.sent1' =>[@$ach{qw(moot1 mootsub)}], | ||
| 90 | ## | ||
| 91 | 'default.tokpp' =>[@$ach{qw(tokpp)}], | ||
| 92 | 'default.xlit' =>[@$ach{qw(xlit)}], | ||
| 93 | 'default.morph' =>[@$ach{qw(tokpp xlit morph)}], | ||
| 94 | 'default.mlatin' =>[@$ach{qw(tokpp xlit mlatin)}], | ||
| 95 | 'default.msafe' =>[@$ach{qw(tokpp xlit morph mlatin msafe)}], | ||
| 96 | 'default.langid' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid)}], | ||
| 97 | 'default.langid' =>[@$ach{qw(tokpp xlit morph mlatin langid)}], | ||
| 98 | 'default.moot' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot)}], | ||
| 99 | 'default.moot1' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot1)}], | ||
| 100 | 'default.lemma' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot mootsub)}], | ||
| 101 | 'default.lemma1' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot1 mootsub)}], | ||
| 102 | 'default.base' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid)}], | ||
| 103 | 'default.type' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid)}], | ||
| 104 | ## | ||
| 105 | 'norm' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot mootsub)}], | ||
| 106 | 'norm1' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot1 mootsub)}], | ||
| 107 | 'all' =>[@$ach{qw(tokpp xlit morph mlatin msafe langid moot mootsub)}], ##-- old dta clients use 'all'! | ||
| 108 | 'clean' =>[@$ach{qw(clean)}], | ||
| 109 | 'null' =>[$ach->{null}], | ||
| 110 | }; | ||
| 111 | #$chains->{'default'} = [map {@{$chains->{$_}}} qw(default.type sub.sent)]; | ||
| 112 | |||
| 113 | ##-- chain aliases | ||
| 114 | $chains->{'default'} = $chains->{lemma} = $chains->{'norm'}; | ||
| 115 | $chains->{'default1'} = $chains->{lemma1} = $chains->{'norm1'}; | ||
| 116 | |||
| 117 | ##-- sanitize chains | ||
| 118 | foreach (values %{$ach->{chains}}) { | ||
| 119 | @$_ = grep {ref($_)} @$_; | ||
| 120 | } | ||
| 121 | |||
| 122 | ##-- set default chain | ||
| 123 | $ach->{chain} = $ach->{chains}{$ach->{defaultChain}}; | ||
| 124 | |||
| 125 | ##-- force default labels | ||
| 126 | foreach (grep {UNIVERSAL::isa($ach->{$_},'DTA::CAB::Analyzer')} keys(%$ach)) { | ||
| 127 | next if ($_ =~ /^(?:langid)$/); ##-- keep these labels | ||
| 128 | ($ach->{$_}{label} = $_) =~ s/1$//; ##-- truncate '1' suffix for label (e.g. dmoot1, moot1) | ||
| 129 | } | ||
| 130 | return $ach; | ||
| 131 | } | ||
| 132 | |||
| 133 | ## \@analyzers = $ach->chain() | ||
| 134 | ## \@analyzers = $ach->chain(\%opts) | ||
| 135 | ## + get selected analyzer chain | ||
| 136 | ## + inherited from DTA::CAB::Chain::Multi | ||
| 137 | ## - calls setupChains() if $ach->{chain} is empty | ||
| 138 | ## - checks for $opts{chain} and returns $ach->{chains}{ $opts{chain} } if available | ||
| 139 | |||
| 140 | ## $ach = $ach->ensureChain() | ||
| 141 | ## + checks for $ach->{chain}, calls $ach->setupChains() if needed | ||
| 142 | ## + inherited from DTA::CAB::Chain::Multi | ||
| 143 | |||
| 144 | ##============================================================================== | ||
| 145 | ## Methods: I/O | ||
| 146 | ##============================================================================== | ||
| 147 | |||
| 148 | |||
| 149 | ##============================================================================== | ||
| 150 | ## Methods: Persistence | ||
| 151 | ##============================================================================== | ||
| 152 | |||
| 153 | ##====================================================================== | ||
| 154 | ## Methods: Persistence: Perl | ||
| 155 | |||
| 156 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 157 | ## + returns list of keys not to be saved | ||
| 158 | ## + default just greps for CODE-refs | ||
| 159 | ## + inherited from DTA::CAB::Chain::Multi: override appends {chain},{chains} | ||
| 160 | |||
| 161 | ## $saveRef = $obj->savePerlRef() | ||
| 162 | ## + return reference to be saved (top-level objects only) | ||
| 163 | ## + inherited from DTA::CAB::Persistent | ||
| 164 | |||
| 165 | ## $loadedObj = $CLASS_OR_OBJ->loadPerlRef($ref) | ||
| 166 | ## + default implementation just clobbers $CLASS_OR_OBJ with $ref and blesses | ||
| 167 | ## + inherited from DTA::CAB::Persistent | ||
| 168 | |||
| 169 | ##============================================================================== | ||
| 170 | ## Methods: Analysis | ||
| 171 | ##============================================================================== | ||
| 172 | |||
| 173 | ##------------------------------------------------------------------------ | ||
| 174 | ## Methods: Analysis: v1.x: Utils | ||
| 175 | |||
| 176 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 177 | ## + alias for $anl->can("analyze${name}") && (!exists($opts{"doAnalyze${name}"}) || $opts{"doAnalyze${name}"}) | ||
| 178 | ## + override checks $anl->{autoClean} flag | ||
| 179 | sub doAnalyze { | ||
| 180 | my ($anl,$opts,$name) = @_; | ||
| 181 | return 1 if ($anl->{autoClean} && $name eq 'Clean'); | ||
| 182 | return $anl->SUPER::doAnalyze($opts,$name); | ||
| 183 | } | ||
| 184 | |||
| 185 | |||
| 186 | ##------------------------------------------------------------------------ | ||
| 187 | ## Methods: Analysis: v1.x: API | ||
| 188 | |||
| 189 | ## $doc = $ach->analyzeDocument($doc,\%opts) | ||
| 190 | ## + analyze a DTA::CAB::Document $doc | ||
| 191 | ## + top-level API routine | ||
| 192 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 193 | |||
| 194 | ## $doc = $ach->analyzeTypes($doc,$types,\%opts) | ||
| 195 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 196 | ## + Chain default calls $a->analyzeTypes for each analyzer $a in the chain | ||
| 197 | ## + INHERITED from DTA::CAB::Chain | ||
| 198 | |||
| 199 | ## $doc = $ach->analyzeTokens($doc,\%opts) | ||
| 200 | ## + perform token-wise analysis of all tokens $doc->{body}[$si]{tokens}[$wi] | ||
| 201 | ## + default implementation just shallow copies tokens in $doc->{types} | ||
| 202 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 203 | |||
| 204 | ## $doc = $ach->analyzeSentences($doc,\%opts) | ||
| 205 | ## + perform sentence-wise analysis of all sentences $doc->{body}[$si] | ||
| 206 | ## + Chain default calls $a->analyzeSentences for each analyzer $a in the chain | ||
| 207 | ## + INHERITED from DTA::CAB::Chain | ||
| 208 | |||
| 209 | ## $doc = $ach->analyzeLocal($doc,\%opts) | ||
| 210 | ## + perform local document-level analysis of $doc | ||
| 211 | ## + Chain default calls $a->analyzeLocal for each analyzer $a in the chain | ||
| 212 | ## + INHERITED from DTA::CAB::Chain | ||
| 213 | |||
| 214 | ## $doc = $ach->analyzeClean($doc,\%opts) | ||
| 215 | ## + cleanup any temporary data associated with $doc | ||
| 216 | ## + Chain default calls $a->analyzeClean for each analyzer $a in the chain, | ||
| 217 | ## then superclass Analyzer->analyzeClean | ||
| 218 | sub analyzeClean { | ||
| 219 | my ($ach,$doc,$opts) = @_; | ||
| 220 | $ach->SUPER::analyzeClean($doc,$opts); ##-- inherited from DTA::CAB::Chain (chain-local cleanup) | ||
| 221 | $ach->analyzeClean_rm_undef($doc,$opts); ##-- remove keys with undef values from tokens | ||
| 222 | return $doc if (!$ach->{autoClean} && !exists($opts->{doAnalyzeClean})); ##-- don't "clean" by default | ||
| 223 | return $ach->{clean}->analyzeClean($doc,$opts); | ||
| 224 | } | ||
| 225 | |||
| 226 | |||
| 227 | ##------------------------------------------------------------------------ | ||
| 228 | ## Methods: Analysis: v1.x: Wrappers | ||
| 229 | |||
| 230 | ## $tok = $ach->analyzeToken($tok_or_string,\%opts) | ||
| 231 | ## + perform type- and token-analyses on $tok_or_string | ||
| 232 | ## + wrapper for $ach->analyzeDocument() | ||
| 233 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 234 | |||
| 235 | ## $tok = $ach->analyzeSentence($sent_or_array,\%opts) | ||
| 236 | ## + perform type-, token-, and sentence-analyses on $sent_or_array | ||
| 237 | ## + wrapper for $ach->analyzeDocument() | ||
| 238 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 239 | |||
| 240 | ## $rpc_xml_base64 = $anl->analyzeData($data_str,\%opts) | ||
| 241 | ## + analyze a raw (formatted) data string $data_str with internal parsing & formatting | ||
| 242 | ## + wrapper for $anl->analyzeDocument() | ||
| 243 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 244 | |||
| 245 | ##============================================================================== | ||
| 246 | ## Methods: XML-RPC | ||
| 247 | ## + INHERITED from DTA::CAB::Chain::Multi | ||
| 248 | |||
| 249 | 1; ##-- be happy | ||
| 250 | |||
| 251 | __END__ |