| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Lemmatizer.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: lemma extractor for TAGH analyses or bare text | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Lemmatizer; | ||
| 8 | use DTA::CAB::Analyzer; | ||
| 9 | use Carp; | ||
| 10 | use strict; | ||
| 11 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 12 | |||
| 13 | ##============================================================================== | ||
| 14 | ## Globals | ||
| 15 | |||
| 16 | ## $GET_MORPH | ||
| 17 | ## + code string: \@morph_analyses = "$GET_MORPH"->() | ||
| 18 | ## + available vars: $tok, $lz | ||
| 19 | our $GET_MORPH = '$tok->{morph}'; | ||
| 20 | |||
| 21 | ## $GET_DMOOT_MORPH | ||
| 22 | ## + code string: \@morph_analyses = "$GET_DMOOT_MORPH"->() | ||
| 23 | ## + available vars: $tok, $lz | ||
| 24 | our $GET_DMOOT_MORPH = '$tok->{dmoot} ? $tok->{dmoot}{morph} : undef'; | ||
| 25 | |||
| 26 | ## $GET_TEXT | ||
| 27 | ## + code string: get text for analysis $_ | ||
| 28 | ## + available vars: $tok, $tokm (array of analyses), $ma (current analysis), $lz (analyzer obj), | ||
| 29 | our $GET_TEXT = '$tok->{xlit} ? $tok->{xlit}{latin1Text} : $tok->{text}'; | ||
| 30 | |||
| 31 | ## $GET_MOOT_ANALYSES | ||
| 32 | ## + code string: \@morph_analyses = "$GET_DMOOT_MORPH"->() | ||
| 33 | ## + available vars: $tok, $lz | ||
| 34 | our $GET_MOOT_ANALYSES = '$tok->{moot} ? $tok->{moot}{analyses} : undef'; | ||
| 35 | |||
| 36 | ## $GET_MOOT_TEXT | ||
| 37 | ## + code string: get text for analysis $_ | ||
| 38 | ## + available vars: $tok, $tokm (array of analyses), $ma (current analysis), $lz (analyzer obj), | ||
| 39 | our $GET_MOOT_TEXT = '$tok->{moot} ? $tok->{moot}{word} : ('.$GET_TEXT.')'; | ||
| 40 | |||
| 41 | ##============================================================================== | ||
| 42 | ## Methods | ||
| 43 | |||
| 44 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 45 | ## + object structure, %args: | ||
| 46 | ## analyzeGet => $code, ##-- pseudo-accessor: @morph_analyses = "$code"->(\@toks) | ||
| 47 | ## analyzeWhich => $which, ##-- e.g. 'Types','Tokens','Sentences','Local': default=Types | ||
| 48 | ## ## + the underlying analysis is always performed by the analyzeTypes() method! (default='Types') | ||
| 49 | ## analyzeLabel => $label, ##-- ouput label (default='lemma') | ||
| 50 | sub new { | ||
| 51 | my $that = shift; | ||
| 52 | my $lz = $that->SUPER::new( | ||
| 53 | ##-- analysis selection | ||
| 54 | label => 'lemma', | ||
| 55 | analyzeLabel => 'lemma', | ||
| 56 | analyzeGet => $GET_MORPH, | ||
| 57 | analyzeGetText => $GET_TEXT, | ||
| 58 | analyzeWhich => 'Types', | ||
| 59 | #typeKeys => undef, | ||
| 60 | |||
| 61 | ##-- user args | ||
| 62 | @_ | ||
| 63 | ); | ||
| 64 | return $lz; | ||
| 65 | } | ||
| 66 | |||
| 67 | ## @keys = $anl->typeKeys() | ||
| 68 | ## + returns list of type-wise keys to be expanded for this analyzer by expandTypes() | ||
| 69 | ## + override returns @{$lt->{typeKeys}} | ||
| 70 | sub typeKeys { | ||
| 71 | return @{$_[0]{typeKeys}} if ($_[0]{typeKeys} && @{$_[0]{typeKeys}}); | ||
| 72 | return qw(); | ||
| 73 | } | ||
| 74 | |||
| 75 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 76 | ## + override: only allow analyzeSentences() | ||
| 77 | sub doAnalyze { | ||
| 78 | my $anl = shift; | ||
| 79 | return 0 if (defined($_[1]) && $_[1] ne $anl->{analyzeWhich}); | ||
| 80 | return $anl->SUPER::doAnalyze(@_); | ||
| 81 | } | ||
| 82 | |||
| 83 | ## \@toks = $anl->_analyzeGuts(\@toks,\%opts) | ||
| 84 | ## + guts: analyze all tokens in \@toks | ||
| 85 | sub _analyzeGuts { | ||
| 86 | my ($lz,$toks,$opts) = @_; | ||
| 87 | |||
| 88 | ##-- common vars | ||
| 89 | my $lab = $lz->{label}; | ||
| 90 | my $alab = defined($lz->{analyzeLabel}) ? $lz->{analyzeLabel} : $lab; | ||
| 91 | my $lab_txt = $lab."_text"; | ||
| 92 | my $lab_key = $lab."_key"; | ||
| 93 | |||
| 94 | ##-- prepare map $key2a = { "$text\t$hi" => $analysis, ... } | ||
| 95 | my $key2a = {}; | ||
| 96 | my ($tok,$tokm,$ma,$txt,$key); | ||
| 97 | my $prep_code = | ||
| 98 | 'foreach $tok (@$toks) { | ||
| 99 | next if (!($tokm='.$lz->{analyzeGet}.')); | ||
| 100 | foreach (grep {defined($_)} @$tokm) { | ||
| 101 | $txt = $_->{$lab_txt} = '.$lz->{analyzeGetText}.'; | ||
| 102 | $key = $_->{$lab_key} = $txt."\t".(defined($_->{hi}) ? $_->{hi} : $_->{details}); | ||
| 103 | next if (exists($key2a->{$key})); | ||
| 104 | $key2a->{$key} = $_; | ||
| 105 | } | ||
| 106 | }'; | ||
| 107 | my $prep_sub = eval "sub { $prep_code }"; | ||
| 108 | $lz->logcluck("_analyzeGuts(): could not compile preprocessing sub {$prep_code}: $@") if (!$prep_sub); | ||
| 109 | $prep_sub->(); | ||
| 110 | |||
| 111 | ##-- lemmatize, type-wise by (text+analysis)-pair | ||
| 112 | my ($lemma,$tag,$lemmaFromMorph); | ||
| 113 | foreach (values %$key2a) { | ||
| 114 | $lemma = defined($_->{hi}) ? $_->{hi} : $_->{details}; | ||
| 115 | if (defined($lemma) && $lemma ne '' && $lemma =~ /^[^\]]+\[/) { ##-- tagh analysis (vs. tokenizer-supplied analysis) | ||
| 116 | #$lemma =~ s/\~e?t(?=\W)/\~en/g; ##-- rename verb inflection morphs [BUG: l("Christ/N\en~tum[_NN]") = Christenenum] | ||
| 117 | $lemma =~ s/\[<([^\>\]]*)>\]/<$1>/g; ##-- unquote taghm-2.5 "diamond-tags", e.g "kurz<A>\@Stiel<N>~ig[_ADJD]<none>" | ||
| 118 | $lemma =~ s/\[.*$//; ##-- trim everything after first non-character symbol | ||
| 119 | #$lemma =~ s/(?:\/[A-Za-z]{1,2})|(?:\bge\\\|)|(?:[\\\~\|\=\+\#\x{ac}])//g; ##-- hack: remove "ge\|" prefixes too (but not e.g. "ver\|", "be\|", etc.) | ||
| 120 | $lemma =~ s/(?:\/[A-Za-z]{1,2})|(?:\[?<[^>]+>\]?)|(?:[\@\\\~\|\=\+\#\x{ac}])//g; ##-- unhack: don't remove "ge\|" prefixes (for consistency e.g. with dwds-kc20) | ||
| 121 | $lemmaFromMorph = 1; | ||
| 122 | } else { | ||
| 123 | $lemma = $_->{$lab_txt}; | ||
| 124 | $lemma =~ s/\x{ac}//g; | ||
| 125 | $lemmaFromMorph = 0; | ||
| 126 | } | ||
| 127 | ##-- extract tag if available | ||
| 128 | $tag = $_->{tag} || ($_->{hi} && $_->{hi} =~ m/\[_([^\]]+)\]/ ? $1 : ''); | ||
| 129 | ## | ||
| 130 | ##-- normalization | ||
| 131 | $lemma =~ s/(?:^\s+|\s+\z)//g; | ||
| 132 | $lemma =~ s/\s+/_/g; | ||
| 133 | if ($lemmaFromMorph && $tag =~ /^(?:NE|XY)$/) { | ||
| 134 | ##-- retain morphology-supplied original case for names and symbols | ||
| 135 | ; | ||
| 136 | } | ||
| 137 | elsif ($_->{tag} ne 'XY') { | ||
| 138 | ##-- lower-case all lemmata here, otherwise Anschlußstelle->AnschlußStelle (mantis #23127) | ||
| 139 | $lemma = lc($lemma); | ||
| 140 | $lemma =~ s/(?:^|(?<=[\-\_]))(.)/\U$1\E/g | ||
| 141 | if (#$_->{tag} ? ($_->{tag}=~m/^N/) : ##-- disabled 2016-06-01 for Helsinki-style english morphology; see ucTags key in Analyzer::MootSub | ||
| 142 | ($_->{hi} && $_->{hi}=~m/\[_N/) | ||
| 143 | ); | ||
| 144 | } | ||
| 145 | $_->{$alab} = $lemma; | ||
| 146 | } | ||
| 147 | |||
| 148 | ##-- postprocessing: re-expand types | ||
| 149 | my $postp_code = | ||
| 150 | 'foreach $tok (@$toks) { | ||
| 151 | next if (!($tokm='.$lz->{analyzeGet}.')); | ||
| 152 | foreach (grep {defined($_)} @$tokm) { | ||
| 153 | $_->{$alab} = $key2a->{$_->{$lab_key}}{$alab}; | ||
| 154 | delete(@$_{$lab_key,$lab_txt}); | ||
| 155 | } | ||
| 156 | }'; | ||
| 157 | my $postp_sub = eval "sub { $postp_code }"; | ||
| 158 | $lz->logcluck("_analyzeGuts(): could not compile postprocessing sub {$postp_code}: $@") if (!$postp_sub); | ||
| 159 | $postp_sub->(); | ||
| 160 | |||
| 161 | return $toks; | ||
| 162 | } | ||
| 163 | |||
| 164 | |||
| 165 | ## $doc = $anl->analyzeTypes($doc,\%types,\%opts) | ||
| 166 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 167 | sub analyzeTypes { | ||
| 168 | my ($lz,$doc,$types,$opts) = @_; | ||
| 169 | return $doc if ($lz->{analyzeWhich} ne 'Types'); | ||
| 170 | $lz->_analyzeGuts([values %$types],$opts); | ||
| 171 | return $doc; | ||
| 172 | } | ||
| 173 | |||
| 174 | ## $doc = $anl->analyzeOther($which, $doc,\%opts) | ||
| 175 | ## + analyze all tokens in $doc | ||
| 176 | sub analyzeOther { | ||
| 177 | my ($anl,$which,$doc,$opts) = @_; | ||
| 178 | return $doc if (defined($which) && $which ne $anl->{analyzeWhich}); | ||
| 179 | $anl->_analyzeGuts([map {@{$_->{tokens}}} @{$doc->{body}}],$opts); | ||
| 180 | return $doc; | ||
| 181 | } | ||
| 182 | |||
| 183 | sub analyzeTokens { return $_[0]->analyzeOther('Tokens',@_[1..$#_]); } | ||
| 184 | sub analyzeSentences { return $_[0]->analyzeOther('Sentences',@_[1..$#_]); } | ||
| 185 | sub analyzeLocal { return $_[0]->analyzeOther('Local',@_[1..$#_]); } | ||
| 186 | sub analyzeClean { return $_[0]->analyzeOther('Clean',@_[1..$#_]); } | ||
| 187 | |||
| 188 | |||
| 189 | |||
| 190 | 1; ##-- be happy | ||
| 191 | |||
| 192 | __END__ |