| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::LangId::Simple.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: language identification using stopword lists | ||
| 6 | |||
| 7 | ##============================================================================== | ||
| 8 | package DTA::CAB::Analyzer::LangId::Simple; | ||
| 9 | use DTA::CAB::Analyzer::Dict::Json; | ||
| 10 | use Carp; | ||
| 11 | use strict; | ||
| 12 | our @ISA = qw(DTA::CAB::Analyzer::Dict::Json); | ||
| 13 | |||
| 14 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 15 | ## + object structure: see DTA::CAB::Analyzer::Dict::Json | ||
| 16 | sub new { | ||
| 17 | my $that = shift; | ||
| 18 | my $lid = $that->SUPER::new( | ||
| 19 | ##-- analysis selection | ||
| 20 | label => 'lang', | ||
| 21 | #slabel => 'lang', ##-- sentence-level label | ||
| 22 | #vlabel => 'lang_counts', ##-- DEBUG: verbose sentence-level counts, empty or undef for none | ||
| 23 | defaultLang => 'de', | ||
| 24 | defaultCount => 0.1, ##-- bonus count for default lang (characters) | ||
| 25 | minSentLen => 2, ##-- minimum number of tokens in sentence required before guessing | ||
| 26 | minSentChars => 8, ##-- minimum number of text characters in sentence required begore guessing | ||
| 27 | |||
| 28 | ##-- user args | ||
| 29 | @_ | ||
| 30 | ); | ||
| 31 | return $lid; | ||
| 32 | } | ||
| 33 | |||
| 34 | ##============================================================================== | ||
| 35 | ## Methods: Prepare | ||
| 36 | |||
| 37 | ## $bool = $dic->ensureLoaded() | ||
| 38 | ## + ensures analyzer data is loaded from default files | ||
| 39 | sub ensureLoaded { | ||
| 40 | my $lid = shift; | ||
| 41 | return $lid->SUPER::ensureLoaded(@_) && $lid->decodeDictValues(); | ||
| 42 | } | ||
| 43 | |||
| 44 | |||
| 45 | ##============================================================================== | ||
| 46 | ## Methods: Analysis | ||
| 47 | |||
| 48 | ##------------------------------------------------------------------------ | ||
| 49 | ## Methods: Analysis: v1.x: API | ||
| 50 | |||
| 51 | ## $doc = $anl->analyzeTypes($doc,\%types,\%opts) | ||
| 52 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 53 | sub analyzeTypes { | ||
| 54 | my ($lid,$doc,$types,$opts) = @_; | ||
| 55 | |||
| 56 | ##-- common vars | ||
| 57 | my $label = $lid->{label} || $lid->defaultLabel; | ||
| 58 | my $slabel = $lid->{slabel} || $label; | ||
| 59 | my $swd = $lid->{ttd}{dict}; | ||
| 60 | my $allow_re = defined($lid->{allowRegex}) ? qr($lid->{allowRegex}) : undef; | ||
| 61 | my $l0 = $lid->{defaultLang}; | ||
| 62 | my (@l); | ||
| 63 | |||
| 64 | ##-- word-wise analysis | ||
| 65 | my ($l,$prev); | ||
| 66 | foreach (values %$types) { | ||
| 67 | next if (defined($allow_re) && $_->{text} !~ $allow_re); | ||
| 68 | |||
| 69 | ##-- list check | ||
| 70 | @l = (defined($l=$swd->{lc($_->{text})}) ? @$l : qw()); | ||
| 71 | |||
| 72 | ##-- local analysis check(s) | ||
| 73 | if (!$_->{xlit} || !$_->{xlit}{isLatinExt}) { | ||
| 74 | if ($_->{text} =~ /^\p{Greek}{2,}$/) { push(@l, 'el'); } | ||
| 75 | elsif ($_->{text} =~ /^\p{Hebrew}{2,}$/) { push(@l, 'he'); } | ||
| 76 | elsif ($_->{text} =~ /^\p{Arabic}{2,}$/) { push(@l, 'ar'); } | ||
| 77 | elsif ($_->{text} =~ /[[:alpha:]]{2,}/ && $_->{text} !~ /\p{Latin}/) { push(@l,'xy'); } ##-- combination of latin and non-latin characters | ||
| 78 | } | ||
| 79 | if ($_->{text} =~ /[\p{InMathematicalOperators}]/) { push(@l,'xy'); } | ||
| 80 | elsif ($_->{text} =~ /[[:alpha:]](?:.?)[[:digit:]]/ && $_->{text} !~ m{^[a-zA-Z]+://}) { | ||
| 81 | ##-- don't treat links as 'xy' specials | ||
| 82 | push(@l,'xy'); | ||
| 83 | } | ||
| 84 | |||
| 85 | ##-- latin: use {mlatin}, but don't count known NE; workaround for mantis bug #6737 | ||
| 86 | push(@l, 'la') if ($_->{mlatin} && (!$_->{morph} || !grep {$_->{hi} =~ /\[_NE\]/} @{$_->{morph}})); | ||
| 87 | |||
| 88 | ##-- default language: use {morph} and {msafe}, disregarding NE,FM | ||
| 89 | push(@l, $l0) if ($l0 && $_->{morph} && ($_->{msafe}//1) && grep {$_->{hi} !~ /\[_(?:FM|NE)\]/} @{$_->{morph}}); | ||
| 90 | |||
| 91 | ##-- exlex language (not really) | ||
| 92 | #push(@l, $l0, 'exlex') if (($_->{exlex} && $_->{exlex} ne $_->{text})); | ||
| 93 | |||
| 94 | ##-- make unique | ||
| 95 | if (@l) { | ||
| 96 | $prev = ''; | ||
| 97 | $_->{$label} = [map {$prev eq $_ ? qw() : ($prev=$_)} sort @l]; | ||
| 98 | } else { | ||
| 99 | $_->{$label} = undef; | ||
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | return $doc; | ||
| 104 | } | ||
| 105 | |||
| 106 | |||
| 107 | ## $doc = $anl->analyzeSentences($doc,\%opts) | ||
| 108 | sub analyzeSentences { | ||
| 109 | my ($lid,$doc,$opts) = @_; | ||
| 110 | |||
| 111 | ##-- common vars | ||
| 112 | my $label = $lid->{label} || $lid->defaultLabel; | ||
| 113 | my $slabel = $lid->{slabel} || $label; | ||
| 114 | my $vlabel = $lid->{vlabel}; | ||
| 115 | my $l0 = $lid->{defaultLang} // ''; | ||
| 116 | my $n0 = $l0 ? ($lid->{defaultCount}//0) : 0; | ||
| 117 | my $minlen = $lid->{minSentLen} // 0; | ||
| 118 | my $minchrs= $lid->{minSentChars} // 0; | ||
| 119 | my $nil = []; | ||
| 120 | |||
| 121 | ##-- ye olde loope | ||
| 122 | my (%ln,$s,$nchrs,$l,$n,$w); | ||
| 123 | foreach $s (@{$doc->{body}}) { | ||
| 124 | ##-- check minimum sentence length in tokens | ||
| 125 | next if (@{$s->{tokens}} < $minlen); | ||
| 126 | |||
| 127 | ##-- count number of stopword-CHARACTERS per language | ||
| 128 | %ln = ($l0=>$n0); | ||
| 129 | $nchrs = 0; | ||
| 130 | foreach $w (@{$s->{tokens}}) { | ||
| 131 | $nchrs += length($w->{text}); | ||
| 132 | $ln{$_} += length($w->{text}) foreach (@{$w->{$label}//$nil}); | ||
| 133 | } | ||
| 134 | next if ($nchrs < $minchrs); | ||
| 135 | |||
| 136 | ##-- get top-ranked language for this sentence | ||
| 137 | ($l,$n) = ($l0,$n0); | ||
| 138 | foreach (sort keys %ln) { | ||
| 139 | ($l,$n)=($_,$ln{$_}) if ($n < $ln{$_}); | ||
| 140 | } | ||
| 141 | $s->{$slabel} = $l; | ||
| 142 | $s->{$vlabel} = {%ln} if ($vlabel); ##-- DEBUG | ||
| 143 | } | ||
| 144 | |||
| 145 | return $doc; | ||
| 146 | } | ||
| 147 | |||
| 148 | |||
| 149 | 1; ##-- be happy | ||
| 150 | |||
| 151 | |||
| 152 | __END__ |