CAB/Analyzer/LangId/Simple.pm untested

filter: Statementstmt 0.0% 0 / 79 Branchbran 0.0% 0 / 38 Conditioncond 0.0% 0 / 55 MC/DCmcdc 0.0% 0 / 33 Subroutinesub 0.0% 0 / 7 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 216 CC 20 SCAR 50.0
CC
20
Cov
0%
analyzeTypes
240.0
analyzeSentences
42.0
new
2.0
CRAP 148.5
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::LangId::Simple.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: language identification using stopword lists
6  
7 ##==============================================================================
8 package DTA::CAB::Analyzer::LangId::Simple;
9 use DTA::CAB::Analyzer::Dict::Json;
10 use Carp;
11 use strict;
12 our @ISA = qw(DTA::CAB::Analyzer::Dict::Json);
13  
14 ## $obj = CLASS_OR_OBJ->new(%args)
15 ## + object structure: see DTA::CAB::Analyzer::Dict::Json
16 sub new {
17 my $that = shift;
18 my $lid = $that->SUPER::new(
19 ##-- analysis selection
20 label => 'lang',
21 #slabel => 'lang', ##-- sentence-level label
22 #vlabel => 'lang_counts', ##-- DEBUG: verbose sentence-level counts, empty or undef for none
23 defaultLang => 'de',
24 defaultCount => 0.1, ##-- bonus count for default lang (characters)
25 minSentLen => 2, ##-- minimum number of tokens in sentence required before guessing
26 minSentChars => 8, ##-- minimum number of text characters in sentence required begore guessing
27  
28 ##-- user args
29 @_
30 );
31 return $lid;
32 }
33  
34 ##==============================================================================
35 ## Methods: Prepare
36  
37 ## $bool = $dic->ensureLoaded()
38 ## + ensures analyzer data is loaded from default files
39 sub ensureLoaded {
40 my $lid = shift;
41 return $lid->SUPER::ensureLoaded(@_) && $lid->decodeDictValues();
42 }
43  
44  
45 ##==============================================================================
46 ## Methods: Analysis
47  
48 ##------------------------------------------------------------------------
49 ## Methods: Analysis: v1.x: API
50  
51 ## $doc = $anl->analyzeTypes($doc,\%types,\%opts)
52 ## + perform type-wise analysis of all (text) types in $doc->{types}
53 sub analyzeTypes {
54 my ($lid,$doc,$types,$opts) = @_;
55  
56 ##-- common vars
57 my $label = $lid->{label} || $lid->defaultLabel;
58 my $slabel = $lid->{slabel} || $label;
59 my $swd = $lid->{ttd}{dict};
60 my $allow_re = defined($lid->{allowRegex}) ? qr($lid->{allowRegex}) : undef;
61 my $l0 = $lid->{defaultLang};
62 my (@l);
63  
64 ##-- word-wise analysis
65 my ($l,$prev);
66 foreach (values %$types) {
67 next if (defined($allow_re) && $_->{text} !~ $allow_re);
68  
69 ##-- list check
70 @l = (defined($l=$swd->{lc($_->{text})}) ? @$l : qw());
71  
72 ##-- local analysis check(s)
73 if (!$_->{xlit} || !$_->{xlit}{isLatinExt}) {
74 if ($_->{text} =~ /^\p{Greek}{2,}$/) { push(@l, 'el'); }
75 elsif ($_->{text} =~ /^\p{Hebrew}{2,}$/) { push(@l, 'he'); }
76 elsif ($_->{text} =~ /^\p{Arabic}{2,}$/) { push(@l, 'ar'); }
77 elsif ($_->{text} =~ /[[:alpha:]]{2,}/ && $_->{text} !~ /\p{Latin}/) { push(@l,'xy'); } ##-- combination of latin and non-latin characters
78 }
79 if ($_->{text} =~ /[\p{InMathematicalOperators}]/) { push(@l,'xy'); }
80 elsif ($_->{text} =~ /[[:alpha:]](?:.?)[[:digit:]]/ && $_->{text} !~ m{^[a-zA-Z]+://}) {
81 ##-- don't treat links as 'xy' specials
82 push(@l,'xy');
83 }
84  
85 ##-- latin: use {mlatin}, but don't count known NE; workaround for mantis bug #6737
86 push(@l, 'la') if ($_->{mlatin} && (!$_->{morph} || !grep {$_->{hi} =~ /\[_NE\]/} @{$_->{morph}}));
87  
88 ##-- default language: use {morph} and {msafe}, disregarding NE,FM
89 push(@l, $l0) if ($l0 && $_->{morph} && ($_->{msafe}//1) && grep {$_->{hi} !~ /\[_(?:FM|NE)\]/} @{$_->{morph}});
90  
91 ##-- exlex language (not really)
92 #push(@l, $l0, 'exlex') if (($_->{exlex} && $_->{exlex} ne $_->{text}));
93  
94 ##-- make unique
95 if (@l) {
96 $prev = '';
97 $_->{$label} = [map {$prev eq $_ ? qw() : ($prev=$_)} sort @l];
98 } else {
99 $_->{$label} = undef;
100 }
101 }
102  
103 return $doc;
104 }
105  
106  
107 ## $doc = $anl->analyzeSentences($doc,\%opts)
108 sub analyzeSentences {
109 my ($lid,$doc,$opts) = @_;
110  
111 ##-- common vars
112 my $label = $lid->{label} || $lid->defaultLabel;
113 my $slabel = $lid->{slabel} || $label;
114 my $vlabel = $lid->{vlabel};
115 my $l0 = $lid->{defaultLang} // '';
116 my $n0 = $l0 ? ($lid->{defaultCount}//0) : 0;
117 my $minlen = $lid->{minSentLen} // 0;
118 my $minchrs= $lid->{minSentChars} // 0;
119 my $nil = [];
120  
121 ##-- ye olde loope
122 my (%ln,$s,$nchrs,$l,$n,$w);
123 foreach $s (@{$doc->{body}}) {
124 ##-- check minimum sentence length in tokens
125 next if (@{$s->{tokens}} < $minlen);
126  
127 ##-- count number of stopword-CHARACTERS per language
128 %ln = ($l0=>$n0);
129 $nchrs = 0;
130 foreach $w (@{$s->{tokens}}) {
131 $nchrs += length($w->{text});
132 $ln{$_} += length($w->{text}) foreach (@{$w->{$label}//$nil});
133 }
134 next if ($nchrs < $minchrs);
135  
136 ##-- get top-ranked language for this sentence
137 ($l,$n) = ($l0,$n0);
138 foreach (sort keys %ln) {
139 ($l,$n)=($_,$ln{$_}) if ($n < $ln{$_});
140 }
141 $s->{$slabel} = $l;
142 $s->{$vlabel} = {%ln} if ($vlabel); ##-- DEBUG
143 }
144  
145 return $doc;
146 }
147  
148  
149 1; ##-- be happy
150  
151  
152 __END__