CAB/Analyzer/Unidecode.pm untested

filter: Statementstmt 0.0% 0 / 56 Branchbran 0.0% 0 / 10 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 12 Podpod 0.0% 0 / 3 totaltotal 0.0% 0 / 81 CC 6 SCAR 27.8
CC
6
Cov
0%
analyzeTypes
42.0
new
2.0
ensureLoaded
2.0
CRAP 16.1
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Analyzer::Unidecode.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: latin-1 approximator (old)
6  
7 package DTA::CAB::Analyzer::Unidecode;
8  
9 use DTA::CAB::Analyzer;
10 use DTA::CAB::Datum ':all';
11 use DTA::CAB::Token;
12  
13 use Unicode::Normalize; ##-- compatibility decomposition 'KD' (see Unicode TR #15)
14 use Text::Unidecode; ##-- last-ditch effort: transliterate to ASCII
15 #use Unicode::UCD; ##-- unicode character names, info, etc.
16 #use Unicode::CharName; ##-- ... faster access to character name, block
17  
18 use Encode qw(encode decode);
19 use IO::File;
20 use Carp;
21  
22 use strict;
23  
24 ##==============================================================================
25 ## Globals
26 ##==============================================================================
27  
28 our @ISA = qw(DTA::CAB::Analyzer);
29  
30 ##==============================================================================
31 ## Constructors etc.
32 ##==============================================================================
33  
34 ## $obj = CLASS_OR_OBJ->new(%args)
35 ## + object structure, new:
36 ## label => $key, ##-- token analysis key (default='xlit')
37 sub new {
38 my $that = shift;
39 return $that->SUPER::new(
40 ##-- options
41 label => 'xlit',
42  
43 ##-- user args
44 @_
45 );
46 }
47  
48 ##==============================================================================
49 ## Methods: I/O
50 ##==============================================================================
51  
52 ## $bool = $aut->ensureLoaded()
53 ## + ensures analysis data is loaded
54 sub ensureLoaded { return 1; }
55  
56 ##==============================================================================
57 ## Methods: Analysis: v1.x
58 ##==============================================================================
59  
60 ## $doc = $xlit->analyzeTypes($doc,\%types,\%opts)
61 ## + perform type-wise analysis of all (text) types in values(%types)
62 ## + sets
63 ## $tok->{$anl->{label}} = { latin1Text=>$latin1Text, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt }
64 ## with:
65 ## $latin1Text = $str ##-- best latin-1 approximation of $token->{text}
66 ## $isLatin1 = $bool ##-- true iff $token->{text} is losslessly encodable as latin1
67 ## $isLatinExt = $bool, ##-- true iff $token->{text} is losslessly encodable as latin-extended
68 sub analyzeTypes {
69 my ($xlit,$doc,$types,$opts) = @_;
70 $types = $doc->types if (!$types);
71 my $akey = $xlit->{label};
72  
73 my ($tok, $w,$uc, $ld,$l0,$l, $isLatin1,$isLatinExt);
74 foreach $tok (values(%$types)) {
75 $w = $tok->{text};
76 $uc = Unicode::Normalize::NFKC($w); ##-- compatibility(?) decomposition + canonical composition
77  
78 ##-- construct latin-1 approximation
79 if (
80 #$uc =~ m([^\p{inBasicLatin}\p{inLatin1Supplement}]) #)
81 $uc =~ m([^\x{00}-\x{ff}]) #)
82 )
83 {
84 $l0 = $uc;
85  
86 ##-- special handling for some character sequences
87 $l0 =~ s/\x{0363}/a/g; ##-- COMBINING LATIN SMALL LETTER A
88 $l0 =~ s/\x{0364}/e/g; ##-- COMBINING LATIN SMALL LETTER E
89 $l0 =~ s/\x{0365}/i/g; ##-- COMBINING LATIN SMALL LETTER I
90 $l0 =~ s/\x{0366}/o/g; ##-- COMBINING LATIN SMALL LETTER O
91  
92 ##-- default: copy plain latin-1 characters, transliterate rest with Text::Unidecode::unidecode()
93 $l = join('',
94 map {
95 (
96 #$_ =~ m(\p{inBasicLatin}|\p{InLatin1Supplement}) #)
97 $_ =~ m([\x{00}-\x{ff}]) #)
98 ? $_ ##-- Latin-1 character: just copy
99 : Text::Unidecode::unidecode($_) ##-- Non-Latin-1: transliterate
100 )
101 } split(//,$l0)
102 );
103 $l = decode('latin1',$l);
104  
105 if (
106 #$l =~ m([^\p{inBasicLatin}\p{inLatin1Supplement}]) #)
107 $l =~ m([^\x{00}-\x{ff}]) #)
108 ) {
109 ##-- sanity check
110 $xlit->logwarn("analyzeTypes(): transliteration resulted in non-latin-1 string: '$l' for utf-8 '$w'");
111 }
112  
113 ##-- set properties
114 $isLatin1 = 0;
115 $isLatinExt = ($uc =~ m([^\p{Latin}]) ? 0 : 1);
116 } else {
117 $l = $uc;
118 $isLatin1 = $isLatinExt = 1;
119 }
120  
121 ##-- return
122 #return [ $l, $isLatin1, $isLatinExt ];
123 #$tok->{$akey} = [ $l, $isLatin1, $isLatinExt ];
124 $tok->{$akey} = { latin1Text=>$l, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt };
125 }
126 return $doc;
127 }
128  
129 1; ##-- be happy
130  
131 __END__