| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Unidecode.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: latin-1 approximator (old) | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Unidecode; | ||
| 8 | |||
| 9 | use DTA::CAB::Analyzer; | ||
| 10 | use DTA::CAB::Datum ':all'; | ||
| 11 | use DTA::CAB::Token; | ||
| 12 | |||
| 13 | use Unicode::Normalize; ##-- compatibility decomposition 'KD' (see Unicode TR #15) | ||
| 14 | use Text::Unidecode; ##-- last-ditch effort: transliterate to ASCII | ||
| 15 | #use Unicode::UCD; ##-- unicode character names, info, etc. | ||
| 16 | #use Unicode::CharName; ##-- ... faster access to character name, block | ||
| 17 | |||
| 18 | use Encode qw(encode decode); | ||
| 19 | use IO::File; | ||
| 20 | use Carp; | ||
| 21 | |||
| 22 | use strict; | ||
| 23 | |||
| 24 | ##============================================================================== | ||
| 25 | ## Globals | ||
| 26 | ##============================================================================== | ||
| 27 | |||
| 28 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 29 | |||
| 30 | ##============================================================================== | ||
| 31 | ## Constructors etc. | ||
| 32 | ##============================================================================== | ||
| 33 | |||
| 34 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 35 | ## + object structure, new: | ||
| 36 | ## label => $key, ##-- token analysis key (default='xlit') | ||
| 37 | sub new { | ||
| 38 | my $that = shift; | ||
| 39 | return $that->SUPER::new( | ||
| 40 | ##-- options | ||
| 41 | label => 'xlit', | ||
| 42 | |||
| 43 | ##-- user args | ||
| 44 | @_ | ||
| 45 | ); | ||
| 46 | } | ||
| 47 | |||
| 48 | ##============================================================================== | ||
| 49 | ## Methods: I/O | ||
| 50 | ##============================================================================== | ||
| 51 | |||
| 52 | ## $bool = $aut->ensureLoaded() | ||
| 53 | ## + ensures analysis data is loaded | ||
| 54 | sub ensureLoaded { return 1; } | ||
| 55 | |||
| 56 | ##============================================================================== | ||
| 57 | ## Methods: Analysis: v1.x | ||
| 58 | ##============================================================================== | ||
| 59 | |||
| 60 | ## $doc = $xlit->analyzeTypes($doc,\%types,\%opts) | ||
| 61 | ## + perform type-wise analysis of all (text) types in values(%types) | ||
| 62 | ## + sets | ||
| 63 | ## $tok->{$anl->{label}} = { latin1Text=>$latin1Text, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt } | ||
| 64 | ## with: | ||
| 65 | ## $latin1Text = $str ##-- best latin-1 approximation of $token->{text} | ||
| 66 | ## $isLatin1 = $bool ##-- true iff $token->{text} is losslessly encodable as latin1 | ||
| 67 | ## $isLatinExt = $bool, ##-- true iff $token->{text} is losslessly encodable as latin-extended | ||
| 68 | sub analyzeTypes { | ||
| 69 | my ($xlit,$doc,$types,$opts) = @_; | ||
| 70 | $types = $doc->types if (!$types); | ||
| 71 | my $akey = $xlit->{label}; | ||
| 72 | |||
| 73 | my ($tok, $w,$uc, $ld,$l0,$l, $isLatin1,$isLatinExt); | ||
| 74 | foreach $tok (values(%$types)) { | ||
| 75 | $w = $tok->{text}; | ||
| 76 | $uc = Unicode::Normalize::NFKC($w); ##-- compatibility(?) decomposition + canonical composition | ||
| 77 | |||
| 78 | ##-- construct latin-1 approximation | ||
| 79 | if ( | ||
| 80 | #$uc =~ m([^\p{inBasicLatin}\p{inLatin1Supplement}]) #) | ||
| 81 | $uc =~ m([^\x{00}-\x{ff}]) #) | ||
| 82 | ) | ||
| 83 | { | ||
| 84 | $l0 = $uc; | ||
| 85 | |||
| 86 | ##-- special handling for some character sequences | ||
| 87 | $l0 =~ s/\x{0363}/a/g; ##-- COMBINING LATIN SMALL LETTER A | ||
| 88 | $l0 =~ s/\x{0364}/e/g; ##-- COMBINING LATIN SMALL LETTER E | ||
| 89 | $l0 =~ s/\x{0365}/i/g; ##-- COMBINING LATIN SMALL LETTER I | ||
| 90 | $l0 =~ s/\x{0366}/o/g; ##-- COMBINING LATIN SMALL LETTER O | ||
| 91 | |||
| 92 | ##-- default: copy plain latin-1 characters, transliterate rest with Text::Unidecode::unidecode() | ||
| 93 | $l = join('', | ||
| 94 | map { | ||
| 95 | ( | ||
| 96 | #$_ =~ m(\p{inBasicLatin}|\p{InLatin1Supplement}) #) | ||
| 97 | $_ =~ m([\x{00}-\x{ff}]) #) | ||
| 98 | ? $_ ##-- Latin-1 character: just copy | ||
| 99 | : Text::Unidecode::unidecode($_) ##-- Non-Latin-1: transliterate | ||
| 100 | ) | ||
| 101 | } split(//,$l0) | ||
| 102 | ); | ||
| 103 | $l = decode('latin1',$l); | ||
| 104 | |||
| 105 | if ( | ||
| 106 | #$l =~ m([^\p{inBasicLatin}\p{inLatin1Supplement}]) #) | ||
| 107 | $l =~ m([^\x{00}-\x{ff}]) #) | ||
| 108 | ) { | ||
| 109 | ##-- sanity check | ||
| 110 | $xlit->logwarn("analyzeTypes(): transliteration resulted in non-latin-1 string: '$l' for utf-8 '$w'"); | ||
| 111 | } | ||
| 112 | |||
| 113 | ##-- set properties | ||
| 114 | $isLatin1 = 0; | ||
| 115 | $isLatinExt = ($uc =~ m([^\p{Latin}]) ? 0 : 1); | ||
| 116 | } else { | ||
| 117 | $l = $uc; | ||
| 118 | $isLatin1 = $isLatinExt = 1; | ||
| 119 | } | ||
| 120 | |||
| 121 | ##-- return | ||
| 122 | #return [ $l, $isLatin1, $isLatinExt ]; | ||
| 123 | #$tok->{$akey} = [ $l, $isLatin1, $isLatinExt ]; | ||
| 124 | $tok->{$akey} = { latin1Text=>$l, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt }; | ||
| 125 | } | ||
| 126 | return $doc; | ||
| 127 | } | ||
| 128 | |||
| 129 | 1; ##-- be happy | ||
| 130 | |||
| 131 | __END__ |