| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Analyzer::Unicruft.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: latin-1 approximator | ||
| 6 | |||
| 7 | package DTA::CAB::Analyzer::Unicruft; | ||
| 8 | |||
| 9 | use DTA::CAB::Analyzer; | ||
| 10 | use DTA::CAB::Datum ':all'; | ||
| 11 | use DTA::CAB::Token; | ||
| 12 | |||
| 13 | use Unicruft; | ||
| 14 | use Unicode::Normalize; ##-- compatibility decomposition 'KD' (see Unicode TR #15) | ||
| 15 | #use Unicode::UCD; ##-- unicode character names, info, etc. | ||
| 16 | #use Unicode::CharName; ##-- ... faster access to character name, block | ||
| 17 | #use Text::Unidecode; ##-- last-ditch effort: transliterate to ASCII | ||
| 18 | |||
| 19 | use Encode qw(encode decode); | ||
| 20 | use IO::File; | ||
| 21 | use Carp; | ||
| 22 | |||
| 23 | use strict; | ||
| 24 | |||
| 25 | ##============================================================================== | ||
| 26 | ## Globals | ||
| 27 | ##============================================================================== | ||
| 28 | |||
| 29 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Constructors etc. | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 36 | ## + object structure, new: | ||
| 37 | ## label => 'xlit', ##-- analyzer label | ||
| 38 | ## + object structure, INHERITED from Analyzer: | ||
| 39 | ## label => $label, ##-- analyzer label (default: from class name) | ||
| 40 | sub new { | ||
| 41 | my $that = shift; | ||
| 42 | return $that->SUPER::new( | ||
| 43 | ##-- options | ||
| 44 | label => 'xlit', | ||
| 45 | |||
| 46 | ##-- user args | ||
| 47 | @_ | ||
| 48 | ); | ||
| 49 | } | ||
| 50 | |||
| 51 | ##============================================================================== | ||
| 52 | ## Methods: version | ||
| 53 | |||
| 54 | ## $version_or_undef = $anl->version() | ||
| 55 | sub version { | ||
| 56 | return "Unicruft-${Unicruft::VERSION} libunicruft-".Unicruft::library_version(); | ||
| 57 | } | ||
| 58 | |||
| 59 | ##============================================================================== | ||
| 60 | ## Methods: I/O | ||
| 61 | ##============================================================================== | ||
| 62 | |||
| 63 | ## $bool = $anl->ensureLoaded() | ||
| 64 | ## + ensures analysis data is loaded | ||
| 65 | ## + always returns 1, but reports Unicruft module + library version if (!$anl->{loaded}) | ||
| 66 | sub ensureLoaded { | ||
| 67 | my $anl = shift; | ||
| 68 | $anl->info("using Unicruft.xs v$Unicruft::VERSION; libunicruft v", Unicruft::library_version) | ||
| 69 | if (!$anl->{loaded}); | ||
| 70 | return $anl->{loaded}=1; | ||
| 71 | } | ||
| 72 | |||
| 73 | ##============================================================================== | ||
| 74 | ## Methods: Analysis: v1.x | ||
| 75 | ##============================================================================== | ||
| 76 | |||
| 77 | ## $doc = $xlit->analyzeTypes($doc,\%types,\%opts) | ||
| 78 | ## + perform type-wise analysis of all (text) types in values(%types) | ||
| 79 | ## + sets: | ||
| 80 | ## $tok->{$anl->{label}} = { latin1Text=>$latin1Text, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt } | ||
| 81 | ## with: | ||
| 82 | ## $latin1Text = $str ##-- best latin-1 approximation of $token->{text} | ||
| 83 | ## $isLatin1 = $bool ##-- true iff $token->{text} is losslessly encodable as latin1 | ||
| 84 | ## $isLatinExt = $bool, ##-- true iff $token->{text} is losslessly encodable as latin-extended | ||
| 85 | sub analyzeTypes { | ||
| 86 | my ($xlit,$doc,$types,$opts) = @_; | ||
| 87 | $types = $doc->types if (!$types); | ||
| 88 | my $akey = $xlit->{label}; | ||
| 89 | |||
| 90 | my ($tok, $w,$uc, $ld, $isLatin1,$isLatinExt); | ||
| 91 | foreach $tok (values(%$types)) { | ||
| 92 | next if (defined($tok->{$akey})); ##-- avoid re-analysis | ||
| 93 | $w = $tok->{text}; | ||
| 94 | |||
| 95 | ##-- 2010-01-23: Mantis Bug #140: 'µ'="\x{b5}" gets mapped to 'm' rather than | ||
| 96 | ## + (unicruft-v0.07) 'u' | ||
| 97 | ## + (unicruft-v0.08) 'µ' (identity) | ||
| 98 | ## + problem is NFKC-decomposition which maps | ||
| 99 | ## 'µ'="\x{b5}" = Latin1 Supplement / MICRO SIGN | ||
| 100 | ## to | ||
| 101 | ## "\x{03bc}" = Greek and Coptic / GREEK SMALL LETTER MU | ||
| 102 | ## + solution (hack): use NFC (canonical composition only) | ||
| 103 | ## rather than NFKC (compatibility decomposition + canonical composition) here, | ||
| 104 | ## and let Unicruft take care of decomposition | ||
| 105 | ## + potentially problematic cases (from unicode normalization form techreport | ||
| 106 | ## @ http://unicode.org/reports/tr15/ : fi ligature, 2^5, long-S + diacritics) | ||
| 107 | ## are all handled correctly by unicruft | ||
| 108 | #$uc = Unicode::Normalize::NFKC($w); ##-- compatibility(?) decomposition + canonical composition | ||
| 109 | $uc = Unicode::Normalize::NFC($w); ##-- canonical composition only | ||
| 110 | |||
| 111 | ##-- construct latin-1/de approximation | ||
| 112 | $ld = decode('latin1',Unicruft::utf8_to_latin1_de($uc)); | ||
| 113 | |||
| 114 | ##-- special handling for double-initial-caps, e.g. "AUf", "CHristus", "GOtt", etc. | ||
| 115 | $ld = ucfirst(lc($ld)) if ($ld =~ /^[[:upper:]]{2}[[:lower:]]+$/); | ||
| 116 | |||
| 117 | ##-- properties | ||
| 118 | if ( | ||
| 119 | #$uc !~ m([^\p{inBasicLatin}\p{inLatin1Supplement}]) #) | ||
| 120 | $uc =~ m(^[\x{00}-\x{ff}]*$) #) | ||
| 121 | ) | ||
| 122 | { | ||
| 123 | $isLatin1 = $isLatinExt = 1; | ||
| 124 | } | ||
| 125 | elsif ($uc =~ m(^[\x{00}-\x{ff}\p{Latin}\p{IsPunct}\p{IsMark}\x{a75b}\x{fffc}-\x{ffff}]*$)) | ||
| 126 | { | ||
| 127 | $isLatin1 = 0; | ||
| 128 | $isLatinExt = 1; | ||
| 129 | } | ||
| 130 | else | ||
| 131 | { | ||
| 132 | $isLatin1 = $isLatinExt = 0; | ||
| 133 | } | ||
| 134 | |||
| 135 | ##-- update token | ||
| 136 | $tok->{$akey} = { latin1Text=>$ld, isLatin1=>$isLatin1, isLatinExt=>$isLatinExt }; | ||
| 137 | } | ||
| 138 | |||
| 139 | return $doc; | ||
| 140 | } | ||
| 141 | |||
| 142 | |||
| 143 | 1; ##-- be happy | ||
| 144 | |||
| 145 | __END__ |