| 1 | ##---------------------------------------------------------------------------## | ||
| 2 | ## File: | ||
| 3 | ## $Id: CharEnt.pm,v 1.17 2010/12/31 18:23:02 ehood Exp $ | ||
| 4 | ## Author: | ||
| 5 | ## Earl Hood earl@earlhood.com | ||
| 6 | ## Description: | ||
| 7 | ## POD after __END__ | ||
| 8 | ##---------------------------------------------------------------------------## | ||
| 9 | ## Copyright (C) 1997-2002 Earl Hood, earl@earlhood.com | ||
| 10 | ## | ||
| 11 | ## This program is free software; you can redistribute it and/or modify | ||
| 12 | ## it under the terms of the GNU General Public License as published by | ||
| 13 | ## the Free Software Foundation; either version 2 of the License, or | ||
| 14 | ## (at your option) any later version. | ||
| 15 | ## | ||
| 16 | ## This program is distributed in the hope that it will be useful, | ||
| 17 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 18 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 19 | ## GNU General Public License for more details. | ||
| 20 | ## | ||
| 21 | ## You should have received a copy of the GNU General Public License | ||
| 22 | ## along with this program; if not, write to the Free Software | ||
| 23 | ## Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA | ||
| 24 | ## 02111-1307, USA | ||
| 25 | ##---------------------------------------------------------------------------## | ||
| 26 | |||
| 27 | package MHonArc::CharEnt; | ||
| 28 | |||
| 29 | use strict; | ||
| 30 | use MHonArc::CharMaps; | ||
| 31 | use MHonArc::Char; | ||
| 32 | |||
| 33 | our $VERSION = '2.6.24'; | ||
| 34 | |||
| 35 | ##--------------------------------------------------------------------------- | ||
| 36 | ## Charset specification to mapping | ||
| 37 | ##--------------------------------------------------------------------------- | ||
| 38 | ## NOTE: The mapping uses a single name for a charset. | ||
| 39 | ## The CHARSETALIASES resource can be used to map aka names (aliases) | ||
| 40 | ## to the names used here. | ||
| 41 | ## NOTE: UTF-8 does not require a map since UTF-8 is decoded straight | ||
| 42 | ## to &#xHHHH; entity references. | ||
| 43 | ## NOTE: iso-2022-{jp,kr} are translated to euc-{jp,kr} first before | ||
| 44 | ## conversion. | ||
| 45 | |||
| 46 | my %CharsetMaps = ( | ||
| 47 | 'iso-8859-1' => 'MHonArc/CharEnt/ISO8859_1.pm', | ||
| 48 | 'iso-8859-2' => 'MHonArc/CharEnt/ISO8859_2.pm', | ||
| 49 | 'iso-8859-3' => 'MHonArc/CharEnt/ISO8859_3.pm', | ||
| 50 | 'iso-8859-4' => 'MHonArc/CharEnt/ISO8859_4.pm', | ||
| 51 | 'iso-8859-5' => 'MHonArc/CharEnt/ISO8859_5.pm', | ||
| 52 | 'iso-8859-6' => 'MHonArc/CharEnt/ISO8859_6.pm', | ||
| 53 | 'iso-8859-7' => 'MHonArc/CharEnt/ISO8859_7.pm', | ||
| 54 | 'iso-8859-8' => 'MHonArc/CharEnt/ISO8859_8.pm', | ||
| 55 | 'iso-8859-9' => 'MHonArc/CharEnt/ISO8859_9.pm', | ||
| 56 | 'iso-8859-10' => 'MHonArc/CharEnt/ISO8859_10.pm', | ||
| 57 | 'iso-8859-11' => 'MHonArc/CharEnt/ISO8859_11.pm', | ||
| 58 | 'iso-8859-13' => 'MHonArc/CharEnt/ISO8859_13.pm', | ||
| 59 | 'iso-8859-14' => 'MHonArc/CharEnt/ISO8859_14.pm', | ||
| 60 | 'iso-8859-15' => 'MHonArc/CharEnt/ISO8859_15.pm', | ||
| 61 | 'iso-8859-16' => 'MHonArc/CharEnt/ISO8859_16.pm', | ||
| 62 | 'cp866' => 'MHonArc/CharEnt/CP866.pm', | ||
| 63 | 'cp949' => 'MHonArc/CharEnt/CP949.pm', # euc-kr | ||
| 64 | 'cp932' => 'MHonArc/CharEnt/CP932.pm', # shiftjis | ||
| 65 | 'cp936' => 'MHonArc/CharEnt/CP936.pm', # GBK | ||
| 66 | 'cp950' => 'MHonArc/CharEnt/CP950.pm', | ||
| 67 | 'cp1250' => 'MHonArc/CharEnt/CP1250.pm', | ||
| 68 | 'cp1251' => 'MHonArc/CharEnt/CP1251.pm', | ||
| 69 | 'cp1252' => 'MHonArc/CharEnt/CP1252.pm', | ||
| 70 | 'cp1253' => 'MHonArc/CharEnt/CP1253.pm', | ||
| 71 | 'cp1254' => 'MHonArc/CharEnt/CP1254.pm', | ||
| 72 | 'cp1255' => 'MHonArc/CharEnt/CP1255.pm', | ||
| 73 | 'cp1256' => 'MHonArc/CharEnt/CP1256.pm', | ||
| 74 | 'cp1257' => 'MHonArc/CharEnt/CP1257.pm', | ||
| 75 | 'cp1258' => 'MHonArc/CharEnt/CP1258.pm', | ||
| 76 | 'koi-0' => 'MHonArc/CharEnt/KOI_0.pm', | ||
| 77 | 'koi-7' => 'MHonArc/CharEnt/KOI_7.pm', | ||
| 78 | 'koi8-a' => 'MHonArc/CharEnt/KOI8_A.pm', | ||
| 79 | 'koi8-b' => 'MHonArc/CharEnt/KOI8_B.pm', | ||
| 80 | 'koi8-e' => 'MHonArc/CharEnt/KOI8_E.pm', | ||
| 81 | 'koi8-f' => 'MHonArc/CharEnt/KOI8_F.pm', | ||
| 82 | 'koi8-r' => 'MHonArc/CharEnt/KOI8_R.pm', | ||
| 83 | 'koi8-u' => 'MHonArc/CharEnt/KOI8_U.pm', | ||
| 84 | 'gost19768-87' => 'MHonArc/CharEnt/GOST19768_87.pm', | ||
| 85 | 'viscii' => 'MHonArc/CharEnt/VISCII.pm', | ||
| 86 | 'macarabic' => 'MHonArc/CharEnt/AppleArabic.pm', | ||
| 87 | 'maccentraleurroman' => 'MHonArc/CharEnt/AppleCenteuro.pm', | ||
| 88 | 'maccroatian' => 'MHonArc/CharEnt/AppleCroatian.pm', | ||
| 89 | 'maccyrillic' => 'MHonArc/CharEnt/AppleCyrillic.pm', | ||
| 90 | 'macgreek' => 'MHonArc/CharEnt/AppleGreek.pm', | ||
| 91 | 'machebrew' => 'MHonArc/CharEnt/AppleHebrew.pm', | ||
| 92 | 'macicelandic' => 'MHonArc/CharEnt/AppleIceland.pm', | ||
| 93 | 'macromanian' => 'MHonArc/CharEnt/AppleRomanian.pm', | ||
| 94 | 'macroman' => 'MHonArc/CharEnt/AppleRoman.pm', | ||
| 95 | 'macthai' => 'MHonArc/CharEnt/AppleThai.pm', | ||
| 96 | 'macturkish' => 'MHonArc/CharEnt/AppleTurkish.pm', | ||
| 97 | 'big5-eten' => 'MHonArc/CharEnt/BIG5_ETEN.pm', | ||
| 98 | 'big5-hkscs' => 'MHonArc/CharEnt/BIG5_HKSCS.pm', | ||
| 99 | 'gb2312' => 'MHonArc/CharEnt/GB2312.pm', | ||
| 100 | 'euc-jp' => 'MHonArc/CharEnt/EUC_JP.pm', | ||
| 101 | 'hp-roman8' => 'MHonArc/CharEnt/HP_ROMAN8.pm', | ||
| 102 | ); | ||
| 103 | |||
| 104 | my $char_maps = MHonArc::CharMaps->new(\%CharsetMaps); | ||
| 105 | |||
| 106 | ############################################################################### | ||
| 107 | ## Routines | ||
| 108 | ############################################################################### | ||
| 109 | |||
| 110 | sub str2sgml { | ||
| 111 | my $data = shift; | ||
| 112 | my $charset = lc shift; | ||
| 113 | |||
| 114 | my $data_r = ref($data) ? $data : \$data; | ||
| 115 | $charset =~ tr/_/-/; | ||
| 116 | |||
| 117 | # UTF-8 can be converted algorithmically. | ||
| 118 | if ($charset eq 'utf-8') { | ||
| 119 | _utf8_to_sgml($data_r); | ||
| 120 | return $$data_r; | ||
| 121 | } | ||
| 122 | # If us-ascii, use simple s/// operation. | ||
| 123 | if ($charset eq 'us-ascii') { | ||
| 124 | $$data_r =~ s/([$HTMLSpecials])/$HTMLSpecials{$1}/go; | ||
| 125 | return $$data_r; | ||
| 126 | } | ||
| 127 | |||
| 128 | MHonArc::Char::map_conv($data_r, $charset, $char_maps, \%HTMLSpecials); | ||
| 129 | } | ||
| 130 | |||
| 131 | ##---------------------------------------------------------------------------## | ||
| 132 | ## Private Routines. | ||
| 133 | |||
| 134 | # Array of masks for lead byte in UTF-8 (for Perl <5.6) | ||
| 135 | # This could be computed on-the-fly, but using an array is faster | ||
| 136 | my @utf8_lb_mask = ( | ||
| 137 | 0x3F, 0x1F, 0xF, 0x7, 0x3, 0x1 # 1, 2, 3, 4, 5, 6 bytes, respectively | ||
| 138 | ); | ||
| 139 | # Regex pattern for UTF-8 data | ||
| 140 | my $utf8_re = q/([\x00-\x7F]| | ||
| 141 | [\xC0-\xDF][\x80-\xBF]| | ||
| 142 | \xE0 [\xA0-\xBF][\x80-\xBF]| | ||
| 143 | [\xE1-\xEF][\x80-\xBF]{2}| | ||
| 144 | \xF0 [\x90-\xBF][\x80-\xBF]{2}| | ||
| 145 | [\xF1-\xF7][\x80-\xBF]{3}| | ||
| 146 | \xF8 [\x88-\xBF][\x80-\xBF]{3}| | ||
| 147 | [\xF9-\xFB][\x80-\xBF]{4}| | ||
| 148 | \xFC [\x84-\xBF][\x80-\xBF]{4}| | ||
| 149 | \xFD [\x80-\xBF]{5}| | ||
| 150 | .)/; | ||
| 151 | # A lax regex for UTF-8 data. Used for utf-8-aware perl since perl | ||
| 152 | # will validate sequences | ||
| 153 | my $utf8_re_lax = q/([\x00-\x7F]| | ||
| 154 | [\xC0-\xDF][\x00-\xFF]| | ||
| 155 | [\xE0-\xEF][\x00-\xFF]{2}| | ||
| 156 | [\xF0-\xF7][\x00-\xFF]{3}| | ||
| 157 | [\xF8-\xFB][\x00-\xFF]{4}| | ||
| 158 | [\xFC-\xFD][\x00-\xFF]{5}| | ||
| 159 | .)/; | ||
| 160 | |||
| 161 | sub _utf8_to_sgml { | ||
| 162 | my $data_r = shift; | ||
| 163 | |||
| 164 | if ($] >= 5.006) { | ||
| 165 | # UTF-8-aware perl | ||
| 166 | # Have to enable warnings to get stricter utf-8 checks for Perl 5.8 | ||
| 167 | use warnings; | ||
| 168 | my ($char, $ord, $malformed); | ||
| 169 | |||
| 170 | # Define local warn handle to suppress malformed utf-8 warning | ||
| 171 | # messages and to flag when such occurrences happen. | ||
| 172 | my $cur_sig_warn = $SIG{__WARN__}; | ||
| 173 | local $SIG{__WARN__} = sub { | ||
| 174 | $malformed = 1; | ||
| 175 | #warn @_; | ||
| 176 | # invoke current warn handler, if defined | ||
| 177 | &$cur_sig_warn if defined($cur_sig_warn) && ref($cur_sig_warn); | ||
| 178 | }; | ||
| 179 | $$data_r =~ s{ | ||
| 180 | $utf8_re_lax | ||
| 181 | }{ | ||
| 182 | # Bug #26577: Perl 5.10 changed unpack behavior | ||
| 183 | $char = ($] >= 5.010)? unpack('C0U*',$1): unpack('U0U*',$1); | ||
| 184 | if ($malformed || | ||
| 185 | (($char & 0xFFFE) == 0xFFFE) || | ||
| 186 | (($char & 0xFFFF) == 0xFFFF) || | ||
| 187 | ($char >= 0xFDD0 && $char <= 0xFDEF) || | ||
| 188 | ($char >= 0xD800 && $char <= 0xDFFF) | ||
| 189 | ) { | ||
| 190 | # Some of the if() checks may be handled by perl directly, | ||
| 191 | # but such checks can be disabled when perl is built. | ||
| 192 | $malformed = 0; | ||
| 193 | '�'; | ||
| 194 | } else { | ||
| 195 | ($char <= 0x7F) | ||
| 196 | ? $HTMLSpecials{$1} || sprintf('%c',$char) | ||
| 197 | : sprintf('&#x%X;',$char); | ||
| 198 | } | ||
| 199 | }gxeso; | ||
| 200 | |||
| 201 | } else { | ||
| 202 | # non-UTF-8-aware perl | ||
| 203 | my ($i, $n, $char); | ||
| 204 | $$data_r =~ s{ | ||
| 205 | $utf8_re | ||
| 206 | }{ | ||
| 207 | if (($n = length($1)) == 1) { | ||
| 208 | my $ord = ord($1); | ||
| 209 | if ($ord > 0x7F) { | ||
| 210 | # Malformed sequence | ||
| 211 | '�'; | ||
| 212 | } else { | ||
| 213 | # 7-bit ASCII | ||
| 214 | $HTMLSpecials{$1} || $1; | ||
| 215 | } | ||
| 216 | } else { | ||
| 217 | # Multi-byte sequence | ||
| 218 | $char = (unpack('C',substr($1,0,1)) & | ||
| 219 | $utf8_lb_mask[$n-1]) << ($n-1)*6; | ||
| 220 | for ($i=1; $i < $n; ++$i) { | ||
| 221 | $char |= ((unpack('C',substr($1,$i,1)) & 0x3F) << | ||
| 222 | (($n-$i-1)*6)); | ||
| 223 | } | ||
| 224 | if ($char <= 0x7F || # should only be single byte sequence | ||
| 225 | (($char & 0xFFFE) == 0xFFFE) || # not a char | ||
| 226 | (($char & 0xFFFF) == 0xFFFF) || # not a char | ||
| 227 | ($char >= 0xFDD0 && $char <= 0xFDEF) || # not a char | ||
| 228 | ($char >= 0xD800 && $char <= 0xDFFF) # surrogates | ||
| 229 | ) { | ||
| 230 | '�'; | ||
| 231 | } else { | ||
| 232 | sprintf('&#x%X;',$char); | ||
| 233 | } | ||
| 234 | } | ||
| 235 | }gxseo; | ||
| 236 | } | ||
| 237 | } | ||
| 238 | |||
| 239 | ##---------------------------------------------------------------------------## | ||
| 240 | 1; | ||
| 241 | __END__ |