| 1 | ##---------------------------------------------------------------------------## | ||
| 2 | ## File: | ||
| 3 | ## $Id: MhaEncode.pm,v 1.3 2003/03/05 22:17:15 ehood Exp $ | ||
| 4 | ## Author: | ||
| 5 | ## Earl Hood earl@earlhood.com | ||
| 6 | ## Description: | ||
| 7 | ## POD after __END__. | ||
| 8 | ##---------------------------------------------------------------------------## | ||
| 9 | ## Copyright (C) 2002 Earl Hood, earl@earlhood.com | ||
| 10 | ## | ||
| 11 | ## This program is free software; you can redistribute it and/or modify | ||
| 12 | ## it under the terms of the GNU General Public License as published by | ||
| 13 | ## the Free Software Foundation; either version 2 of the License, or | ||
| 14 | ## (at your option) any later version. | ||
| 15 | ## | ||
| 16 | ## This program is distributed in the hope that it will be useful, | ||
| 17 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 18 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 19 | ## GNU General Public License for more details. | ||
| 20 | ## | ||
| 21 | ## You should have received a copy of the GNU General Public License | ||
| 22 | ## along with this program; if not, write to the Free Software | ||
| 23 | ## Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA | ||
| 24 | ## 02111-1307, USA | ||
| 25 | ##---------------------------------------------------------------------------## | ||
| 26 | |||
| 27 | package MHonArc::UTF8::MhaEncode; | ||
| 28 | |||
| 29 | use strict; | ||
| 30 | use MHonArc::CharMaps; | ||
| 31 | use MHonArc::Char; | ||
| 32 | |||
| 33 | our $VERSION = '2.6.24'; | ||
| 34 | |||
| 35 | my %CharsetMaps = ( | ||
| 36 | 'iso-8859-1' => 'MHonArc/UTF8/ISO8859_1.pm', | ||
| 37 | 'iso-8859-2' => 'MHonArc/UTF8/ISO8859_2.pm', | ||
| 38 | 'iso-8859-3' => 'MHonArc/UTF8/ISO8859_3.pm', | ||
| 39 | 'iso-8859-4' => 'MHonArc/UTF8/ISO8859_4.pm', | ||
| 40 | 'iso-8859-5' => 'MHonArc/UTF8/ISO8859_5.pm', | ||
| 41 | 'iso-8859-6' => 'MHonArc/UTF8/ISO8859_6.pm', | ||
| 42 | 'iso-8859-7' => 'MHonArc/UTF8/ISO8859_7.pm', | ||
| 43 | 'iso-8859-8' => 'MHonArc/UTF8/ISO8859_8.pm', | ||
| 44 | 'iso-8859-9' => 'MHonArc/UTF8/ISO8859_9.pm', | ||
| 45 | 'iso-8859-10' => 'MHonArc/UTF8/ISO8859_10.pm', | ||
| 46 | 'iso-8859-11' => 'MHonArc/UTF8/ISO8859_11.pm', | ||
| 47 | 'iso-8859-13' => 'MHonArc/UTF8/ISO8859_13.pm', | ||
| 48 | 'iso-8859-14' => 'MHonArc/UTF8/ISO8859_14.pm', | ||
| 49 | 'iso-8859-15' => 'MHonArc/UTF8/ISO8859_15.pm', | ||
| 50 | 'iso-8859-16' => 'MHonArc/UTF8/ISO8859_16.pm', | ||
| 51 | 'cp866' => 'MHonArc/UTF8/CP866.pm', | ||
| 52 | 'cp949' => 'MHonArc/UTF8/CP949.pm', # euc-kr | ||
| 53 | 'cp932' => 'MHonArc/UTF8/CP932.pm', # shiftjis | ||
| 54 | 'cp936' => 'MHonArc/UTF8/CP936.pm', # GBK | ||
| 55 | 'cp950' => 'MHonArc/UTF8/CP950.pm', | ||
| 56 | 'cp1250' => 'MHonArc/UTF8/CP1250.pm', | ||
| 57 | 'cp1251' => 'MHonArc/UTF8/CP1251.pm', | ||
| 58 | 'cp1252' => 'MHonArc/UTF8/CP1252.pm', | ||
| 59 | 'cp1253' => 'MHonArc/UTF8/CP1253.pm', | ||
| 60 | 'cp1254' => 'MHonArc/UTF8/CP1254.pm', | ||
| 61 | 'cp1255' => 'MHonArc/UTF8/CP1255.pm', | ||
| 62 | 'cp1256' => 'MHonArc/UTF8/CP1256.pm', | ||
| 63 | 'cp1257' => 'MHonArc/UTF8/CP1257.pm', | ||
| 64 | 'cp1258' => 'MHonArc/UTF8/CP1258.pm', | ||
| 65 | 'koi-0' => 'MHonArc/UTF8/KOI_0.pm', | ||
| 66 | 'koi-7' => 'MHonArc/UTF8/KOI_7.pm', | ||
| 67 | 'koi8-a' => 'MHonArc/UTF8/KOI8_A.pm', | ||
| 68 | 'koi8-b' => 'MHonArc/UTF8/KOI8_B.pm', | ||
| 69 | 'koi8-e' => 'MHonArc/UTF8/KOI8_E.pm', | ||
| 70 | 'koi8-f' => 'MHonArc/UTF8/KOI8_F.pm', | ||
| 71 | 'koi8-r' => 'MHonArc/UTF8/KOI8_R.pm', | ||
| 72 | 'koi8-u' => 'MHonArc/UTF8/KOI8_U.pm', | ||
| 73 | 'gost19768-87' => 'MHonArc/UTF8/GOST19768_87.pm', | ||
| 74 | 'viscii' => 'MHonArc/UTF8/VISCII.pm', | ||
| 75 | 'macarabic' => 'MHonArc/UTF8/AppleArabic.pm', | ||
| 76 | 'maccentraleurroman' => 'MHonArc/UTF8/AppleCenteuro.pm', | ||
| 77 | 'maccroatian' => 'MHonArc/UTF8/AppleCroatian.pm', | ||
| 78 | 'maccyrillic' => 'MHonArc/UTF8/AppleCyrillic.pm', | ||
| 79 | 'macgreek' => 'MHonArc/UTF8/AppleGreek.pm', | ||
| 80 | 'machebrew' => 'MHonArc/UTF8/AppleHebrew.pm', | ||
| 81 | 'macicelandic' => 'MHonArc/UTF8/AppleIceland.pm', | ||
| 82 | 'macromanian' => 'MHonArc/UTF8/AppleRomanian.pm', | ||
| 83 | 'macroman' => 'MHonArc/UTF8/AppleRoman.pm', | ||
| 84 | 'macthai' => 'MHonArc/UTF8/AppleThai.pm', | ||
| 85 | 'macturkish' => 'MHonArc/UTF8/AppleTurkish.pm', | ||
| 86 | 'big5-eten' => 'MHonArc/UTF8/BIG5_ETEN.pm', | ||
| 87 | 'big5-hkscs' => 'MHonArc/UTF8/BIG5_HKSCS.pm', | ||
| 88 | 'gb2312' => 'MHonArc/UTF8/GB2312.pm', | ||
| 89 | 'euc-jp' => 'MHonArc/UTF8/EUC_JP.pm', | ||
| 90 | 'hp-roman8' => 'MHonArc/UTF8/HP_ROMAN8.pm', | ||
| 91 | ); | ||
| 92 | |||
| 93 | my $char_maps = MHonArc::CharMaps->new(\%CharsetMaps); | ||
| 94 | |||
| 95 | ##---------------------------------------------------------------------------## | ||
| 96 | |||
| 97 | # We do not care for valid sequences, just that we catch everything | ||
| 98 | my $utf8_re = q/[\x00-\x7F]| | ||
| 99 | [\xC0-\xDF][\x00-\xFF]| | ||
| 100 | [\xE0-\xEF][\x00-\xFF]{2}| | ||
| 101 | [\xF0-\xF7][\x00-\xFF]{3}| | ||
| 102 | [\xF8-\xFB][\x00-\xFF]{4}| | ||
| 103 | [\xFC\xFD][\x00-\xFF]{5}| | ||
| 104 | [\x80-\xFF]/; | ||
| 105 | |||
| 106 | # Return the length of an utf-8 string | ||
| 107 | sub utf8_length { | ||
| 108 | my $n = 0; | ||
| 109 | while ($_[0] =~ m/($utf8_re)/gox) { ++$n; } | ||
| 110 | $n; | ||
| 111 | } | ||
| 112 | |||
| 113 | ##---------------------------------------------------------------------------## | ||
| 114 | |||
| 115 | ## Version of TEXTCLIPFUNC for utf8 strings for versions of Perl without | ||
| 116 | ## decent utf8 support (Perl <= 5.6.x). | ||
| 117 | sub clip { | ||
| 118 | my $str = shift; # Unfortunately, it is much easier to make a copy | ||
| 119 | my $len = shift; # Clip length | ||
| 120 | my $is_html = shift; # If entity references should be considered | ||
| 121 | my $has_tags = shift; # If html tags should be stripped | ||
| 122 | |||
| 123 | # If not HTML text, things are alot easier | ||
| 124 | if (!$is_html) { | ||
| 125 | # do nothing if we know for sure there is nothing to do | ||
| 126 | return $str | ||
| 127 | if length($str) <= $len; | ||
| 128 | |||
| 129 | # Get $len utf8 chars | ||
| 130 | $str =~ m/^((?:$utf8_re){1,$len})/x; | ||
| 131 | return $1; | ||
| 132 | } | ||
| 133 | |||
| 134 | $str =~ s/<[^>]*>//g if $has_tags; | ||
| 135 | return $str if length($str) <= $len; # nothing to do | ||
| 136 | |||
| 137 | my ($utf8_len, $er_len); | ||
| 138 | my $text = ""; | ||
| 139 | my $subtext = ""; | ||
| 140 | my $sub_len = $len; | ||
| 141 | my $real_len = 0; | ||
| 142 | |||
| 143 | while ($str ne '') { | ||
| 144 | if (!($str =~ s/^((?:$utf8_re){1,$sub_len})//x)) { | ||
| 145 | # pattern should always match, but just in-case... | ||
| 146 | warn qq/Warning: MHonArc::UTF8::MhaEncode::clip:/, | ||
| 147 | qq/ Internal error/; | ||
| 148 | return $text . $str; | ||
| 149 | } | ||
| 150 | $subtext = $1; | ||
| 151 | |||
| 152 | # check for clipped entity reference | ||
| 153 | if (($str ne '') && ($subtext =~ /\&[^;]*\Z/)) { | ||
| 154 | if ($str =~ s/^([^;]*;)//) { | ||
| 155 | $subtext .= $1; | ||
| 156 | } else { | ||
| 157 | warn qq/Warning: MHonArc::UTF8::MhaEncode::clip: malformed/, | ||
| 158 | qq/ entity reference detected\n/; | ||
| 159 | $subtext .= $str; | ||
| 160 | $str = ''; | ||
| 161 | } | ||
| 162 | } | ||
| 163 | |||
| 164 | # compute entity reference lengths to determine "real" character | ||
| 165 | # count and not raw character count. | ||
| 166 | $er_len = 0; | ||
| 167 | while ($subtext =~ /(\&[^;]+);/g) { | ||
| 168 | $er_len += length($1); | ||
| 169 | } | ||
| 170 | |||
| 171 | # done if we have enough | ||
| 172 | $utf8_len = utf8_length($subtext); | ||
| 173 | $real_len += $utf8_len - $er_len; | ||
| 174 | $text .= $subtext; | ||
| 175 | last if ($real_len >= $len); | ||
| 176 | $sub_len = $len - $real_len; | ||
| 177 | } | ||
| 178 | $text; | ||
| 179 | } | ||
| 180 | |||
| 181 | sub to_utf8 { | ||
| 182 | my $data = shift; | ||
| 183 | my $charset = lc shift; | ||
| 184 | my $data_r = ref($data) ? $data : \$data; | ||
| 185 | |||
| 186 | return $$data_r | ||
| 187 | if ($charset eq 'us-ascii' | ||
| 188 | || $charset eq 'utf-8' | ||
| 189 | || $charset eq 'utf8'); | ||
| 190 | MHonArc::Char::map_conv($data_r, $charset, $char_maps); | ||
| 191 | } | ||
| 192 | |||
| 193 | sub str2sgml { | ||
| 194 | my $data = shift; | ||
| 195 | my $charset = lc shift; | ||
| 196 | my $data_r = ref($data) ? $data : \$data; | ||
| 197 | |||
| 198 | if ($charset eq 'us-ascii') { | ||
| 199 | if ($$data_r =~ /[\x80-\xFF]/) { | ||
| 200 | $charset = 'iso-8859-1'; | ||
| 201 | } else { | ||
| 202 | $$data_r =~ s/([$HTMLSpecials])/$HTMLSpecials{$1}/go; | ||
| 203 | return $$data_r; | ||
| 204 | } | ||
| 205 | } | ||
| 206 | if ($charset eq 'utf-8' || $charset eq 'utf8') { | ||
| 207 | $$data_r =~ s/([$HTMLSpecials])/$HTMLSpecials{$1}/go; | ||
| 208 | return $$data_r; | ||
| 209 | } | ||
| 210 | MHonArc::Char::map_conv($data_r, $charset, $char_maps); | ||
| 211 | $$data_r =~ s/([$HTMLSpecials])/$HTMLSpecials{$1}/go; | ||
| 212 | $$data_r; | ||
| 213 | } | ||
| 214 | |||
| 215 | ##---------------------------------------------------------------------------## | ||
| 216 | 1; | ||
| 217 | __END__ |