| 1 | package Text::ASCII::Convert; | ||||||||||
| 2 | 1 | ▶ | use strict; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 3 | 1 | ▶ | use warnings FATAL => 'all'; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 4 | 1 | ▶ | use Encode qw(decode FB_CROAK LEAVE_SRC); | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 5 | 1 | ▶ | use base 'Exporter'; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 6 | 1 | our @EXPORT = qw(convert_to_ascii); | |||||||||
| 7 | 1 | our @EXPORT_OK = qw(convert_to_ascii); | |||||||||
| 8 | 1 | our $VERSION = '0.38'; | |||||||||
| 9 | 1 | my %char_map; | |||||||||
| 10 | |||||||||||
| 11 | =encoding utf8 | ||||||||||
| 12 | |||||||||||
| 13 | =head1 NAME | ||||||||||
| 14 | |||||||||||
| 15 | Text::ASCII::Convert - Perl module to convert non-ASCII characters to their ASCII equivalents | ||||||||||
| 16 | |||||||||||
| 17 | =head1 SYNOPSIS | ||||||||||
| 18 | |||||||||||
| 19 | use Text::ASCII::Convert; | ||||||||||
| 20 | |||||||||||
| 21 | print convert_to_ascii("Ýou hãve a nèw vòice-mãil"); | ||||||||||
| 22 | |||||||||||
| 23 | # prints "You have a new voice-mail" | ||||||||||
| 24 | |||||||||||
| 25 | =head1 DESCRIPTION | ||||||||||
| 26 | |||||||||||
| 27 | This module attempts to convert non-ASCII characters in a string to their closet ASCII homoglyph. The input | ||||||||||
| 28 | can be a string of Unicode characters or a string of UTF-8 octets. The output is always a string of ASCII characters | ||||||||||
| 29 | in the range 0x00 to 0x7F. | ||||||||||
| 30 | |||||||||||
| 31 | This is most useful for catching spam that uses non-ASCII characters to obfuscate words. For example, | ||||||||||
| 32 | |||||||||||
| 33 | Ýou hãve a nèw vòice-mãil | ||||||||||
| 34 | You havé Reꞓeìved an Enꞓryptéd Company Maíl | ||||||||||
| 35 | |||||||||||
| 36 | would be converted to | ||||||||||
| 37 | |||||||||||
| 38 | You have a new voice-mail | ||||||||||
| 39 | You have ReCeived an EnCrypted Company Mail | ||||||||||
| 40 | |||||||||||
| 41 | Unlike other transliteration software, this plugin converts non-ASCII characters | ||||||||||
| 42 | to their ASCII equivalents based on appearance instead of meaning. For example, the | ||||||||||
| 43 | German eszett character 'ß' is converted to the Roman letter 'B' instead of 'ss' | ||||||||||
| 44 | because it resembles a 'B' in appearance. Likewise, the Greek letter Sigma ('Σ') is | ||||||||||
| 45 | converted to 'E' and a lower case Omega ('ω') is converted to 'w' even though these | ||||||||||
| 46 | letters have different lexical meanings. | ||||||||||
| 47 | |||||||||||
| 48 | Not all non-ASCII characters are converted. For example, the Japanese Hiragana | ||||||||||
| 49 | character 'あ' is not converted because it does not resemble any ASCII character. | ||||||||||
| 50 | Characters that have no ASCII equivalent are replaced by spaces. To avoid long runs | ||||||||||
| 51 | of spaces, multiple spaces are collapsed into a single space. For example, | ||||||||||
| 52 | |||||||||||
| 53 | Find 💋💘Singles💋💘 in your Area | ||||||||||
| 54 | |||||||||||
| 55 | would be converted to | ||||||||||
| 56 | |||||||||||
| 57 | Find Singles in your Area | ||||||||||
| 58 | |||||||||||
| 59 | The plugin also removes zero-width characters such as the zero-width | ||||||||||
| 60 | space (U+200B) and zero-width non-joiner (U+200C) that are often used to | ||||||||||
| 61 | obfuscate words. | ||||||||||
| 62 | |||||||||||
| 63 | Control characters such as tabs, newlines, and carriage returns are retained. | ||||||||||
| 64 | |||||||||||
| 65 | =head1 AUTHORS | ||||||||||
| 66 | |||||||||||
| 67 | Kent Oyer <kent@mxguardian.net> | ||||||||||
| 68 | |||||||||||
| 69 | =head1 LICENSE AND COPYRIGHT | ||||||||||
| 70 | |||||||||||
| 71 | Copyright (C) 2023 MXGuardian LLC | ||||||||||
| 72 | |||||||||||
| 73 | This program is free software: you can redistribute it and/or modify | ||||||||||
| 74 | it under the terms of the GNU General Public License as published by | ||||||||||
| 75 | the Free Software Foundation, either version 3 of the License, or | ||||||||||
| 76 | (at your option) any later version. | ||||||||||
| 77 | |||||||||||
| 78 | This program is distributed in the hope that it will be useful, | ||||||||||
| 79 | but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||||||||
| 80 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE | ||||||||||
| 81 | file included with this distribution for more information. | ||||||||||
| 82 | |||||||||||
| 83 | You should have received a copy of the GNU General Public License | ||||||||||
| 84 | along with this program. If not, see https://www.gnu.org/licenses/. | ||||||||||
| 85 | |||||||||||
| 86 | =cut | ||||||||||
| 87 | |||||||||||
| 88 | UNITCHECK { | ||||||||||
| 89 | # build character map from __DATA__ section | ||||||||||
| 90 | while (<DATA>) { | ||||||||||
| 91 | chomp; | ||||||||||
| 92 | my ($key,$value) = split /\s+/; | ||||||||||
| 93 | my $ascii = join('', map { chr(hex($_)) } split /\+/, $value); | ||||||||||
| 94 | $char_map{chr(hex($key))} = $ascii; | ||||||||||
| 95 | } | ||||||||||
| 96 | close DATA; | ||||||||||
| 97 | }; | ||||||||||
| 98 | |||||||||||
| 99 | # Converts a string of Unicode characters (or UTF-8 encoded bytes) to a string of ASCII characters | ||||||||||
| 100 | # in the range 0x00 to 0x7F. Non-ASCII characters are replaced with their ASCII equivalents. | ||||||||||
| 101 | # Zero-width characters and combining marks are removed. Multiple spaces are collapsed into a single space. | ||||||||||
| 102 | # | ||||||||||
| 103 | 7 | ▶ | sub convert_to_ascii { | ||||||||
|
Subroutine
convert_to_ascii: called CC=4 SCAR=13.9
Pod
undocumented
| |||||||||||
| 104 | 7 | ▶ | return $_[0] if !defined $_[0]; | ||||||||
|
Branch: 1 branch
| |||||||||||
| 105 | # Decode as UTF-8 if the input is syntactically valid UTF-8, otherwise | ||||||||||
| 106 | # treat it as already-decoded characters. FB_CROAK dies on invalid input | ||||||||||
| 107 | # (caught by eval) and LEAVE_SRC keeps the source buffer untouched. A plain | ||||||||||
| 108 | # decode (rather than a validating regex) avoids Perl's "Complex regular | ||||||||||
| 109 | # subexpression recursion limit (65534) exceeded" on inputs over ~64K chars. | ||||||||||
| 110 | 7 | my $decoded = eval { decode('UTF-8', $_[0], FB_CROAK | LEAVE_SRC) }; | |||||||||
| 111 | 7 | ▶ | my $str = defined $decoded ? $decoded : $_[0]; | ||||||||
|
Branch: 1 branch
| |||||||||||
| 112 | # remove zero-width characters and combining marks | ||||||||||
| 113 | 7 | $str =~ s/[\xAD\x{034F}\x{200A}-\x{200F}\x{202A}\x{202B}\x{202C}\x{2060}\x{FEFF}]|\p{Combining_Mark}//g; | |||||||||
| 114 | # replace non-ascii characters with ascii equivalents | ||||||||||
| 115 | 7 | ▶ | $str =~ s/([^[:ascii:]])/defined($char_map{$1})?$char_map{$1}:' '/eg; | ||||||||
|
Branch: 1 branch
| |||||||||||
| 116 | # collapse spaces | ||||||||||
| 117 | 7 | $str =~ s/\x{20}+/ /g; | |||||||||
| 118 | 7 | return $str; | |||||||||
| 119 | } | ||||||||||
| 120 | |||||||||||
| 121 | |||||||||||
| 122 | 1 | 1; | |||||||||
| 123 | |||||||||||
| 124 | __DATA__ | ||||||||||