lib/Text/ASCII/Convert.pm

filter: Statementstmt 100% 26 / 26 Branchbran 83.3% 5 / 6 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 5 / 5 Podpod 0.0% 0 / 1 totaltotal 94.7% 36 / 38 CC 4 SCAR 9.2
CC
4
Cov
97%
convert_to_ascii
4.0
BEGIN
1.0
BEGIN
1.0
CRAP 2.5
↑ Summary
1 package Text::ASCII::Convert;
2 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use warnings FATAL => 'all';
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use Encode qw(decode FB_CROAK LEAVE_SRC);
Subroutine BEGIN: called CC=1 SCAR=0.0
5 1 ▶use base 'Exporter';
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 our @EXPORT = qw(convert_to_ascii);
7 1 our @EXPORT_OK = qw(convert_to_ascii);
8 1 our $VERSION = '0.38';
9 1 my %char_map;
10  
11 =encoding utf8
12  
13 =head1 NAME
14  
15 Text::ASCII::Convert - Perl module to convert non-ASCII characters to their ASCII equivalents
16  
17 =head1 SYNOPSIS
18  
19 use Text::ASCII::Convert;
20  
21 print convert_to_ascii("Ýou hãve a nèw vòice-mãil");
22  
23 # prints "You have a new voice-mail"
24  
25 =head1 DESCRIPTION
26  
27 This module attempts to convert non-ASCII characters in a string to their closet ASCII homoglyph. The input
28 can be a string of Unicode characters or a string of UTF-8 octets. The output is always a string of ASCII characters
29 in the range 0x00 to 0x7F.
30  
31 This is most useful for catching spam that uses non-ASCII characters to obfuscate words. For example,
32  
33 Ýou hãve a nèw vòice-mãil
34 You havé Reꞓeìved an Enꞓryptéd Company Maíl
35  
36 would be converted to
37  
38 You have a new voice-mail
39 You have ReCeived an EnCrypted Company Mail
40  
41 Unlike other transliteration software, this plugin converts non-ASCII characters
42 to their ASCII equivalents based on appearance instead of meaning. For example, the
43 German eszett character 'ß' is converted to the Roman letter 'B' instead of 'ss'
44 because it resembles a 'B' in appearance. Likewise, the Greek letter Sigma ('Σ') is
45 converted to 'E' and a lower case Omega ('ω') is converted to 'w' even though these
46 letters have different lexical meanings.
47  
48 Not all non-ASCII characters are converted. For example, the Japanese Hiragana
49 character 'あ' is not converted because it does not resemble any ASCII character.
50 Characters that have no ASCII equivalent are replaced by spaces. To avoid long runs
51 of spaces, multiple spaces are collapsed into a single space. For example,
52  
53 Find 💋💘Singles💋💘 in your Area
54  
55 would be converted to
56  
57 Find Singles in your Area
58  
59 The plugin also removes zero-width characters such as the zero-width
60 space (U+200B) and zero-width non-joiner (U+200C) that are often used to
61 obfuscate words.
62  
63 Control characters such as tabs, newlines, and carriage returns are retained.
64  
65 =head1 AUTHORS
66  
67 Kent Oyer <kent@mxguardian.net>
68  
69 =head1 LICENSE AND COPYRIGHT
70  
71 Copyright (C) 2023 MXGuardian LLC
72  
73 This program is free software: you can redistribute it and/or modify
74 it under the terms of the GNU General Public License as published by
75 the Free Software Foundation, either version 3 of the License, or
76 (at your option) any later version.
77  
78 This program is distributed in the hope that it will be useful,
79 but WITHOUT ANY WARRANTY; without even the implied warranty of
80 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE
81 file included with this distribution for more information.
82  
83 You should have received a copy of the GNU General Public License
84 along with this program. If not, see https://www.gnu.org/licenses/.
85  
86 =cut
87  
88 UNITCHECK {
89 # build character map from __DATA__ section
90 while (<DATA>) {
91 chomp;
92 my ($key,$value) = split /\s+/;
93 my $ascii = join('', map { chr(hex($_)) } split /\+/, $value);
94 $char_map{chr(hex($key))} = $ascii;
95 }
96 close DATA;
97 };
98  
99 # Converts a string of Unicode characters (or UTF-8 encoded bytes) to a string of ASCII characters
100 # in the range 0x00 to 0x7F. Non-ASCII characters are replaced with their ASCII equivalents.
101 # Zero-width characters and combining marks are removed. Multiple spaces are collapsed into a single space.
102 #
103 7 ▶sub convert_to_ascii {
Subroutine convert_to_ascii: called CC=4 SCAR=13.9
Pod undocumented
104 7 ▶ return $_[0] if !defined $_[0];
Branch: 1 branch
BranchTrueFalseTotal
unless defined $_[0] 0 7 7
105 # Decode as UTF-8 if the input is syntactically valid UTF-8, otherwise
106 # treat it as already-decoded characters. FB_CROAK dies on invalid input
107 # (caught by eval) and LEAVE_SRC keeps the source buffer untouched. A plain
108 # decode (rather than a validating regex) avoids Perl's "Complex regular
109 # subexpression recursion limit (65534) exceeded" on inputs over ~64K chars.
110 7 my $decoded = eval { decode('UTF-8', $_[0], FB_CROAK | LEAVE_SRC) };
111 7 ▶ my $str = defined $decoded ? $decoded : $_[0];
Branch: 1 branch
BranchTrueFalseTotal
defined $decoded ? : 2 5 7
112 # remove zero-width characters and combining marks
113 7 $str =~ s/[\xAD\x{034F}\x{200A}-\x{200F}\x{202A}\x{202B}\x{202C}\x{2060}\x{FEFF}]|\p{Combining_Mark}//g;
114 # replace non-ascii characters with ascii equivalents
115 7 ▶ $str =~ s/([^[:ascii:]])/defined($char_map{$1})?$char_map{$1}:' '/eg;
Branch: 1 branch
BranchTrueFalseTotal
defined $char_map{$1} ? : 44 9 53
116 # collapse spaces
117 7 $str =~ s/\x{20}+/ /g;
118 7 return $str;
119 }
120  
121  
122 1 1;
123  
124 __DATA__
↑ Summary