blib/lib/Test/utf8.pm

filter: Statementstmt 96.9% 95 / 98 Branchbran 100% 14 / 14 Conditioncond 87.5% 14 / 16 MC/DCmcdc 87.5% 7 / 8 Subroutinesub 95.2% 20 / 21 Podpod 75.0% 6 / 8 totaltotal 94.5% 156 / 165 CC 16 SCAR 9.4
CC
16
Cov
96%
t_flagged_utf8
6.0
is_valid_string
4.0
is_sane_utf8
4.0
CRAP 2.6
↑ Summary
1 package Test::utf8;
2  
3 7 ▶use 5.007003;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 7 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 7 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 7 ▶use base qw(Exporter);
Subroutine BEGIN: called CC=1 SCAR=0.0
9  
10 7 ▶use Encode;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 7 ▶use charnames ':full';
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 7 our $VERSION = "1.03";
14  
15 7 our @EXPORT = qw(
16 is_valid_string is_dodgy_utf8 is_sane_utf8
17 is_within_ascii is_within_latin1 is_within_latin_1
18 is_flagged_utf8 isnt_flagged_utf8
19 );
20  
21 # A Regexp string to match valid UTF8 bytes
22 # this info comes from page 78 of "The Unicode Standard 4.0"
23 # published by the Unicode Consortium
24 7 our $valid_utf8_regexp = <<'REGEX' ;
25 [\x{00}-\x{7f}]
26 | [\x{c2}-\x{df}][\x{80}-\x{bf}]
27 | \x{e0} [\x{a0}-\x{bf}][\x{80}-\x{bf}]
28 | [\x{e1}-\x{ec}][\x{80}-\x{bf}][\x{80}-\x{bf}]
29 | \x{ed} [\x{80}-\x{9f}][\x{80}-\x{bf}]
30 | [\x{ee}-\x{ef}][\x{80}-\x{bf}][\x{80}-\x{bf}]
31 | \x{f0} [\x{90}-\x{bf}][\x{80}-\x{bf}]
32 | [\x{f1}-\x{f3}][\x{80}-\x{bf}][\x{80}-\x{bf}][\x{80}-\x{bf}]
33 | \x{f4} [\x{80}-\x{8f}][\x{80}-\x{bf}][\x{80}-\x{bf}]
34 REGEX
35  
36 =head1 NAME
37  
38 Test::utf8 - handy utf8 tests
39  
40 =head1 SYNOPSIS
41  
42 # check the string is good
43 is_valid_string($string); # check the string is valid
44 is_sane_utf8($string); # check not double encoded
45  
46 # check the string has certain attributes
47 is_flagged_utf8($string1); # has utf8 flag set
48 is_within_ascii($string2); # only has ascii chars in it
49 isnt_within_ascii($string3); # has chars outside the ascii range
50 is_within_latin_1($string4); # only has latin-1 chars in it
51 isnt_within_ascii($string5); # has chars outside the latin-1 range
52  
53 =head1 DESCRIPTION
54  
55 This module is a collection of tests useful for dealing with utf8 strings in
56 Perl.
57  
58 This module has two types of tests: The validity tests check if a string is
59 valid and not corrupt, whereas the characteristics tests will check that string
60 has a given set of characteristics.
61  
62 =head2 Validity Tests
63  
64 =over
65  
66 =item is_valid_string($string, $testname)
67  
68 Checks if the string is "valid", i.e. this passes and returns true unless
69 the internal utf8 flag hasn't been set on scalar that isn't made up of a valid
70 utf-8 byte sequence.
71  
72 This should I<never> happen and, in theory, this test should always pass. Unless
73 you (or a module you use) goes monkeying around inside a scalar using Encode's
74 private functions or XS code you shouldn't ever end up in a situation where
75 you've got a corrupt scalar. But if you do, and you do, then this function
76 should help you detect the problem.
77  
78 To be clear, here's an example of the error case this can detect:
79  
80 my $mark = "Mark";
81 my $leon = "L\x{e9}on";
82 is_valid_string($mark); # passes, not utf-8
83 is_valid_string($leon); # passes, not utf-8
84  
85 my $iloveny = "I \x{2665} NY";
86 is_valid_string($iloveny); # passes, proper utf-8
87  
88 my $acme = "L\x{c3}\x{a9}on";
89 Encode::_utf8_on($acme); # (please don't do things like this)
90 is_valid_string($acme); # passes, proper utf-8 byte sequence upgraded
91  
92 Encode::_utf8_on($leon); # (this is why you don't do things like this)
93 is_valid_string($leon); # fails! the byte \x{e9} isn't valid utf-8
94  
95 =cut
96  
97 sub is_valid_string($;$)
98 12 ▶{
Subroutine is_valid_string: called CC=4 SCAR=13.9
Pod documented
99 12 my $string = shift;
100 12 ▶ my $name = shift || "valid string test";
Condition100%2/2
shift() || 'valid string test'
l!l
111
MC/DC100%1/1
shift() || 'valid string test'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
101  
102 # check we're a utf8 string - if not, we pass.
103 12 ▶ unless (Encode::is_utf8($string))
Branch: 1 branch
BranchTrueFalseTotal
unless (&Encode::is_utf8($string)) 5 7 12
104 5 { return _pass($name) }
105  
106 # work out at what byte (if any) we have an invalid byte sequence
107 # and return the correct test result
108 7 my $pos = _invalid_sequence_at_byte($string);
109 7 ▶ if (_ok(!defined($pos), $name)) { return 1 }
Branch: 1 branch
BranchTrueFalseTotal
if (_ok(!defined($pos), $name)) 4 3 7
110 3 _diag("malformed byte sequence starting at byte $pos");
111 3 return;
112 }
113  
114 sub _invalid_sequence_at_byte($)
115 7 ▶{
Subroutine _invalid_sequence_at_byte: called CC=3 SCAR=11.0
116 7 my $string = shift;
117  
118 # examine the bytes that make up the string (not the chars)
119 # by turning off the utf8 flag (no, use bytes doesn't
120 # work, we're dealing with a regexp)
121 7 Encode::_utf8_off($string); ## no critic (ProtectPrivateSubs)
122  
123 # work out the index of the first non matching byte
124 7 my $result = $string =~ m/^($valid_utf8_regexp)*/ogx;
125  
126 # if we matched all the string return the empty list
127 7 ▶ my $pos = pos $string || 0;
Condition100%2/2
pos $string || 0
l!l
52
MC/DC100%1/1
pos $string || 0
pos $string
Truth table100%2/2
A
A= pos $string
Aresult
11
00
128 7 ▶ return if $pos == length($string);
Branch: 1 branch
BranchTrueFalseTotal
if $pos == length $string 4 3 7
129  
130 # otherwise return the position we found
131 3 return $pos
132 }
133  
134 =item is_sane_utf8($string, $name)
135  
136 This test fails if the string contains something that looks like it
137 might be dodgy utf8, i.e. containing something that looks like the
138 multi-byte sequence for a latin-1 character but perl hasn't been
139 instructed to treat as such. Strings that are not utf8 always
140 automatically pass.
141  
142 Some examples may help:
143  
144 # This will pass as it's a normal latin-1 string
145 is_sane_utf8("Hello L\x{e9}eon");
146  
147 # this will fail because the \x{c3}\x{a9} looks like the
148 # utf8 byte sequence for e-acute
149 my $string = "Hello L\x{c3}\x{a9}on";
150 is_sane_utf8($string);
151  
152 # this will pass because the utf8 is correctly interpreted as utf8
153 Encode::_utf8_on($string)
154 is_sane_utf8($string);
155  
156 Obviously this isn't a hundred percent reliable. The edge case where
157 this will fail is where you have C<\x{c2}> (which is "LATIN CAPITAL
158 LETTER WITH CIRCUMFLEX") or C<\x{c3}> (which is "LATIN CAPITAL LETTER
159 WITH TILDE") followed by one of the latin-1 punctuation symbols.
160  
161 # a capital letter A with tilde surrounded by smart quotes
162 # this will fail because it'll see the "\x{c2}\x{94}" and think
163 # it's actually the utf8 sequence for the end smart quote
164 is_sane_utf8("\x{93}\x{c2}\x{94}");
165  
166 However, since this hardly comes up this test is reasonably reliable
167 in most cases. Still, care should be applied in cases where dynamic
168 data is placed next to latin-1 punctuation to avoid false negatives.
169  
170 There exists two situations to cause this test to fail; The string
171 contains utf8 byte sequences and the string hasn't been flagged as
172 utf8 (this normally means that you got it from an external source like
173 a C library; When Perl needs to store a string internally as utf8 it
174 does it's own encoding and flagging transparently) or a utf8 flagged
175 string contains byte sequences that when translated to characters
176 themselves look like a utf8 byte sequence. The test diagnostics tells
177 you which is the case.
178  
179 =cut
180  
181 # build my regular expression out of the latin-1 bytes
182 # NOTE: This won't work if our locale is nonstandard will it?
183 7 my $re_bit = join "|", map { Encode::encode("utf8",chr($_)) } (127..255);
184  
185 sub is_sane_utf8($;$)
186 6 ▶{
Subroutine is_sane_utf8: called CC=4 SCAR=13.9
Pod documented
187 6 my $string = shift;
188 6 ▶ my $name = shift || "sane utf8";
Condition100%2/2
shift() || 'sane utf8'
l!l
15
MC/DC100%1/1
shift() || 'sane utf8'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
189  
190 # regexp in scalar context with 'g', meaning this loop will run for
191 # each match. Should only have to run it once, but will redo if
192 # the failing case turns out to be allowed in %allowed.
193 6 ▶ while ($string =~ /($re_bit)/o)
Branch: 1 branch
BranchTrueFalseTotal
if ($string =~ /($re_bit)/o) 2 4 6
194 {
195 # work out what the double encoded string was
196 2 my $bytes = $1;
197  
198 2 my $index = $+[0] - length($bytes);
199 2 my $codes = join '', map { sprintf '<%00x>', ord($_) } split //, $bytes;
200  
201 # what character does that represent?
202 2 my $char = Encode::decode("utf8",$bytes);
203 2 my $ord = ord($char);
204 2 my $hex = sprintf '%00x', $ord;
205 2 $char = charnames::viacode($ord);
206  
207 # print out diagnostic messages
208 2 _fail($name);
209 2 _diag(qq{Found dodgy chars "$codes" at char $index\n});
210 2 ▶ if (Encode::is_utf8($string))
Branch: 1 branch
BranchTrueFalseTotal
if (&Encode::is_utf8($string)) { } 1 1 2
211 1 { _diag("Chars in utf8 string look like utf8 byte sequence.") }
212 else
213 1 { _diag("String not flagged as utf8...was it meant to be?\n") }
214 2 _diag("Probably originally a $char char - codepoint $ord (dec),"
215 ." $hex (hex)\n");
216  
217 2 return 0;
218 }
219  
220 # got this far, must have passed.
221 4 _ok(1,$name);
222 4 return 1;
223 }
224  
225 # historic name of method; deprecated
226 5 ▶sub is_dodgy_utf8 { goto &is_sane_utf8 }
Subroutine is_dodgy_utf8: called CC=1 SCAR=0.0
Pod undocumented
227  
228 =back
229  
230 =head2 String Characteristic Tests
231  
232 These routines allow you to check the range of characters in a string.
233 Note that these routines are blind to the actual encoding perl
234 internally uses to store the characters, they just check if the
235 string contains only characters that can be represented in the named
236 encoding:
237  
238 =over
239  
240 =item is_within_ascii
241  
242 Tests that a string only contains characters that are in the ASCII
243 character set.
244  
245 =cut
246  
247 sub is_within_ascii($;$)
248 3 ▶{
Subroutine is_within_ascii: called CC=3 SCAR=11.0
Pod documented
249 3 my $string = shift;
250 3 ▶ my $name = shift || "within ascii";
Condition100%2/2
shift() || 'within ascii'
l!l
12
MC/DC100%1/1
shift() || 'within ascii'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
251  
252 # look for anything that isn't ascii or pass
253 3 ▶ $string =~ /([^\x{00}-\x{7f}])/ or return _pass($name);
Branch: 1 branch
BranchTrueFalseTotal
$string =~ /([^\x{00}-\x{7f}])/ or return _pass($name) 1 2 3
254  
255 # explain why we failed
256 1 my $dec = ord($1);
257 1 my $hex = sprintf '%02x', $dec;
258  
259 1 _fail($name);
260 1 _diag("Char $+[0] not ASCII (it's $dec dec / $hex hex)");
261  
262 1 return 0;
263 }
264  
265 =item is_within_latin_1
266  
267 Tests that a string only contains characters that are in latin-1.
268  
269 =cut
270  
271 sub is_within_latin_1($;$)
272 4 ▶{
Subroutine is_within_latin_1: called CC=3 SCAR=11.0
Pod documented
273 4 my $string = shift;
274 4 ▶ my $name = shift || "within latin-1";
Condition100%2/2
shift() || 'within latin-1'
l!l
13
MC/DC100%1/1
shift() || 'within latin-1'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
275  
276 # look for anything that isn't ascii or pass
277 4 ▶ $string =~ /([^\x{00}-\x{ff}])/ or return _pass($name);
Branch: 1 branch
BranchTrueFalseTotal
$string =~ /([^\x{00}-\x{ff}])/ or return _pass($name) 1 3 4
278  
279 # explain why we failed
280 1 my $dec = ord($1);
281 1 my $hex = sprintf '%x', $dec;
282  
283 1 _fail($name);
284 1 _diag("Char $+[0] not Latin-1 (it's $dec dec / $hex hex)");
285  
286 1 return 0;
287 }
288  
289 3 ▶sub is_within_latin1 { goto &is_within_latin_1 }
Subroutine is_within_latin1: called CC=1 SCAR=0.0
Pod undocumented
290  
291 =back
292  
293 Simply check if a scalar is or isn't flagged as utf8 by perl's
294 internals:
295  
296 =over
297  
298 =item is_flagged_utf8($string, $name)
299  
300 Passes if the string is flagged by perl's internals as utf8, fails if
301 it's not.
302  
303 =cut
304  
305 sub is_flagged_utf8
306 3 ▶{
Subroutine is_flagged_utf8: called CC=2 SCAR=6.9
Pod documented
307 3 my $string = shift;
308 3 ▶ my $name = shift || "flagged as utf8";
Condition100%2/2
shift() || 'flagged as utf8'
l!l
12
MC/DC100%1/1
shift() || 'flagged as utf8'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
309 3 return _ok(Encode::is_utf8($string),$name);
310 }
311  
312 =item isnt_flagged_utf8($string,$name)
313  
314 The opposite of C<is_flagged_utf8>, passes if and only if the string
315 isn't flagged as utf8 by perl's internals.
316  
317 Note: you can refer to this function as C<isn't_flagged_utf8> if you
318 really want to.
319  
320 =cut
321  
322 sub isnt_flagged_utf8($;$)
323 3 ▶{
Subroutine isnt_flagged_utf8: called CC=2 SCAR=6.9
Pod documented
324 3 my $string = shift;
325 3 ▶ my $name = shift || "not flagged as utf8";
Condition100%2/2
shift() || 'not flagged as utf8'
l!l
12
MC/DC100%1/1
shift() || 'not flagged as utf8'
shift()
Truth table100%2/2
A
A= shift()
Aresult
11
00
326 3 return _ok(!Encode::is_utf8($string), $name);
327 }
328  
329 sub isn::t_flagged_utf8($;$)
330 0 ▶{
Subroutine t_flagged_utf8: not called CC=2 SCAR=17.9
331 0 my $string = shift;
332 0 ▶ my $name = shift || "not flagged as utf8";
Condition0%0/2
shift() || 'not flagged as utf8'
l!l
00
MC/DC0%0/1
shift() || 'not flagged as utf8'
shift()
Truth table0%0/2
A
A= shift()
Aresult
11
00
333 0 return _ok(!Encode::is_utf8($string), $name);
334 }
335  
336 =back
337  
338 =head1 AUTHOR
339  
340 Written by Mark Fowler B<mark@twoshortplanks.com>
341  
342 =head1 COPYRIGHT
343  
344 Copyright Mark Fowler 2004,2012. All rights reserved.
345  
346 This program is free software; you can redistribute it
347 and/or modify it under the same terms as Perl itself.
348  
349 =head1 BUGS
350  
351 None known. Please report any to me via the CPAN RT system. See
352 http://rt.cpan.org/ for more details.
353  
354 =head1 SEE ALSO
355  
356 L<Test::DoubleEncodedEntities> for testing for double encoded HTML
357 entities.
358  
359 =cut
360  
361 ##########
362  
363 # shortcuts for Test::Builder.
364  
365 7 ▶use Test::Builder;
Subroutine BEGIN: called CC=1 SCAR=0.0
366 7 my $tester = Test::Builder->new();
367  
368 sub _ok
369 31 ▶{
Subroutine _ok: called CC=1 SCAR=0.0
370 31 local $Test::Builder::Level = $Test::Builder::Level + 1;
371 31 return $tester->ok(@_)
372 }
373 sub _diag
374 11 ▶{
Subroutine _diag: called CC=1 SCAR=0.0
375 11 local $Test::Builder::Level = $Test::Builder::Level + 1;
376 11 $tester->diag(@_);
377 11 return;
378 }
379  
380 sub _fail
381 4 ▶{
Subroutine _fail: called CC=1 SCAR=0.0
382 4 local $Test::Builder::Level = $Test::Builder::Level + 1;
383 4 return _ok(0,@_)
384 }
385  
386 sub _pass
387 10 ▶{
Subroutine _pass: called CC=1 SCAR=0.0
388 10 local $Test::Builder::Level = $Test::Builder::Level + 1;
389 10 return _ok(1,@_)
390 }
391  
392  
393 7 1;
↑ Summary