blib/lib/Go/Tokenize.pm

filter: Statementstmt 76.9% 50 / 65 Branchbran 0.0% 0 / 4 Conditioncond 66.6% 2 / 3 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 88.8% 8 / 9 Podpod 50.0% 1 / 2 totaltotal 71.7% 61 / 85 CC 5 SCAR 19.5
CC
5
Cov
72%
tokenize
20.0
make_re
2.0
BEGIN
1.0
CRAP 7.0
↑ Summary
1 package Go::Tokenize;
2 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 1 ▶use utf8;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 1 require Exporter;
7 1 our @ISA = qw(Exporter);
8 1 our @EXPORT_OK = qw/tokenize/;
9 1 our %EXPORT_TAGS = (
10 all => \@EXPORT_OK,
11 );
12 1 our $VERSION = '0.02';
13  
14 1 ▶use Text::LineNumber;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 1 ▶use C::Tokenize qw!$comment_re!;
Subroutine BEGIN: called CC=1 SCAR=0.0
16  
17 1 our $bt_string_re = qr!`[^`]*`!;
18 1 our $q_string_re = qr!"(\\"|[^"])*"!;
19 1 our $string_re = qr!(?:$bt_string_re|$q_string_re)!;
20  
21 # https://golang.org/ref/spec#Keywords
22  
23 # PAUSE thinks this is package switch without the newline
24 1 our @keywords = qw!
25 break default func interface select
26 case defer go map struct
27 chan else goto package
28 switch
29 const fallthrough if range type
30 continue for import return var
31 !;
32  
33 # https://golang.org/ref/spec#Operators_and_punctuation
34  
35 1 our $operator_re;
36 {
37 # Perl makes an error message "Possible attempt to separate words with
38 # commas at lib/Go/Tokenize.pm line 40." See
39 # https://stackoverflow.com/questions/19573977/
40 1 ▶no warnings 'qw';
Subroutine BEGIN: called CC=1 SCAR=0.0
41 1 my @operators = (qw@
42 + & += &= && == != ( )
43 - | -= |= || < <= [ ]
44 * ^ *= ^= <- > >= { }
45 / << /= <<= ++ = := , ;
46 % >> %= >>= -- ! ... . :
47 &^ &^=
48 @);
49 1 $operator_re = make_re (@operators);
50 }
51  
52 1 our $keyword_re = make_re (@keywords);
53  
54 1 our $integer_re = qr!
55 0
56 |
57 [1-9][0-9_]*
58 |
59 0[bB][01_]+
60 |
61 0[oO]?[0-7_]+
62 |
63 0[xX][0-9a-fA-F_]+
64 !x;
65  
66 1 our $numeric_re = qr!
67 u?int(?:8|16|32|64)?
68 |
69 float(?:32|64)
70 |
71 complex(?:64|128)
72 |
73 byte
74 |
75 rune
76 |
77 uintptr
78 !x;
79  
80 # https://perldoc.perl.org/perlre
81 # https://perldoc.perl.org/perlunicode
82 # https://golang.org/ref/spec#unicode_letter
83 # PropertyValueAliases.txt
84  
85 # https://golang.org/ref/spec#Letters_and_digits
86  
87 1 my $letter = qr!\p{L}|_!;
88  
89 1 our $identifier_re = qr!$letter(?:$letter|\p{Nd})*!;
90  
91 1 our $rune_re = qr!
92 '(?:
93 .
94 |
95 \\u(?:[0-9a-fA-F]{4})
96 |
97 \\U(?:[0-9a-fA-F]{8})
98 |
99 \\o(?:[0-7]{3})
100 |
101 \\x(?:[0-9a-fA-F]{2})
102 |
103 # https://golang.org/ref/spec#escaped_char
104 \\[abfnrtv\\'"]
105 )'!x;
106  
107 1 our $whitespace_re = qr!\x20|\x09|\x0D|\x0A!;
108  
109 1 our $go_re = qr!
110 # Comment must go before everything else.
111 (?<comment>$comment_re)
112 |
113 # String must go before everything except comments.
114 (?<string>$string_re)
115 |
116 (?<keyword>$keyword_re)
117 |
118 (?<operator>$operator_re)
119 |
120 (?<integer>$integer_re)
121 |
122 (?<numeric>$numeric_re)
123 |
124 (?<identifier>$identifier_re)
125 |
126 (?<whitespace>$whitespace_re)
127 !x;
128  
129 1 our @types = (qw!
130 comment
131 identifier
132 integer
133 keyword
134 numeric
135 operator
136 rune
137 string
138 whitespace
139 !);
140  
141 sub tokenize
142 0 ▶{
Subroutine tokenize: not called CC=4 SCAR=30.0
Pod documented
143 0 my ($go) = @_;
144 0 my $tln = Text::LineNumber->new ($go);
145 0 my @tokens;
146 0 ▶ while ($go =~ /($go_re)/g) {
Branch: 1 branch
BranchTrueFalseTotal
if ($go =~ /($Go::Tokenize::go_re)/g) 0 0 0
147 0 my %token;
148 0 $token{contents} = $1;
149 0 $token{end} = pos ($go);
150 0 $token{start} = $token{end} - length ($token{contents}) + 1;
151 0 for my $type (@types) {
152 0 ▶ if ($+{$type}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($+{$type}) 0 0 0
153 0 $token{type} = $type;
154 0 last;
155 }
156 }
157 0 $token{line} = $tln->off2lnr ($token{start});
158 0 push @tokens, \%token;
159 }
160 0 return \@tokens;
161 }
162  
163 sub make_re
164 2 ▶{
Subroutine make_re: called CC=2 SCAR=7.0
Pod undocumented
165 2 ▶ my @sorted = sort {length ($b) <=> length ($a) || $a cmp $b} @_;
Condition66%2/3
length $b <=> length $a || $a cmp $b
l!l&&r!l&&!r
1201760
MC/DC0%0/2
length $b <=> length $a || $a cmp $b
length $b <=> length $a $a cmp $b
Truth table66%2/3
A || B
A= length $b <=> length $aB= $a cmp $b
ABresult
1X1
011
000
166 2 my @quoted = map {quotemeta ($_)} @sorted;
167 2 my $re = join '|', @quoted;
168 2 return $re;
169 }
170  
171 1 1;
↑ Summary