TT2/Tokenizer.pm

filter: Statementstmt 97.3% 148 / 152 Branchbran 89.0% 57 / 64 Conditioncond 61.1% 11 / 18 MC/DCmcdc 30.0% 3 / 10 Subroutinesub 100% 7 / 7 Podpod 0.0% 0 / 2 totaltotal 89.3% 226 / 253 CC 42 SCAR 28.0
CC
42
Cov
92%
_tokenize_expression
22.0
tokenize
15.2
new
10.9
CRAP 16.5
« TT2/Emitter.pm ↑ Summary TT2/Filters.pm »
1 package Jinja2::TT2::Tokenizer;
2  
3 3 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 3 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 3 ▶use v5.20;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 # Token types
8 3 ▶use constant {
Subroutine BEGIN: called CC=1 SCAR=0.0
9 3 TOK_TEXT => 'TEXT',
10 TOK_VAR_START => 'VAR_START',
11 TOK_VAR_END => 'VAR_END',
12 TOK_STMT_START => 'STMT_START',
13 TOK_STMT_END => 'STMT_END',
14 TOK_COMMENT => 'COMMENT',
15 TOK_EXPR => 'EXPR',
16 TOK_NAME => 'NAME',
17 TOK_STRING => 'STRING',
18 TOK_NUMBER => 'NUMBER',
19 TOK_OPERATOR => 'OPERATOR',
20 TOK_PIPE => 'PIPE',
21 TOK_DOT => 'DOT',
22 TOK_COMMA => 'COMMA',
23 TOK_COLON => 'COLON',
24 TOK_LPAREN => 'LPAREN',
25 TOK_RPAREN => 'RPAREN',
26 TOK_LBRACKET => 'LBRACKET',
27 TOK_RBRACKET => 'RBRACKET',
28 TOK_LBRACE => 'LBRACE',
29 TOK_RBRACE => 'RBRACE',
30 TOK_ASSIGN => 'ASSIGN',
31 TOK_TILDE => 'TILDE',
32 TOK_EOF => 'EOF',
33 3 };
34  
35 # Jinja2 keywords
36 3 my %KEYWORDS = map { $_ => 1 } qw(
37 if elif else endif
38 for in endfor
39 block endblock
40 extends include import from
41 macro endmacro call endcall
42 filter endfilter
43 set
44 raw endraw
45 with endwith
46 autoescape endautoescape
47 and or not
48 is
49 true false True False none None
50 recursive
51 as
52 ignore missing
53 without context
54 scoped
55 );
56  
57 3 ▶sub new {
Subroutine new: called CC=7 SCAR=23.8
Pod undocumented
58 3 my ($class, %opts) = @_;
59 return bless {
60 # Jinja2 delimiters (can be customized)
61 block_start => $opts{block_start} // '{%',
62 block_end => $opts{block_end} // '%}',
63 variable_start => $opts{variable_start} // '{{',
64 variable_end => $opts{variable_end} // '}}',
65 comment_start => $opts{comment_start} // '{#',
66 3 ▶ comment_end => $opts{comment_end} // '#}',
Condition50%6/12
$opts{"block_start"} // "{%"
l!l
03
$opts{"block_end"} // "%}"
l!l
03
$opts{"variable_start"} // "{{"
l!l
03
$opts{"variable_end"} // "}}"
l!l
03
$opts{"comment_start"} // "{#"
l!l
03
$opts{"comment_end"} // "#}"
l!l
03
MC/DC0%0/1
$opts{"block_start"} // "{%"
$opts{"block_start"}
MC/DC0%0/1
$opts{"block_end"} // "%}"
$opts{"block_end"}
MC/DC0%0/1
$opts{"variable_start"} // "{{"
$opts{"variable_start"}
MC/DC0%0/1
$opts{"variable_end"} // "}}"
$opts{"variable_end"}
MC/DC0%0/1
$opts{"comment_start"} // "{#"
$opts{"comment_start"}
MC/DC0%0/1
$opts{"comment_end"} // "#}"
$opts{"comment_end"}
Truth table50%1/2
A
A= $opts{"block_start"}
Aresult
11
00
Truth table50%1/2
A
A= $opts{"block_end"}
Aresult
11
00
Truth table50%1/2
A
A= $opts{"variable_start"}
Aresult
11
00
Truth table50%1/2
A
A= $opts{"variable_end"}
Aresult
11
00
Truth table50%1/2
A
A= $opts{"comment_start"}
Aresult
11
00
Truth table50%1/2
A
A= $opts{"comment_end"}
Aresult
11
00
67 }, $class;
68 }
69  
70 77 ▶sub tokenize {
Subroutine tokenize: called CC=15 SCAR=27.2
Pod undocumented
71 77 my ($self, $template) = @_;
72  
73 77 my @tokens;
74 77 my $pos = 0;
75 77 my $len = length($template);
76  
77 77 ▶ while ($pos < $len) {
Branch: 1 branch
BranchTrueFalseTotal
if ($pos < $len) 142 77 219
78 142 my $remaining = substr($template, $pos);
79  
80 # Check for comment {# ... #}
81 142 ▶ if ($remaining =~ /^\{#-?\s*(.*?)\s*-?#\}/s) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\{#-?\s*(.*?)\s*-?#\}/su) 3 139 142
82 3 my $comment = $1;
83 3 push @tokens, { type => TOK_COMMENT, value => $comment, pos => $pos };
84 3 $pos += length($&);
85 3 next;
86 }
87  
88 # Check for variable {{ ... }}
89 139 ▶ if ($remaining =~ /^\{\{-?\s*/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\{\{-?\s*/u) 54 85 139
90 54 my $match_start = $&;
91 54 ▶ my $strip_before = ($match_start =~ /-/) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$match_start =~ /-/u ? : 2 52 54
92 54 $pos += length($match_start);
93  
94 54 push @tokens, {
95 type => TOK_VAR_START,
96 value => '{{',
97 strip_before => $strip_before,
98 pos => $pos - length($match_start)
99 };
100  
101 # Tokenize the expression inside
102 54 my ($expr_tokens, $new_pos) = $self->_tokenize_expression(
103 $template, $pos, '}}'
104 );
105 54 push @tokens, @$expr_tokens;
106 54 $pos = $new_pos;
107  
108 # Match closing }}
109 54 $remaining = substr($template, $pos);
110 54 ▶ if ($remaining =~ /^\s*(-?)\}\}/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\s*(-?)\}\}/u) { } 54 0 54
111 54 ▶ my $strip_after = $1 ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$1 ? : 2 52 54
112 54 push @tokens, {
113 type => TOK_VAR_END,
114 value => '}}',
115 strip_after => $strip_after,
116 pos => $pos
117 };
118 54 $pos += length($&);
119 } else {
120 0 die "Unclosed variable tag at position $pos";
121 }
122 54 next;
123 }
124  
125 # Check for statement {% ... %}
126 85 ▶ if ($remaining =~ /^\{%-?\s*/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\{%-?\s*/u) 55 30 85
127 55 my $match_start = $&;
128 55 ▶ my $strip_before = ($match_start =~ /-/) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$match_start =~ /-/u ? : 0 55 55
129 55 $pos += length($match_start);
130  
131 55 push @tokens, {
132 type => TOK_STMT_START,
133 value => '{%',
134 strip_before => $strip_before,
135 pos => $pos - length($match_start)
136 };
137  
138 # Tokenize the statement inside
139 55 my ($stmt_tokens, $new_pos) = $self->_tokenize_expression(
140 $template, $pos, '%}'
141 );
142 55 push @tokens, @$stmt_tokens;
143 55 $pos = $new_pos;
144  
145 # Match closing %}
146 55 $remaining = substr($template, $pos);
147 55 ▶ if ($remaining =~ /^\s*(-?)%\}/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\s*(-?)%\}/u) { } 55 0 55
148 55 ▶ my $strip_after = $1 ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$1 ? : 0 55 55
149 55 push @tokens, {
150 type => TOK_STMT_END,
151 value => '%}',
152 strip_after => $strip_after,
153 pos => $pos
154 };
155 55 $pos += length($&);
156 } else {
157 0 die "Unclosed statement tag at position $pos";
158 }
159 55 next;
160 }
161  
162 # Plain text - find the next tag or end
163 30 my $text_end = length($remaining);
164  
165 # Find position of next tag
166 30 for my $pattern ('{{', '{%', '{#') {
167 90 my $idx = index($remaining, $pattern);
168 90 ▶ if ($idx >= 0 && $idx < $text_end) {
Branch: 1 branch
BranchTrueFalseTotal
if ($idx >= 0 and $idx < $text_end) 29 61 90
Condition100%3/3
$idx >= 0 and $idx < $text_end
!ll&&!rl&&r
56529
MC/DC100%2/2
$idx >= 0 and $idx < $text_end
$idx >= 0 $idx < $text_end
Truth table100%3/3
A and B
A= $idx >= 0B= $idx < $text_end
ABresult
0X0
100
111
169 29 $text_end = $idx;
170 }
171 }
172  
173 30 ▶ if ($text_end > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($text_end > 0) 30 0 30
174 30 my $text = substr($remaining, 0, $text_end);
175 30 push @tokens, { type => TOK_TEXT, value => $text, pos => $pos };
176 30 $pos += $text_end;
177 30 next;
178 }
179  
180 # Fallback - shouldn't happen normally
181 0 $pos++;
182 }
183  
184 77 push @tokens, { type => TOK_EOF, value => '', pos => $pos };
185 77 return @tokens;
186 }
187  
188 109 ▶sub _tokenize_expression {
Subroutine _tokenize_expression: called CC=22 SCAR=30.9
189 109 my ($self, $template, $pos, $end_marker) = @_;
190  
191 109 my @tokens;
192 109 my $len = length($template);
193 109 my $end_re = quotemeta($end_marker);
194 109 $end_re =~ s/\\\}/-?\\\}/; # Allow whitespace control -
195  
196 109 ▶ while ($pos < $len) {
Branch: 1 branch
BranchTrueFalseTotal
if ($pos < $len) 468 0 468
197 468 my $remaining = substr($template, $pos);
198  
199 # Check for end marker (with optional whitespace control)
200 468 ▶ last if $remaining =~ /^\s*-?$end_re/ || $remaining =~ /^\s*$end_re/;
Branch: 1 branch
BranchTrueFalseTotal
if $remaining =~ /^\s*-?$end_re/u or $remaining =~ /^\s*$end_re/u 109 359 468
Condition66%2/3
$remaining =~ /^\s*-?$end_re/u or $remaining =~ /^\s*$end_re/u
l!l&&r!l&&!r
1090359
MC/DC50%1/2
$remaining =~ /^\s*-?$end_re/u or $remaining =~ /^\s*$end_re/u
$remaining =~ /^\s*-?$end_re/u $remaining =~ /^\s*$end_re/u
Truth table66%2/3
A or B
A= $remaining =~ /^\s*-?$end_re/uB= $remaining =~ /^\s*$end_re/u
ABresult
1X1
011
000
201  
202 # Skip whitespace
203 359 ▶ if ($remaining =~ /^(\s+)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^(\s+)/u) 84 275 359
204 84 $pos += length($1);
205 84 next;
206 }
207  
208 # String literals (single or double quoted)
209 275 ▶ if ($remaining =~ /^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')/u) 7 268 275
210 7 push @tokens, { type => TOK_STRING, value => $1, pos => $pos };
211 7 $pos += length($1);
212 7 next;
213 }
214  
215 # Numbers (integer or float)
216 268 ▶ if ($remaining =~ /^(\d+(?:_\d+)*(?:\.\d+)?(?:[eE][+-]?\d+)?)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^(\d+(?:_\d+)*(?:\.\d+)?(?:[eE][+-]?\d+)?)/u) 14 254 268
217 14 push @tokens, { type => TOK_NUMBER, value => $1, pos => $pos };
218 14 $pos += length($1);
219 14 next;
220 }
221  
222 # Multi-character operators
223 254 ▶ if ($remaining =~ /^(==|!=|<=|>=|<|>|\*\*|\/\/|and\b|or\b|not\b|in\b|is\b)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ m[^(==|!=|<=|>=|<|>|\*\*|//|and\b|or\b|not\b|in\b|is\b)]u) 16 238 254
224 16 push @tokens, { type => TOK_OPERATOR, value => $1, pos => $pos };
225 16 $pos += length($1);
226 16 next;
227 }
228  
229 # Single character operators
230 238 ▶ if ($remaining =~ /^([+\-*\/%])/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ m[^([+\-*/%])]u) 1 237 238
231 1 push @tokens, { type => TOK_OPERATOR, value => $1, pos => $pos };
232 1 $pos += length($1);
233 1 next;
234 }
235  
236 # Pipe (filter separator)
237 237 ▶ if ($remaining =~ /^\|/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\|/u) 15 222 237
238 15 push @tokens, { type => TOK_PIPE, value => '|', pos => $pos };
239 15 $pos++;
240 15 next;
241 }
242  
243 # Tilde (string concatenation)
244 222 ▶ if ($remaining =~ /^~/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^~/u) 1 221 222
245 1 push @tokens, { type => TOK_TILDE, value => '~', pos => $pos };
246 1 $pos++;
247 1 next;
248 }
249  
250 # Dot
251 221 ▶ if ($remaining =~ /^\./) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\./u) 9 212 221
252 9 push @tokens, { type => TOK_DOT, value => '.', pos => $pos };
253 9 $pos++;
254 9 next;
255 }
256  
257 # Comma
258 212 ▶ if ($remaining =~ /^,/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^,/u) 6 206 212
259 6 push @tokens, { type => TOK_COMMA, value => ',', pos => $pos };
260 6 $pos++;
261 6 next;
262 }
263  
264 # Colon
265 206 ▶ if ($remaining =~ /^:/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^:/u) 2 204 206
266 2 push @tokens, { type => TOK_COLON, value => ':', pos => $pos };
267 2 $pos++;
268 2 next;
269 }
270  
271 # Assignment
272 204 ▶ if ($remaining =~ /^=(?!=)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^=(?!=)/u) 2 202 204
273 2 push @tokens, { type => TOK_ASSIGN, value => '=', pos => $pos };
274 2 $pos++;
275 2 next;
276 }
277  
278 # Parentheses
279 202 ▶ if ($remaining =~ /^\(/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\(/u) 5 197 202
280 5 push @tokens, { type => TOK_LPAREN, value => '(', pos => $pos };
281 5 $pos++;
282 5 next;
283 }
284 197 ▶ if ($remaining =~ /^\)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\)/u) 5 192 197
285 5 push @tokens, { type => TOK_RPAREN, value => ')', pos => $pos };
286 5 $pos++;
287 5 next;
288 }
289  
290 # Brackets
291 192 ▶ if ($remaining =~ /^\[/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\[/u) 3 189 192
292 3 push @tokens, { type => TOK_LBRACKET, value => '[', pos => $pos };
293 3 $pos++;
294 3 next;
295 }
296 189 ▶ if ($remaining =~ /^\]/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\]/u) 3 186 189
297 3 push @tokens, { type => TOK_RBRACKET, value => ']', pos => $pos };
298 3 $pos++;
299 3 next;
300 }
301  
302 # Braces (for dict literals)
303 186 ▶ if ($remaining =~ /^\{/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\{/u) 2 184 186
304 2 push @tokens, { type => TOK_LBRACE, value => '{', pos => $pos };
305 2 $pos++;
306 2 next;
307 }
308 184 ▶ if ($remaining =~ /^\}/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^\}/u) 2 182 184
309 2 push @tokens, { type => TOK_RBRACE, value => '}', pos => $pos };
310 2 $pos++;
311 2 next;
312 }
313  
314 # Names/identifiers/keywords
315 182 ▶ if ($remaining =~ /^([a-zA-Z_][a-zA-Z0-9_]*)/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remaining =~ /^([a-zA-Z_][a-zA-Z0-9_]*)/u) 182 0 182
316 182 my $name = $1;
317 182 push @tokens, { type => TOK_NAME, value => $name, pos => $pos };
318 182 $pos += length($name);
319 182 next;
320 }
321  
322 # Unknown character - skip
323 0 $pos++;
324 }
325  
326 109 return (\@tokens, $pos);
327 }
328  
329 3 1;
330  
331 __END__
« TT2/Emitter.pm ↑ Summary TT2/Filters.pm »