| 1 | package Text::Nunjucks::Lexer; | ||
| 2 | # ABSTRACT: 1:1 port of Nunjucks src/lexer.js -- the template tokenizer | ||
| 3 | |||
| 4 | 21 | our $VERSION = '0.001'; | |
| 5 | |||
| 6 | 21 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 7 | 21 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 8 | 21 | ▶ | use Text::Nunjucks::Lexer::Token; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 21 | ▶ | use Text::Nunjucks::Lexer::Tokenizer; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | |||
| 11 | # 1:1 port of upstream src/lexer.js. lexer.js is one file that holds: the `let` | ||
| 12 | # constants (whitespace/delim/int char sets, the six default tag delimiters and | ||
| 13 | # the TOKEN_* type strings), a module-level `token()` factory, the `Tokenizer` | ||
| 14 | # class, and a `module.exports = { lex, TOKEN_* }`. The port splits it along | ||
| 15 | # its classes (ADR 0020): | ||
| 16 | # | ||
| 17 | # * this module is lexer.js's module scope and exports: lex() plus the | ||
| 18 | # TOKEN_* names as class methods, mirroring module.exports so a consumer | ||
| 19 | # reads Text::Nunjucks::Lexer->lex and ->TOKEN_SYMBOL just as parser.js | ||
| 20 | # reads lexer.lex / lexer.TOKEN_SYMBOL. The TOKEN_* strings are file | ||
| 21 | # lexicals here, the single source the exported methods return; | ||
| 22 | # * `Tokenizer` is Text::Nunjucks::Lexer::Tokenizer, which also holds the | ||
| 23 | # char-set and default-tag constants only it reads, and reads the TOKEN_* | ||
| 24 | # strings back through this module's exports; | ||
| 25 | # * `token()` builds a Text::Nunjucks::Lexer::Token. | ||
| 26 | |||
| 27 | #### module constants (upstream: the TOKEN_* `let` bindings at the top of lexer.js) | ||
| 28 | |||
| 29 | 21 | my $TOKEN_STRING = 'string'; | |
| 30 | 21 | my $TOKEN_WHITESPACE = 'whitespace'; | |
| 31 | 21 | my $TOKEN_DATA = 'data'; | |
| 32 | 21 | my $TOKEN_BLOCK_START = 'block-start'; | |
| 33 | 21 | my $TOKEN_BLOCK_END = 'block-end'; | |
| 34 | 21 | my $TOKEN_VARIABLE_START = 'variable-start'; | |
| 35 | 21 | my $TOKEN_VARIABLE_END = 'variable-end'; | |
| 36 | 21 | my $TOKEN_COMMENT = 'comment'; | |
| 37 | 21 | my $TOKEN_LEFT_PAREN = 'left-paren'; | |
| 38 | 21 | my $TOKEN_RIGHT_PAREN = 'right-paren'; | |
| 39 | 21 | my $TOKEN_LEFT_BRACKET = 'left-bracket'; | |
| 40 | 21 | my $TOKEN_RIGHT_BRACKET = 'right-bracket'; | |
| 41 | 21 | my $TOKEN_LEFT_CURLY = 'left-curly'; | |
| 42 | 21 | my $TOKEN_RIGHT_CURLY = 'right-curly'; | |
| 43 | 21 | my $TOKEN_OPERATOR = 'operator'; | |
| 44 | 21 | my $TOKEN_COMMA = 'comma'; | |
| 45 | 21 | my $TOKEN_COLON = 'colon'; | |
| 46 | 21 | my $TOKEN_TILDE = 'tilde'; | |
| 47 | 21 | my $TOKEN_PIPE = 'pipe'; | |
| 48 | 21 | my $TOKEN_INT = 'int'; | |
| 49 | 21 | my $TOKEN_FLOAT = 'float'; | |
| 50 | 21 | my $TOKEN_BOOLEAN = 'boolean'; | |
| 51 | 21 | my $TOKEN_NONE = 'none'; | |
| 52 | 21 | my $TOKEN_SYMBOL = 'symbol'; | |
| 53 | 21 | my $TOKEN_SPECIAL = 'special'; | |
| 54 | 21 | my $TOKEN_REGEX = 'regex'; | |
| 55 | |||
| 56 | #### module.exports.lex -- lex(src, opts) { return new Tokenizer(src, opts); } | ||
| 57 | |||
| 58 | 9727 | ▶ | sub lex { |
|
Subroutine
lex: called CC=1 SCAR=1.2
Pod
undocumented
| |||
| 59 | 0 | ▶ | my ( $class, $src, $opts ) = @_; |
|
Subroutine
lex: not called CC=1 SCAR=1.2
Pod
undocumented
| |||
| 60 | 0 | return $class->tokenizer_class->new( $src, $opts ); | |
| 61 | } | ||
| 62 | |||
| 63 | # Named once so the collaborator is reachable through the invocant (house rule). | ||
| 64 | 9727 | ▶ | sub tokenizer_class { 'Text::Nunjucks::Lexer::Tokenizer' } |
|
Subroutine
tokenizer_class: called CC=1 SCAR=0.0
Pod
undocumented
| |||
| 65 | |||
| 66 | #### module.exports.TOKEN_* -- the type strings as class methods | ||
| 67 | |||
| 68 | # Mirrors the `TOKEN_STRING: TOKEN_STRING, ...` half of module.exports: install a | ||
| 69 | # same-named method on this package for each type string, so a consumer reads | ||
| 70 | # Text::Nunjucks::Lexer->TOKEN_SYMBOL exactly as parser.js reads | ||
| 71 | # lexer.TOKEN_SYMBOL. The lexical above stays the single internal source. | ||
| 72 | { | ||
| 73 | 21 | ▶ | no strict 'refs'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 74 | 21 | for my $entry ( | |
| 75 | [ TOKEN_STRING => $TOKEN_STRING ], | ||
| 76 | [ TOKEN_WHITESPACE => $TOKEN_WHITESPACE ], | ||
| 77 | [ TOKEN_DATA => $TOKEN_DATA ], | ||
| 78 | [ TOKEN_BLOCK_START => $TOKEN_BLOCK_START ], | ||
| 79 | [ TOKEN_BLOCK_END => $TOKEN_BLOCK_END ], | ||
| 80 | [ TOKEN_VARIABLE_START => $TOKEN_VARIABLE_START ], | ||
| 81 | [ TOKEN_VARIABLE_END => $TOKEN_VARIABLE_END ], | ||
| 82 | [ TOKEN_COMMENT => $TOKEN_COMMENT ], | ||
| 83 | [ TOKEN_LEFT_PAREN => $TOKEN_LEFT_PAREN ], | ||
| 84 | [ TOKEN_RIGHT_PAREN => $TOKEN_RIGHT_PAREN ], | ||
| 85 | [ TOKEN_LEFT_BRACKET => $TOKEN_LEFT_BRACKET ], | ||
| 86 | [ TOKEN_RIGHT_BRACKET => $TOKEN_RIGHT_BRACKET ], | ||
| 87 | [ TOKEN_LEFT_CURLY => $TOKEN_LEFT_CURLY ], | ||
| 88 | [ TOKEN_RIGHT_CURLY => $TOKEN_RIGHT_CURLY ], | ||
| 89 | [ TOKEN_OPERATOR => $TOKEN_OPERATOR ], | ||
| 90 | [ TOKEN_COMMA => $TOKEN_COMMA ], | ||
| 91 | [ TOKEN_COLON => $TOKEN_COLON ], | ||
| 92 | [ TOKEN_TILDE => $TOKEN_TILDE ], | ||
| 93 | [ TOKEN_PIPE => $TOKEN_PIPE ], | ||
| 94 | [ TOKEN_INT => $TOKEN_INT ], | ||
| 95 | [ TOKEN_FLOAT => $TOKEN_FLOAT ], | ||
| 96 | [ TOKEN_BOOLEAN => $TOKEN_BOOLEAN ], | ||
| 97 | [ TOKEN_NONE => $TOKEN_NONE ], | ||
| 98 | [ TOKEN_SYMBOL => $TOKEN_SYMBOL ], | ||
| 99 | [ TOKEN_SPECIAL => $TOKEN_SPECIAL ], | ||
| 100 | [ TOKEN_REGEX => $TOKEN_REGEX ] | ||
| 101 | ) { | ||
| 102 | 546 | my ( $name, $value ) = @$entry; | |
| 103 | 3271925 | ▶ | *{ __PACKAGE__ . '::' . $name } = sub { $value }; |
|
Subroutine
__ANON__: called CC=1 SCAR=0.0
| |||
| 104 | } | ||
| 105 | } | ||
| 106 | |||
| 107 | 21 | 1; | |
| 108 | |||
| 109 | __END__ | ||