CursorBase.pm untested

filter: Statementstmt 0.0% 0 / 2116 Branchbran 0.0% 0 / 804 Conditioncond 0.0% 0 / 282 MC/DCmcdc 0.0% 0 / 170 Subroutinesub 0.0% 0 / 234 Podpod 0.0% 0 / 173 totaltotal 0.0% 0 / 3779 CC 403 SCAR 48.3
CC
403
Cov
0%
check_old_cclass
1332.0
add_categorical
462.0
_AUTOLEXgenDFA
380.0
CRAP 124.8
1 #!perl
2  
3 # CursorBase.pmc
4 #
5 # Copyright 2008-2010, Larry Wall
6 #
7 # You may copy this software under the terms of the Artistic License,
8 # version 2.0 or later.
9  
10 use strict;
11 use warnings;
12 no warnings 'recursion';
13 use utf8;
14 use NAME;
15 use Stash;
16 use RE_ast;
17 #use Carp::Always;
18 use File::Spec ();
19 use File::Basename ();
20 use File::ShareDir ();
21 use File::Path ();
22 use Config;
23  
24 my $TRIE = 1;
25 my $STORABLE = 1;
26  
27 use feature 'say', 'state';
28  
29 require 'mangle.pl';
30  
31 our $CTX = '';
32 BEGIN {
33 $::DEBUG //= 0 + ($ENV{STD5DEBUG} // 0);
34 }
35 our $DEBUG;
36 use constant DEBUG => $::DEBUG;
37 our %LEXERS; # per language, the cache of lexers, keyed by rule identity
38 our %FATECACHE; # fates we've already turned into linked lists
39 my %lexer_cache = ();
40  
41 sub ::fatestr { my $f = shift;
42 my $text = '';
43 while ($f) {
44 $text .= $f->[1] . " " . $f->[2];
45 $text .= ' ' if $f = $f->[0];
46 }
47 $text;
48 }
49  
50 use DEBUG;
51  
52 sub ::deb {
53 print ::LOG @_, "\n";
54 }
55  
56 package CursorBase;
57  
58 use Carp;
59 use File::Copy;
60 use YAML::XS;
61 use Storable;
62 use Encode;
63 use Scalar::Util 'refaddr';
64 use Try::Tiny;
65  
66 use Term::ANSIColor;
67 our $BLUE = color 'blue';
68 our $GREEN = color 'green';
69 our $CYAN = color 'cyan';
70 our $MAGENTA = color 'magenta';
71 our $YELLOW = color 'yellow';
72 our $RED = color 'red';
73 our $CLEAR = color 'clear';
74  
75 use STD::LazyMap qw(lazymap eager);
76 use constant DEBUG => $::DEBUG;
77  
78 our $REGEXES = { ALL => [] };
79  
80 BEGIN {
81 require Moose;
82 # this prevents us from inheriting from Moose::Object, which saves a
83 # good 20 seconds on DESTROY/DEMOLISHALL
84 Moose::Meta::Class->create('CursorBase');
85 }
86  
87 our $data_dir;
88 BEGIN {
89 # 2 possibilities:
90 # * STD is installed. CursorBase will be in the system @INC somewhere,
91 # with nothing but other modules besides it. Use File::ShareDir.
92 # * STD is being used in place. uniprops will be in the same dir as
93 # CursorBase.pm
94  
95 # yes, the INC entry always uses forward slashes and ends in .pm, always
96 my $path = __PACKAGE__;
97 $path =~ s#::#/#g;
98  
99 $data_dir = File::Basename::dirname($INC{"$path.pm"});
100 if (! -e File::Spec->catfile($data_dir, "uniprops")) {
101 $data_dir = File::ShareDir::dist_dir('STD');
102 }
103 }
104 $::PERL6HERE = $ENV{PERL6HERE} // '⏏';
105 Encode::_utf8_on($::PERL6HERE);
106  
107 binmode(STDIN, ":utf8");
108 binmode(STDERR, ":utf8");
109 binmode(STDOUT, ":utf8");
110 BEGIN {
111 if ($^P || !DEBUG) {
112 open(::LOG, ">&1") or die "Can't create $0.log: $!";
113 }
114 else {
115 open(::LOG, ">$0.log") or die "Can't create $0.log: $!";
116 }
117 binmode(::LOG, ":utf8");
118 }
119  
120 #############################################################
121 # STD::Cursor Accessors
122 #############################################################
123  
124 sub _PARAMS {} # overridden in parametric role packages
125  
126 sub from :lvalue { $_[0]->{_from} //= $_[0]->{_pos} }
127 sub to { $_[0]->{_pos} }
128 sub pos :lvalue { $_[0]->{_pos} }
129 sub chars { $_[0]->{_pos} - ($_[0]->{_from} // $_[0]->{_pos}) }
130 sub Str { no warnings; exists $_[0]->{_from} && defined $_[0]->{_pos} ? substr($::ORIG, $_[0]->{_from}, $_[0]->{_pos} - $_[0]->{_from})//'' : '' }
131 sub xact { $_[0]->{_xact} // die "internal error: cursor has no xact!!!" }
132 sub orig { \$::ORIG }
133 sub WHAT { ref $_[0] || $_[0] }
134  
135 sub item { $_[0] }
136 sub caps { $_[0] && $_[0]->{'~CAPS'} ? @{$_[0]->{'~CAPS'}} : () }
137 sub chunks { die "unimpl" }
138 sub ast { exists $_[0]->{'_ast'} ? $_[0]->{'_ast'} : $_[0]->Str }
139 sub make { $_[0]->{'_ast'} = $_[1]; $_[0] }
140  
141 sub label_id {
142 bless { 'file' => $::FILE->{name}, 'pos' => $_[0]->{_pos} }, 'LABEL';
143 }
144  
145 sub list { my $self = shift;
146 my @result;
147 # can't just do this in numerical order because some might be missing
148 # and we don't know the max
149 for my $k (keys %$self) {
150 $result[$k] = $self->{$k} if $k =~ /^\d/;
151 }
152 \@result;
153 }
154  
155 sub hash { my $self = shift;
156 my %result;
157 for my $k (keys %$self) {
158 $result{$k} = $self->{$k} if $k !~ /^[_\d~]/;
159 }
160 \%result;
161 }
162  
163 sub deb { my $self = shift;
164 my $pos = ref $self && defined $self->{_pos} ? $self->{_pos} : "?";
165 print ::LOG $pos,'/',$self->lineof($pos), "\t", $CTX, ' ', @_, "\n";
166 }
167  
168 sub clean {
169 my $self = shift;
170 delete $self->{_fate};
171 delete $self->{_pos}; # EXPR blows up without this for some reason
172 delete $self->{_reduced};
173 for my $k (values %$self) {
174 next unless ref $k;
175 if (ref $k eq 'ARRAY') {
176 for my $k2 (@$k) {
177 eval {
178 $k2->clean if ref $k2;
179 }
180 }
181 }
182 else {
183 eval {
184 $k->clean;
185 }
186 }
187 }
188 $self;
189 }
190  
191 sub dump {
192 my $self = shift;
193 my %copy = %$self;
194 my $text = YAML::XS::Dump(\%copy);
195 $text;
196 }
197  
198 #############################################################
199 # System interface
200 #############################################################
201  
202 sub sys_compile_module {
203 my ($self, $module, $symlfile, $modfile) = @_;
204  
205 # should be per-module
206 local $STD::ALL;
207 local @STD::herestub;
208 local $::SETTING;
209 local $::CORE;
210 local $::GLOBAL;
211 local $::UNIT;
212 local $::YOU_WERE_HERE;
213  
214 # should only apply to the toplevel
215 local $::moreinput;
216  
217 # STD is missing contextualizers for these, at least in some cases
218 local $::CCSTATE;
219 local $::BORG;
220 local %::RX;
221 local $::XACT;
222 local $::IN_REDUCE;
223 local $::VAR;
224  
225 $self->sys_do_compile_module($module, $symlfile, $modfile);
226 }
227  
228 sub sys_do_compile_module {
229 my ($self, $mod, $syml, $file) = @_;
230  
231 $self->parsefile($file,
232 setting => $ENV{DEFAULT_SETTING_FOR_MODULES} // "CORE",
233 syml_search_path => $::SYML_SEARCH_PATH,
234 tmp_prefix => $::TMP_PREFIX);
235  
236 print STDERR "Compiled $file\n";
237 }
238  
239 sub sys_save_syml {
240 my ($self, $all) = @_;
241  
242 my $file = $::FILE->{name};
243 $file = $::UNIT->{'$?LONGNAME'};
244 my @toks = split '::', $file;
245 $toks[-1] .= '.syml';
246 $file = File::Spec->catfile($::TMP_PREFIX, "syml", @toks);
247 pop @toks;
248 my $path = File::Spec->catdir($::TMP_PREFIX, "syml", @toks);
249  
250 File::Path::mkpath($path);
251  
252 open(SETTING, ">", $file) or die "Can't open new setting file $file: $!";
253 print SETTING Dump($all);
254 close SETTING;
255 }
256  
257 sub sys_get_perl6lib {
258 my $self = shift;
259  
260 if (not @::PERL6LIB) {
261 if ($ENV{PERL6LIB} && !$CursorBase::NOPERL6LIB) {
262 @::PERL6LIB = split /\Q$Config::Config{path_sep}/, $ENV{PERL6LIB};
263 } else {
264 @::PERL6LIB = qw< ./lib . >;
265 }
266 }
267  
268 @::PERL6LIB;
269 }
270  
271 sub sys_find_module {
272 my $self = shift;
273 my $module = shift;
274 my $issetting = shift;
275  
276 my @toks = split '::', $module;
277 my $end = pop @toks;
278  
279 for my $d ($self->sys_get_perl6lib) {
280 for my $ext (qw( .setting .pm6 .pm )) {
281 next if ($issetting xor ($ext eq '.setting'));
282  
283 my $file = File::Spec->catfile($d, @toks, "$end$ext");
284 next unless -f $file;
285  
286 if ($ext eq '.pm') {
287 local $/;
288 open my $pm, "<", $file or next;
289 my $pmtx = <$pm>;
290 close $pm;
291 next if $pmtx =~ /^\s*package\s+\w+\s*;/m; # ignore p5 code
292 }
293  
294 return $file;
295 }
296 }
297  
298 $self->sorry("Can't locate module $module");
299 return;
300 }
301  
302 sub sys_load_modinfo {
303 my $self = shift;
304 my $module = shift;
305  
306 my @toks = split '::', $module;
307 $toks[-1] .= ".syml";
308  
309 my @extra = @$::SYML_SEARCH_PATH;
310 shift @extra;
311 for my $prefix (@extra) {
312 my $file = File::Spec->catfile($prefix, 'syml', @toks);
313 if (-e $file) {
314 return LoadFile($file);
315 }
316 }
317  
318 my ($symlfile) = File::Spec->catfile($::TMP_PREFIX, 'syml', @toks);
319 my ($modfile) = $self->sys_find_module($module, 0)
320 or return undef;
321  
322 unless (-f $symlfile and -M $modfile > -M $symlfile) {
323 $self->sys_compile_module($module, $symlfile, $modfile);
324 }
325 return LoadFile($symlfile);
326 }
327  
328 sub load_lex {
329 my $self = shift;
330 my $setting = shift;
331  
332 if ($setting eq 'NULL') {
333 my $id = "MY:file<NULL.pad>:line(1):pos(0)";
334 my $core = Stash->new('!id' => [$id], '!file' => 'NULL.pad',
335 '!line' => 1);
336 return Stash->new('CORE' => $core, 'MY:file<NULL.pad>' => $core,
337 'SETTING' => $core, $id => $core);
338 }
339  
340 for my $prefix (@{$::SYML_SEARCH_PATH}) {
341 my $file = File::Spec->catfile($prefix, 'syml', "$setting.syml");
342 if (-e $file) {
343 return bless($self->_load_yaml_lex($setting,$file),'Stash');
344 }
345 }
346  
347 die "Unable to load setting $setting. Did you run make?";
348 }
349  
350 sub _load_yaml_lex {
351 my $self = shift;
352 my $setting = shift;
353 my $file = shift;
354 state %LEXS;
355 return $LEXS{$setting} if $LEXS{$setting};
356 # HACK YAML::XS is horribly broken see https://rt.cpan.org/Public/Bug/Display.html?id=53278
357 $LEXS{$setting} = {%{LoadFile($file)}};
358 # say join ' ', sort keys %{ $LEXS{$setting} };
359 $LEXS{$setting};
360 }
361  
362 sub LoadFile {
363 my $file = shift;
364 open my $fh, $file or die "Can't open $file: $!";
365 my $text = do { local $/; <$fh>; };
366 close $fh;
367 Load($text);
368 }
369  
370 #############################################################
371 # Setup/Teardown
372 #############################################################
373  
374 sub new {
375 my $class = shift;
376 # $::ORIG = shift;
377 { no warnings; @::ORIG = unpack("U*", $::ORIG); }
378 $::MEMOS[@::ORIG] = undef; # memos kept by position
379 my %args = ('_pos' => 0, '_from' => 0);
380 while (@_) {
381 my $name = shift;
382 $args{'_' . $name} = shift;
383 }
384 my $self = bless \%args, ref $class || $class;
385 $self->{_xact} = ['MATCH',0,0];
386 $self;
387 }
388  
389 sub parse {
390 my $class = shift;
391 my $text = shift;
392 my %args = @_;
393 local $::FILE = { name => $args{'filename'} // '(eval)' };
394 $class->initparse($text,@_);
395 }
396  
397 sub parsefile {
398 my $class = shift;
399 my $file = shift;
400 my %args = @_;
401 $file =~ s/::/\//g;
402 local $::FILE = { name => $file };
403 open(FILE, '<:utf8', $file) or die "Can't open $file: $!\n";
404 my $text;
405 {
406 local $/;
407 $text = <FILE>;
408 close FILE;
409 }
410  
411 my $result;
412 $result = $class->initparse($text,@_,filename => $file);
413  
414 $result;
415 }
416  
417 ## method initparse ($text, :$rule = 'TOP', :$tmp_prefix = '', :$setting = 'CORE', :$actions = '')
418 sub initparse {
419 my $self = shift;
420 my $text = shift;
421 my %args = @_;
422 my $rule = $args{rule} // 'TOP';
423 my $tmp_prefix = $args{tmp_prefix} // $CursorBase::SET_STD5PREFIX // $ENV{STD5PREFIX} // '.';
424 my $setting = $args{setting} // 'CORE';
425 my $actions = $args{actions} // '';
426 my $filename = $args{filename};
427  
428 local $::TMP_PREFIX = $tmp_prefix;
429 local $::SYML_SEARCH_PATH = $args{syml_search_path} //
430 $CursorBase::NOSTDSYML ? [$::TMP_PREFIX] : [$::TMP_PREFIX, $data_dir];
431  
432 local $::SETTINGNAME = $setting;
433 local $::ACTIONS = $actions;
434 local $::RECURSIVE_PERL = $args{recursive_perl};
435 local @::MEMOS = ();
436  
437 local @::ACTIVE = ();
438  
439 # various bits of info useful for error messages
440 local $::HIGHWATER = 0;
441 local $::HIGHMESS = '';
442 local $::HIGHEXPECT = {};
443 local $::LASTSTATE;
444 local $::LAST_NIBBLE = bless { firstline => 0, lastline => 0 }, 'STD::Cursor';
445 local $::LAST_NIBBLE_MULTILINE = bless { firstline => 0, lastline => 0 }, 'STD::Cursor';
446 local $::GOAL = "(eof)";
447 $text .= "\n" unless substr($text,-1,1) eq "\n";
448 local $::ORIG = $text; # original string
449 local @::ORIG;
450  
451 # This isn't the way it should work; each parse node should instead carry
452 # a reference to the string. But that's a bit tricky to make work right
453 # in perl 5.
454 if ($args{text_return}) {
455 ${$args{text_return}} = $text;
456 }
457  
458 my $result = $self->new()->$rule();
459 delete $result->{_xact};
460  
461 # XXX here attach stuff that will enable :cont
462 if ($::YOU_WERE_HERE) {
463 $result->you_were_here;
464 }
465 elsif ($args{filename} && $args{filename} =~ /\.pm6?$/) {
466 $result->you_were_here;
467 }
468  
469 $result;
470 }
471  
472 sub you_are_here {
473 my $self = shift;
474 $::YOU_WERE_HERE = $::CURLEX;
475 $self;
476 }
477  
478 sub you_were_here {
479 my $self = shift;
480 my $all;
481 # setting?
482 if ($::YOU_WERE_HERE) {
483 $all = $STD::ALL;
484 $all->{SETTING} = $::YOU_WERE_HERE;
485 $all->{CORE} = $::YOU_WERE_HERE if $::UNIT->{'$?LONGNAME'} eq 'CORE';
486 }
487 else {
488 eval { $::UNIT->{'$?SETTING_ID'} = $STD::ALL->{SETTING}->id };
489 warn $@ if $@;
490 eval { $::UNIT->{'$?CORE_ID'} = $STD::ALL->{CORE}->id };
491 warn $@ if $@;
492  
493 $all = {};
494 for my $key (keys %{$STD::ALL}) {
495 next if $key =~ /^MY:file<\w+\.setting>/ or $key eq 'CORE' or $key eq 'SETTING';
496 $all->{$key} = $STD::ALL->{$key};
497 }
498 }
499  
500 $self->sys_save_syml($all);
501 $self;
502 }
503  
504 sub delete {
505 my $self = shift;
506 delete $self->{@_};
507 }
508  
509 { package Match;
510 sub new { my $self = shift;
511 my %args = @_;
512 bless \%args, $self;
513 }
514  
515 sub from { my $self = shift;
516 $self->{_f};
517 }
518  
519 sub to { my $self = shift;
520 $self->{_t};
521 }
522 }
523  
524 #############################################################
525 # STD::Cursor transformations
526 #############################################################
527  
528 sub cursor_xact {
529 if (DEBUG & DEBUG::cursors) {
530 my $self = shift;
531 my $name = shift;
532 my $pedigree = '';
533 for (my $x = $self->{_xact}; $x; $x = $x->[-1]) {
534 my $n = $x->[0];
535 $n =~ s/^RULE // or
536 $n =~ s/^ALT *//;
537 $pedigree .= ($x->[-2] ? " - " : " + ") . $n;
538 }
539 $self->deb("cursor_xact $name$pedigree");
540 $self->{_xact} = [$name,0,$self->{_xact}];
541 return $self;
542 }
543 # doing this in place is slightly dangerous, but seems to work
544 $_[0]->{_xact} = [$_[1],0,$_[0]->{_xact}];
545 return $_[0];
546 }
547  
548 sub cursor_fresh { my $self = shift;
549 my %r;
550 my $lang = @_ && $_[0] ? shift() : ref $self;
551 $self->deb("cursor_fresh lang $lang") if DEBUG & DEBUG::cursors;
552 @r{'_pos','_fate','_xact'} = @$self{'_pos','_fate','_xact'};
553 $r{_herelang} = $self->{_herelang} if $self->{_herelang};
554 bless \%r, ref $lang || $lang;
555 }
556  
557 sub cursor_herelang { my $self = shift;
558 $self->deb("cursor_herelang") if DEBUG & DEBUG::cursors;
559 my %r = %$self;
560 $r{_herelang} = $self;
561 bless \%r, 'STD::Q';
562 }
563  
564 sub prepbind {
565 my $self = shift;
566 delete $self->{_fate};
567 delete $_->{_xact} for @_;
568 $self;
569 }
570  
571 sub cursor_bind { my $self = shift; # this is parent's match cursor
572 my $bindings = shift;
573 my $submatch = shift; # this is the submatch's cursor
574 $self->prepbind($submatch);
575  
576 $self->deb("cursor_bind @$bindings") if DEBUG & DEBUG::cursors;
577 my @caps;
578 @caps = @{$self->{'~CAPS'}} if $self->{'~CAPS'}; # must copy elems
579 my %r = %$self;
580 if ($bindings) {
581 for my $binding (@$bindings) {
582 if (ref $r{$binding} eq 'ARRAY') {
583 push(@{$r{$binding}}, $submatch);
584 }
585 else {
586 $r{$binding} = $submatch;
587 }
588 next if $binding eq 'PRE';
589 next if $binding eq 'POST';
590 push @caps, $binding, $submatch;
591 }
592 $r{'~CAPS'} = \@caps;
593 }
594 $submatch->{_from} = $r{_from} = $r{_pos};
595 $r{_pos} = $submatch->{_pos};
596 $r{_xact} = $self->{_xact};
597 bless \%r, ref $self; # return new match cursor for parent
598 }
599  
600 sub cursor_fate { my $self = shift;
601 my $pkg = shift;
602 my $name = shift;
603 my $retree = shift;
604 # $_[0] is now ref to a $trystate;
605  
606 $self->deb("cursor_fate $pkg $name") if DEBUG & DEBUG::cursors;
607 my $key = refaddr($retree->{$name}) // $name;
608  
609 my $lexer = $::LEXERS{ref $self}->{$key} // do {
610 local %::AUTOLEXED;
611 $self->_AUTOLEXpeek($name,$retree);
612 };
613 if ($self->{_pos} >= $::HIGHWATER) {
614 if ($self->{_pos} > $::HIGHWATER) {
615 %$::HIGHEXPECT = ();
616 $::HIGHMESS = '';
617 }
618 $::HIGHEXPECT->{$lexer->{DBA}}++;
619 $::HIGHWATER = $self->{_pos};
620 }
621  
622 my $P = $self->{_pos};
623 if ($P > @::ORIG) {
624 return sub {};
625 }
626  
627 $self->cursor_fate_dfa($pkg, $name, $lexer, $P);
628 }
629  
630 sub cursor_fate_dfa {
631 my ($self, $pkg, $name, $lexer, $P) = @_;
632  
633 my $state = $lexer->{S};
634 my $p = $P;
635 my @rfates;
636  
637 print ::LOG "=" x 10,"\n$p DFA for ${pkg}::$name in ", ref $self, "\n" if DEBUG & DEBUG::autolexer;
638 CH: {
639 push @rfates, @{ $state->[0] // _jit_dfa_node($lexer, $state) };
640 if (DEBUG & DEBUG::autolexer) {
641 for (@{ $state->[0] }) {
642 my @b;
643 for (my $f = $_; $f; $f = $f->[0]) {
644 push @b, @{$f}[1,2];
645 }
646 print ::LOG " [adding fate @b]\n";
647 }
648 }
649 last if $p == @::ORIG;
650 my $chi = $::ORIG[$p++];
651 print ::LOG "--- ", pack("U", $chi), "\n" if DEBUG & DEBUG::autolexer;
652 if ($state->[1]{$chi}) {
653 $state = $state->[1]{$chi};
654 print ::LOG "specific -> ", $state->[1]{ID}, "\n"
655 if DEBUG & DEBUG::autolexer;
656 redo;
657 }
658  
659 my $dt = $state->[2];
660 while (defined $dt) {
661 if (ref $dt eq 'ARRAY') {
662 if (DEBUG & DEBUG::autolexer) {
663 print ::LOG $dt->[2][-1],
664 (vec($dt->[2][$chi >> 10], $chi & 1023, 1) ?
665 "? yes\n" : "? no\n");
666 }
667 $dt = $dt->[vec($dt->[2][$chi >> 10], $chi & 1023, 1)];
668 } else {
669 print ::LOG " -> ", $$dt->[1]{ID}, "\n" if DEBUG & DEBUG::autolexer;
670 $state = $state->[1]{$chi} = $$dt;
671 redo CH;
672 }
673 }
674 }
675  
676 sub { @rfates ? pop(@rfates) : () };
677 }
678  
679 sub cursor_all { my $self = shift;
680 my $fpos = shift;
681 my $tpos = shift;
682  
683 $self->deb("cursor_all from $fpos to $tpos") if DEBUG & DEBUG::cursors;
684 my %r = %$self;
685 @r{'_from','_pos'} = ($fpos,$tpos);
686  
687 bless \%r, ref $self;
688 }
689  
690 sub makestr { my $self = shift;
691 $self->deb("maketext @_") if DEBUG & DEBUG::cursors;
692 my %r = @_;
693  
694 bless \%r, "Str";
695 }
696  
697 sub cursor_tweak { my $self = shift;
698 my $tpos = shift;
699  
700 if (DEBUG & DEBUG::cursors) {
701 my $peek = substr($::ORIG,$tpos,20);
702 $peek =~ s/\n/\\n/g;
703 $peek =~ s/\t/\\t/g;
704 $self->deb("cursor to $tpos --------->$GREEN$peek$CLEAR");
705 }
706 $self->{_pos} = $tpos;
707 return () if $tpos > @::ORIG;
708  
709 $self;
710 }
711  
712 sub cursor_incr { my $self = shift;
713 my $tpos = $self->{_pos} + 1;
714  
715 $self->panic("Unexpected EOF") if $tpos > length($::ORIG);
716 if (DEBUG & DEBUG::cursors) {
717 my $peek = substr($::ORIG,$tpos,20);
718 $peek =~ s/\n/\\n/g;
719 $peek =~ s/\t/\\t/g;
720 $self->deb("cursor to $tpos --------->$GREEN$peek$CLEAR");
721 }
722 $self->{_pos} = $tpos;
723 return () if $tpos > @::ORIG;
724  
725 $self;
726 }
727  
728 sub cursor { my $self = shift;
729 my $tpos = shift;
730  
731 $self->panic("Unexpected EOF") if $tpos > length($::ORIG);
732 if (DEBUG & DEBUG::cursors) {
733 my $peek = substr($::ORIG,$tpos,20);
734 $peek =~ s/\n/\\n/g;
735 $peek =~ s/\t/\\t/g;
736 $self->deb("cursor to $tpos --------->$GREEN$peek$CLEAR");
737 }
738 my %r = %$self;
739 # $r{_from} = $self->{_pos} // 0;
740 $r{_pos} = $tpos;
741  
742 bless \%r, ref $self;
743 }
744  
745 sub cursor_force { my $self = shift;
746 my $tpos = shift;
747  
748 $self->panic("Unexpected EOF") if $tpos > length($::ORIG);
749 if (DEBUG & DEBUG::cursors) {
750 my $peek = substr($::ORIG,$tpos,20);
751 $peek =~ s/\n/\\n/g;
752 $peek =~ s/\t/\\t/g;
753 $self->deb("cursor to $tpos --------->$GREEN$peek$CLEAR");
754 }
755 my %r = %$self;
756 # $r{_from} = $self->{_pos} // 0;
757 $r{_pos} = $::HIGHWATER = $tpos;
758  
759 bless \%r, ref $self;
760 }
761  
762 sub cursor_rev { my $self = shift;
763 my $fpos = shift;
764  
765 if (DEBUG & DEBUG::cursors) {
766 my $peek = substr($::ORIG,$fpos,20);
767 $peek =~ s/\n/\\n/g;
768 $peek =~ s/\t/\\t/g;
769 $self->deb("cursor_ref to $fpos --------->$GREEN$peek$CLEAR");
770 }
771 my %r = %$self;
772 $r{_pos} = $fpos;
773  
774 bless \%r, ref $self;
775 }
776  
777 #############################################################
778 # Regex service routines
779 #############################################################
780  
781 sub callm { my $self = shift;
782 my $arg = shift;
783 my $class = ref($self) || $self;
784  
785 my $lvl = 0;
786 my $extralvl = 0;
787 my @subs;
788 if (DEBUG & DEBUG::callm_show_subnames) {
789 while (my @c = caller($lvl)) {
790 $lvl++;
791 my $s = $c[3];
792 if ($s =~ /::_/) {
793 next;
794 }
795 elsif ($s =~ /^(?:STD::Cursor|CursorBase)?::/) {
796 next;
797 }
798 elsif ($s =~ /^STD::LazyMap::/) {
799 next;
800 }
801 elsif ($s =~ /^\(eval\)/) {
802 next;
803 }
804 else {
805 $extralvl = $lvl unless $extralvl;
806 $s =~ s/.*:://;
807 push @subs, $s;
808 }
809 }
810 }
811 else {
812 while (my @c = caller($lvl)) { $lvl++; }
813 }
814 my ($package, $file, $line, $subname, $hasargs) = caller(1);
815 my $name = $subname;
816 if (defined $arg) {
817 $name .= " " . $arg;
818 }
819 my $pos = '?';
820 $self->deb($name, " [", $file, ":", $line, "] $class") if DEBUG & DEBUG::trace_call;
821 if (DEBUG & DEBUG::callm_show_subnames) {
822 $RED . join(' ', reverse @subs) . $CLEAR . ':' x $extralvl;
823 }
824 else {
825 ':' x $lvl;
826 }
827 }
828  
829 sub retm {
830 return $_[0] unless DEBUG & DEBUG::trace_call;
831 my $self = shift;
832 warn "Returning non-STD::Cursor: $self\n" unless exists $self->{_pos};
833 my ($package, $file, $line, $subname, $hasargs) = caller(1);
834 $self->deb($subname, " returning @{[$self->{_pos}]}");
835 $self;
836 }
837  
838 sub _MATCHIFY { my $self = shift;
839 my $S = shift;
840 my $name = shift;
841 return () unless @_;
842 my $xact = $self->{_xact};
843 my @result = lazymap( sub { my $x = shift; $x->{_xact} = $xact; $x->_REDUCE($S, $name)->retm() }, @_);
844 if (wantarray) {
845 @result;
846 }
847 else {
848 $result[0];
849 }
850 }
851  
852 sub _MATCHIFYr { my $self = shift;
853 my $S = shift;
854 my $name = shift;
855 return () unless @_;
856 my $var = shift;
857 # $var->{_from} = $self->{_from};
858 my $xact = $self->{_xact};
859 $var->{_xact} = $xact;
860 $var->_REDUCE($S, $name)->retm();
861 }
862  
863 sub _SCANf { my $self = shift;
864  
865 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
866 my $eos = @::ORIG;
867  
868 my $pos = $self->{_pos};
869 my $C = $self->cursor_xact("SCANf $pos");
870 my $xact = $C->xact;
871  
872 lazymap( sub { $self->cursor($_[0])->retm() }, STD::LazyRange->new($xact, $pos,$eos) );
873 }
874  
875 sub _SCANg { my $self = shift;
876  
877 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
878 my $pos = $self->{_pos};
879 my $eos = @::ORIG;
880 my $C = $self->cursor_xact("SCANg $pos");
881 my $xact = $C->xact;
882  
883 lazymap( sub { $C->cursor($_[0])->retm() }, STD::LazyRangeRev->new($xact, $eos,$pos) );
884 }
885  
886 sub _STARf { my $self = shift;
887 my $block = shift;
888 no warnings 'recursion';
889  
890 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
891  
892 my $pos = $self->{_pos};
893 my $C = $self->cursor_xact("SCANf $pos");
894 my $xact = $C->xact;
895  
896 lazymap(sub { $_[0]->retm() },
897 $C->cursor($pos),
898 STD::LazyMap->new(sub { $C->_PLUSf($_[0]) }, $block));
899 }
900  
901 sub _STARg { my $self = shift;
902 my $block = shift;
903 no warnings 'recursion';
904  
905 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
906  
907 my $pos = $self->{_pos};
908 my $C = $self->cursor_xact("STARg $pos");
909 # my $xact = $C->xact;
910  
911 lazymap(sub { $_[0]->retm() }, reverse
912 eager(
913 $C->cursor($self->{_pos}),
914 $C->_PLUSf($block))
915 );
916 }
917  
918 sub _STARr { my $self = shift;
919 my $block = shift;
920 no warnings 'recursion';
921  
922 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
923 my $pos = $self->{_pos};
924 my $prev = $self->cursor_xact("STARr $pos");
925 # my $xact = $prev->xact;
926  
927 my $prev_pos = $prev->{_pos} // 0;
928 my @all;
929 my $eos = @::ORIG;
930  
931 for (;;) {
932 last if $prev->{_pos} == $eos;
933 my @matches = $block->($prev); # XXX shouldn't read whole list
934 # say @matches.perl;
935 last unless @matches;
936 my $first = $matches[0]; # no backtracking into block on ratchet
937 last if $first->{_pos} == $prev_pos;
938 $prev_pos = $first->{_pos};
939 push @all, $first;
940 $prev = $first;
941 }
942 $self->cursor($prev_pos)->retm();
943 }
944  
945 sub _PLUSf { my $self = shift;
946 my $block = shift;
947 no warnings 'recursion';
948  
949 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
950  
951 my $pos = $self->{_pos};
952 my $x = $self->cursor_xact("PLUSf $pos");
953 my $xact = $x->xact;
954  
955 # don't go beyond end of string
956 return () if $self->{_pos} == @::ORIG;
957  
958 lazymap(
959 sub {
960 my $x = $_[0];
961 lazymap(
962 sub {
963 $self->cursor($_[0]->{_pos})->retm()
964 }, $x, STD::LazyMap->new(sub { $x->_PLUSf($_[0]) }, $block)
965 );
966 }, $block->($self)
967 );
968 }
969  
970 sub _PLUSg { my $self = shift;
971 my $block = shift;
972 no warnings 'recursion';
973  
974 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
975  
976 my $pos = $self->{_pos};
977 my $C = $self->cursor_xact("PLUSg $pos");
978 # my $xact = $C->xact;
979  
980 reverse eager($C->_PLUSf($block, @_));
981 }
982  
983 sub _PLUSr { my $self = shift;
984 my $block = shift;
985 no warnings 'recursion';
986  
987 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
988 my @all;
989 my $eos = @::ORIG;
990  
991 my $pos = $self->{_pos};
992 my $to = $self->cursor_xact("PLUSr $pos");
993 # my $xact = $to->xact;
994  
995 for (;;) {
996 last if $to->{_pos} == $eos;
997 my @matches = $block->($to); # XXX shouldn't read whole list
998 last unless @matches;
999 my $first = $matches[0]; # no backtracking into block on ratchet
1000 #$first->deb($matches->perl) if DEBUG;
1001 push @all, $first;
1002 $to = $first;
1003 }
1004 return () unless @all;
1005 $self->cursor($to->{_pos})->retm();
1006 }
1007  
1008 sub _REPSEPf { my $self = shift;
1009 my $sep = shift;
1010 my $block = shift;
1011 no warnings 'recursion';
1012  
1013 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1014  
1015 my @result;
1016 # don't go beyond end of string
1017 return () if $self->{_pos} == @::ORIG;
1018  
1019 my $pos = $self->{_pos};
1020 my $C = $self->cursor_xact("REPSEPf $pos");
1021 # my $xact = $C->xact;
1022  
1023 do {
1024 for my $x ($block->($C)) {
1025 for my $s ($sep->($x)) {
1026 push @result, lazymap(sub { $C->cursor($_[0]->{_pos}) }, $x, $s->_REPSEPf($sep,$block));
1027 }
1028 }
1029 };
1030 lazymap(sub { $_[0]->retm() }, @result);
1031 }
1032  
1033 sub _REPSEPg { my $self = shift;
1034 my $sep = shift;
1035 my $block = shift;
1036 no warnings 'recursion';
1037  
1038 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1039  
1040 my $pos = $self->{_pos};
1041 my $C = $self->cursor_xact("REPSEPg $pos");
1042 # my $xact = $C->xact;
1043  
1044 reverse eager($C->_REPSEPf($sep, $block, @_));
1045 }
1046  
1047 sub _REPSEPr { my $self = shift;
1048 my $sep = shift;
1049 my $block = shift;
1050 no warnings 'recursion';
1051  
1052 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1053 my @all;
1054 my $eos = @::ORIG;
1055  
1056 my $pos = $self->{_pos};
1057 my $to = $self->cursor_xact("REPSEPr $pos");
1058 # my $xact = $C->xact;
1059  
1060 for (;;) {
1061 last if $to->{_pos} == $eos;
1062 my @matches = $block->($to); # XXX shouldn't read whole list
1063 last unless @matches;
1064 my $first = $matches[0]; # no backtracking into block on ratchet
1065 #$first->deb($matches->perl) if DEBUG;
1066 push @all, $first;
1067 my @seps = $sep->($first);
1068 last unless @seps;
1069 my $sep = $seps[0];
1070 $to = $sep;
1071 }
1072 return () unless @all;
1073 $self->cursor($all[-1]->{_pos})->retm;
1074 }
1075  
1076 sub _OPTr { my $self = shift;
1077 my $block = shift;
1078 no warnings 'recursion';
1079  
1080 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1081  
1082 my $pos = $self->{_pos};
1083 my $C = $self->cursor_xact("OPTr $pos");
1084 my $xact = $C->xact;
1085  
1086 my $x = ($block->($C))[0];
1087 my $r = $x // $C->cursor_tweak($pos);
1088 $r->{_xact} = $self->{_xact};
1089 $r->retm();
1090 }
1091  
1092 sub _OPTg { my $self = shift;
1093 my $block = shift;
1094 no warnings 'recursion';
1095  
1096 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1097  
1098 my $pos = $self->{_pos};
1099 my $C = $self->cursor_xact("OPTg $pos");
1100 # my $xact = $C->xact;
1101  
1102 my @x = $block->($C);
1103  
1104 lazymap(sub { $_[0]->retm() },
1105 $block->($C),
1106 $self->cursor($pos));
1107 }
1108  
1109 sub _OPTf { my $self = shift;
1110 my $block = shift;
1111 no warnings 'recursion';
1112  
1113 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1114  
1115 my $pos = $self->{_pos};
1116 my $C = $self->cursor_xact("OPTf $pos");
1117 # my $xact = $C->xact;
1118  
1119 lazymap(sub { $_[0]->retm() },
1120 $C->cursor($C->{_pos}),
1121 $block->($self));
1122 }
1123  
1124 sub _BRACKET { my $self = shift;
1125 my $block = shift;
1126 no warnings 'recursion';
1127  
1128 my $oldlang = ref($self);
1129 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1130 lazymap(sub { bless($_[0],$oldlang)->retm() },
1131 $block->($self));
1132 }
1133  
1134 sub _BRACKETr { my $self = shift;
1135 my $block = shift;
1136 no warnings 'recursion';
1137  
1138 my $oldlang = ref($self);
1139 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1140 my ($val) = $block->($self) or return ();
1141 bless($val,$oldlang)->retm();
1142 }
1143  
1144 sub _PAREN { my $self = shift;
1145 my $block = shift;
1146 no warnings 'recursion';
1147  
1148 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1149 lazymap(sub { $_[0]->retm() },
1150 $block->($self));
1151 }
1152  
1153 sub _NOTBEFORE { my $self = shift;
1154 my $block = shift;
1155 no warnings 'recursion';
1156  
1157 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1158 local $::HIGHEXPECT = {}; # don't count lookahead as expectation
1159 local $::HIGHWATER = $::HIGHWATER;
1160 my @caps;
1161 @caps = @{$self->{'~CAPS'}} if $self->{'~CAPS'}; # must copy elems
1162 my @all = $block->($self);
1163 return () if @all;
1164 $self->{'~CAPS'} = \@caps;
1165 return $self->cursor($self->{_pos})->retm();
1166 }
1167  
1168 sub _NOTCHAR { my $self = shift;
1169 my $block = shift;
1170 no warnings 'recursion';
1171  
1172 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1173 my @all = $block->($self);
1174 return () if @all;
1175 return $self->cursor($self->{_pos}+1)->retm();
1176 }
1177  
1178 sub before { my $self = shift;
1179 my $block = shift;
1180 no warnings 'recursion';
1181  
1182 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1183 local $::HIGHEXPECT = {}; # don't count lookahead as expectation
1184 local $::HIGHWATER = $::HIGHWATER;
1185 my @caps;
1186 @caps = @{$self->{'~CAPS'}} if $self->{'~CAPS'}; # must copy elems
1187 my @all = $block->($self);
1188 if (@all and $all[0]) {
1189 $all[0]->{'~CAPS'} = \@caps;
1190 if ($self->{_ast}) {
1191 $all[0]->{'_ast'} = $self->{_ast};
1192 }
1193 else {
1194 delete $all[0]->{'_ast'};
1195 }
1196 return $all[0]->cursor_all(($self->{_pos}) x 2)->retm();
1197 }
1198 return ();
1199 }
1200  
1201 sub suppose { my $self = shift;
1202 my $block = shift;
1203 no warnings 'recursion';
1204  
1205 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1206 local $::FATALS = 0;
1207 local @::WORRIES;
1208 local %::WORRIES;
1209 local $::HIGHWATER = -1;
1210 local $::HIGHMESS;
1211 local $::HIGHEXPECT = {};
1212 local $::IN_SUPPOSE = 1;
1213 my @all;
1214 eval {
1215 @all = $block->($self);
1216 };
1217 lazymap( sub { $_[0]->retm() }, @all );
1218 }
1219  
1220 sub after { my $self = shift;
1221 my $block = shift;
1222 no warnings 'recursion';
1223  
1224 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1225 local $::HIGHEXPECT = {}; # don't count lookbehind as expectation
1226 my $end = $self->cursor($self->{_pos});
1227 my @caps;
1228 @caps = @{$self->{'~CAPS'}} if $self->{'~CAPS'}; # must copy elems
1229 my @all = $block->($end); # Make sure $_->{_from} == $_->{_pos}
1230 if (@all and $all[0]) {
1231 $all[0]->{'~CAPS'} = \@caps;
1232 if ($self->{_ast}) {
1233 $all[0]->{'_ast'} = $self->{_ast};
1234 }
1235 else {
1236 delete $all[0]->{'_ast'};
1237 }
1238 return $all[0]->cursor_all(($self->{_pos}) x 2)->retm();
1239 }
1240 return ();
1241 }
1242  
1243 sub null { my $self = shift;
1244 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1245 return $self->cursor($self->{_pos})->retm();
1246 }
1247  
1248 sub ws__PEEK { ''; }
1249 sub ws {
1250 my $self = shift;
1251  
1252 local $CTX = $self->callm() if DEBUG & DEBUG::trace_call;
1253 my @stub = return $self if exists $::MEMOS[$self->{_pos}]{ws};
1254  
1255 my $S = $self->{_pos};
1256 my $C = $self->cursor_xact("RULE ws $S");
1257 # my $xact = $C->xact;
1258  
1259 $::MEMOS[$S]{ws} = undef; # exists means we know, undef means no ws before here
1260  
1261 $self->_MATCHIFY($S, 'ws',
1262 $C->_BRACKET( sub { my $C=shift;
1263 do { my @gather;
1264 push @gather, (map { my $C=$_;
1265 (map { my $C=$_;
1266 (map { my $C=$_;
1267 $C->_NOTBEFORE( sub { my $C=shift;
1268 $C
1269 })
1270 } $C->_COMMITRULE())
1271 } $C->before(sub { my $C=shift;
1272 $C->_ALNUM()
1273 }))
1274 } $C->before( sub { my $C=shift;
1275 $C->after(sub { my $C=shift;
1276 $C->_ALNUM_rev()
1277 })
1278 }))
1279 or
1280 push @gather, (map { my $C=$_;
1281 (map { my $C=$_;
1282 scalar(do { $::MEMOS[$C->{_pos}]{ws} = $S unless $C->{_pos} == $S }, $C)
1283 } $C->_STARr(sub { my $C=shift;
1284 $C->_SPACE()
1285 }))
1286 } $C);
1287 @gather;
1288 }
1289 })
1290 );
1291 }
1292  
1293 sub _ASSERT { my $self = shift;
1294 my $block = shift;
1295 no warnings 'recursion';
1296  
1297 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1298 my @all = $block->($self);
1299 if ((@all and $all[0]->{_bool})) {
1300 return $self->cursor($self->{_pos})->retm();
1301 }
1302 return ();
1303 }
1304  
1305 sub _BINDVAR { my $self = shift;
1306 my $var = shift;
1307 my $block = shift;
1308 no warnings 'recursion';
1309  
1310 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1311 lazymap(sub { $$var = $_[0]; $_[0]->retm() },
1312 $block->($self));
1313 }
1314  
1315 sub _SUBSUME { my $self = shift;
1316 my $names = shift;
1317 my $block = shift;
1318 no warnings 'recursion';
1319 no warnings 'recursion';
1320  
1321 local $CTX = $self->callm($names ? "@$names" : "") if DEBUG & DEBUG::trace_call;
1322 lazymap(sub { $self->cursor_bind($names, $_[0])->retm() },
1323 $block->($self->cursor_fresh()));
1324 }
1325  
1326 sub _SUBSUMEr { my $self = shift;
1327 my $names = shift;
1328 my $block = shift;
1329 no warnings 'recursion';
1330 no warnings 'recursion';
1331  
1332 local $CTX = $self->callm($names ? "@$names" : "") if DEBUG & DEBUG::trace_call;
1333 my ($var) = $block->($self->cursor_fresh()) or return ();
1334 $self->cursor_bind($names, $var)->retm();
1335 }
1336  
1337 sub _EXACT_rev { my $self = shift;
1338 my $s = shift() // '';
1339 my @ints = unpack("U*", $s);
1340  
1341 local $CTX = $self->callm($s) if DEBUG & DEBUG::trace_call;
1342 my $P = $self->{_pos} // 0;
1343 while (@ints) {
1344 return () unless ($::ORIG[--$P]//-1) == pop @ints;
1345 }
1346 return $self->cursor($P)->retm();
1347 }
1348  
1349 sub _EXACT { my $self = shift;
1350 my $s = shift() // '';
1351 my @ints = unpack("U*", $s);
1352  
1353 local $CTX = $self->callm($s) if DEBUG & DEBUG::trace_call;
1354 my $P = $self->{_pos} // 0;
1355 while (@ints) {
1356 return () unless ($::ORIG[$P++]//-1) == shift @ints;
1357 }
1358 return $self->cursor($P)->retm();
1359 # if (substr($::ORIG, $P, $len) eq $s) {
1360 # $self->deb("EXACT $s matched @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1361 # my $r = $self->cursor($P+$len);
1362 # $r->retm();
1363 # }
1364 # else {
1365 # $self->deb("EXACT $s didn't match @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1366 # return ();
1367 # }
1368 }
1369  
1370 sub _PATTERN { my $self = shift;
1371 my $qr = shift;
1372  
1373 local $CTX = $self->callm($qr) if DEBUG & DEBUG::trace_call;
1374 my $P = $self->{_pos} // 0;
1375 pos($::ORIG) = $P;
1376 if ($::ORIG =~ /$qr/gc) {
1377 my $len = pos($::ORIG) - $P;
1378 $self->deb("PATTERN $qr matched @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1379 my $r = $self->cursor($P+$len);
1380 $r->retm();
1381 }
1382 else {
1383 $self->deb("PATTERN $qr didn't match at $P") if DEBUG & DEBUG::matchers;
1384 return ();
1385 }
1386 }
1387  
1388 sub _BACKREFn { my $self = shift;
1389 my $n = shift;
1390  
1391 local $CTX = $self->callm($n) if DEBUG & DEBUG::trace_call;
1392 my $P = $self->{_pos} // 0;
1393 my $s = $self->{$n}->Str;
1394 my $len = length($s);
1395 if (substr($::ORIG, $P, $len) eq $s) {
1396 $self->deb("EXACT $s matched @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1397 my $r = $self->cursor($P+$len);
1398 $r->retm();
1399 }
1400 else {
1401 $self->deb("EXACT $s didn't match @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1402 return ();
1403 }
1404 }
1405  
1406 sub _SYM { my $self = shift;
1407 my $s = shift;
1408 my $i = shift;
1409  
1410 $s = $s->[0] if ref $s eq 'ARRAY';
1411  
1412 local $CTX = $self->callm($s) if DEBUG & DEBUG::trace_call;
1413 my $P = $self->{_pos} // 0;
1414 my $len = length($s);
1415 if ($i
1416 ? lc substr($::ORIG, $P, $len) eq lc $s
1417 : substr($::ORIG, $P, $len) eq $s
1418 ) {
1419 $self->deb("SYM $s matched @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1420 my $r = $self->cursor($P+$len);
1421 $r->{sym} = $s;
1422 $r->retm();
1423 }
1424 else {
1425 $self->deb("SYM $s didn't match @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1426 return ();
1427 }
1428 }
1429  
1430 #sub _EXACT_rev { my $self = shift;
1431 # my $s = shift;
1432 #
1433 # local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1434 # my $len = length($s);
1435 # my $from = $self->{_pos} - $len;
1436 # if ($from >= 0 and substr($::ORIG, $from, $len) eq $s) {
1437 # my $r = $self->cursor_rev($from);
1438 # $r->retm();
1439 # }
1440 # else {
1441 ## say "EXACT_rev $s didn't match @{[substr($!orig,$from,$len)]} at $from $len";
1442 # return ();
1443 # }
1444 #}
1445  
1446 sub _ARRAY { my $self = shift;
1447 local $CTX = $self->callm(0+@_) if DEBUG & DEBUG::trace_call;
1448 my $P = $self->{_pos} // 0;
1449 my @array = sort { length($b) <=> length($a) } @_; # XXX suboptimal
1450 my @result = ();
1451 for my $s (@array) {
1452 my $len = length($s);
1453 if (substr($::ORIG, $P, $len) eq $s) {
1454 $self->deb("ARRAY elem $s matched @{[substr($::ORIG,$P,$len)]} at $P $len") if DEBUG & DEBUG::matchers;
1455 my $r = $self->cursor($P+$len);
1456 push @result, $r->retm('');
1457 }
1458 }
1459 return @result;
1460 }
1461  
1462 sub _ARRAY_rev { my $self = shift;
1463 local $CTX = $self->callm(0+@_) if DEBUG & DEBUG::trace_call;
1464 my @array = sort { length($b) <=> length($a) } @_; # XXX suboptimal
1465 my @result = ();
1466 for my $s (@array) {
1467 my $len = length($s);
1468 my $from = $self->{_pos} = $len;
1469 if (substr($::ORIG, $from, $len) eq $s) {
1470 $self->deb("ARRAY_rev elem $s matched @{[substr($::ORIG,$from,$len)]} at $from $len") if DEBUG & DEBUG::matchers;
1471 my $r = $self->cursor_rev($from);
1472 push @result, $r->retm('');
1473 }
1474 }
1475 return @result;
1476 }
1477  
1478 sub _DIGIT { my $self = shift;
1479 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1480 my $P = $self->{_pos};
1481 my $char = substr($::ORIG, $P, 1);
1482 if ($char =~ /^\d$/) {
1483 my $r = $self->cursor($P+1);
1484 return $r->retm();
1485 }
1486 else {
1487 # say "DIGIT didn't match $char at $P";
1488 return ();
1489 }
1490 }
1491  
1492 sub _DIGIT_rev { my $self = shift;
1493 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1494 my $from = $self->{_pos} - 1;
1495 if ($from < 0) {
1496 # say "DIGIT_rev didn't match $char at $from";
1497 return ();
1498 }
1499 my $char = substr($::ORIG, $from, 1);
1500 if ($char =~ /^\d$/) {
1501 my $r = $self->cursor_rev($from);
1502 return $r->retm();
1503 }
1504 else {
1505 # say "DIGIT_rev didn't match $char at $from";
1506 return ();
1507 }
1508 }
1509  
1510 sub ww { my $self = shift;
1511 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1512 my $P = $self->{_pos};
1513 return () unless $P;
1514 my $chars = substr($::ORIG, $P-1, 2);
1515 if ($chars =~ /^\w\w$/) {
1516 my $r = $self->cursor($P);
1517 return $r->retm();
1518 }
1519 else {
1520 # say "ww didn't match $chars at $P";
1521 return ();
1522 }
1523 }
1524  
1525 sub _ALNUM { my $self = shift;
1526 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1527 my $P = $self->{_pos};
1528 my $char = substr($::ORIG, $P, 1);
1529 if ($char =~ /^\w$/) {
1530 my $r = $self->cursor($P+1);
1531 return $r->retm();
1532 }
1533 else {
1534 # say "ALNUM didn't match $char at $P";
1535 return ();
1536 }
1537 }
1538  
1539 sub _ALNUM_rev { my $self = shift;
1540 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1541 my $from = $self->{_pos} - 1;
1542 if ($from < 0) {
1543 # say "ALNUM_rev didn't match $char at $from";
1544 return ();
1545 }
1546 my $char = substr($::ORIG, $from, 1);
1547 if ($char =~ /^\w$/) {
1548 my $r = $self->cursor_rev($from);
1549 return $r->retm();
1550 }
1551 else {
1552 # say "ALNUM_rev didn't match $char at $from";
1553 return ();
1554 }
1555 }
1556  
1557 my $alpha;
1558 BEGIN {
1559 $alpha = "";
1560 for my $ch (0..255) {
1561 my $char = chr($ch);
1562 vec($alpha,$ch,1) = 1 if $char =~ /\w/ and $char !~ /\d/;
1563 }
1564 }
1565 sub alpha { my $self = shift;
1566 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1567 my $P = $self->{_pos};
1568 # my $char = substr($::ORIG, $P, 1);
1569 my $ch = $::ORIG[$P];
1570 if (vec($alpha,$ch,1) or ($ch > 255 and chr($ch) =~ /\pL/)) {
1571 # if ($char =~ /^[_[:alpha:]\pL]$/) {
1572 my $r = $self->cursor($P+1);
1573 return $r->retm();
1574 }
1575 else {
1576 # say "alpha didn't match $char at $P";
1577 return ();
1578 }
1579 }
1580  
1581 sub alpha_rev { my $self = shift;
1582 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1583 my $from = $self->{_pos} - 1;
1584 if ($from < 0) {
1585 return ();
1586 }
1587 my $char = substr($::ORIG, $from, 1);
1588 if ($char =~ /^[_[:alpha:]\pL]$/) {
1589 my $r = $self->cursor_rev($from);
1590 return $r->retm();
1591 }
1592 else {
1593 return ();
1594 }
1595 }
1596  
1597 sub _SPACE { my $self = shift;
1598 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1599 my $P = $self->{_pos};
1600 my $char = substr($::ORIG, $P, 1);
1601 if ($char =~ /^\s$/) {
1602 my $r = $self->cursor($P+1);
1603 return $r->retm();
1604 }
1605 else {
1606 # say "SPACE didn't match $char at $P";
1607 return ();
1608 }
1609 }
1610  
1611 sub _SPACE_rev { my $self = shift;
1612 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1613 my $from = $self->{_pos} - 1;
1614 if ($from < 0) {
1615 # say "SPACE_rev didn't match $char at $from";
1616 return ();
1617 }
1618 my $char = substr($::ORIG, $from, 1);
1619 if ($char =~ /^\s$/) {
1620 my $r = $self->cursor_rev($from);
1621 return $r->retm();
1622 }
1623 else {
1624 # say "SPACE_rev didn't match $char at $from";
1625 return ();
1626 }
1627 }
1628  
1629 sub _HSPACE { my $self = shift;
1630 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1631 my $P = $self->{_pos};
1632 my $char = substr($::ORIG, $P, 1);
1633 if ($char =~ /^[ \t\r]$/ or ($char =~ /^\s$/ and $char !~ /^[\n\f\0x0b\x{2028}\x{2029}]$/)) {
1634 my $r = $self->cursor($P+1);
1635 return $r->retm();
1636 }
1637 else {
1638 # say "HSPACE didn't match $char at $P";
1639 return ();
1640 }
1641 }
1642  
1643 sub _HSPACE_rev { my $self = shift;
1644 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1645 my $from = $self->{_pos} - 1;
1646 if ($from < 0) {
1647 # say "HSPACE_rev didn't match $char at $from";
1648 return ();
1649 }
1650 my $char = substr($::ORIG, $from, 1);
1651 if ($char =~ /^[ \t\r]$/ or ($char =~ /^\s$/ and $char !~ /^[\n\f\0x0b\x{2028}\x{2029}]$/)) {
1652 my $r = $self->cursor_rev($from);
1653 return $r->retm();
1654 }
1655 else {
1656 # say "HSPACE_rev didn't match $char at $from";
1657 return ();
1658 }
1659 }
1660  
1661 sub _VSPACE { my $self = shift;
1662 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1663 my $P = $self->{_pos};
1664 my $char = substr($::ORIG, $P, 1);
1665 if ($char =~ /^[\n\f\x0b\x{2028}\x{2029}]$/) {
1666 my $r = $self->cursor($P+1);
1667 return $r->retm();
1668 }
1669 else {
1670 # say "VSPACE didn't match $char at $P";
1671 return ();
1672 }
1673 }
1674  
1675 sub _VSPACE_rev { my $self = shift;
1676 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1677 my $from = $self->{_pos} - 1;
1678 if ($from < 0) {
1679 # say "VSPACE_rev didn't match $char at $from";
1680 return ();
1681 }
1682 my $char = substr($::ORIG, $from, 1);
1683 if ($char =~ /^[\n\f\x0b\x{2028}\x{2029}]$/) {
1684 my $r = $self->cursor_rev($from);
1685 return $r->retm();
1686 }
1687 else {
1688 # say "VSPACE_rev didn't match $char at $from";
1689 return ();
1690 }
1691 }
1692  
1693 sub _CCLASS { my $self = shift;
1694 my $cc = shift;
1695  
1696 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1697 my $P = $self->{_pos};
1698 my $char = substr($::ORIG, $P, 1);
1699 if ($char =~ /$cc/) {
1700 my $r = $self->cursor($P+1);
1701 return $r->retm();
1702 }
1703 else {
1704 # say "CCLASS didn't match $char at $P";
1705 return ();
1706 }
1707 }
1708  
1709 sub _CCLASS_rev { my $self = shift;
1710 my $cc = shift;
1711  
1712 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1713 my $from = $self->{_pos} - 1;
1714 if ($from < 0) {
1715 # say "CCLASS didn't match $char at $from";
1716 return ();
1717 }
1718 my $char = substr($::ORIG, $from, 1);
1719 if ($char =~ /$cc/) {
1720 my $r = $self->cursor_rev($from);
1721 return $r->retm();
1722 }
1723 else {
1724 # say "CCLASS didn't match $char at $from";
1725 return ();
1726 }
1727 }
1728  
1729 sub _ANY { my $self = shift;
1730 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1731 my $P = $self->{_pos};
1732 if ($P < @::ORIG) {
1733 $self->cursor($P+1)->retm();
1734 }
1735 else {
1736 # say "ANY didn't match anything at $P";
1737 return ();
1738 }
1739 }
1740  
1741 sub _ANY_rev { my $self = shift;
1742 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1743 my $from = $self->{_pos} - 1;
1744 if ($from < 0) {
1745 return ();
1746 }
1747 return $self->cursor_rev($from)->retm();
1748 }
1749  
1750 sub _BOS { my $self = shift;
1751 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1752 my $P = $self->{_pos};
1753 if ($P == 0) {
1754 $self->cursor($P)->retm();
1755 }
1756 else {
1757 return ();
1758 }
1759 }
1760 sub _BOS_rev { $_[0]->_BOS }
1761  
1762 sub _BOL { my $self = shift;
1763 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1764 my $P = $self->{_pos};
1765 if ($P == 0 or substr($::ORIG, $P-1, 1) =~ /^[\n\f\x0b\x{2028}\x{2029}]$/) {
1766 $self->cursor($P)->retm();
1767 }
1768 else {
1769 return ();
1770 }
1771 }
1772 sub _BOL_rev { $_[0]->_BOL }
1773  
1774 sub _EOS { my $self = shift;
1775 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1776 my $P = $self->{_pos};
1777 if ($P == @::ORIG) {
1778 $self->cursor($P)->retm();
1779 }
1780 else {
1781 return ();
1782 }
1783 }
1784 sub _EOS_rev { $_[0]->_EOS }
1785  
1786 sub _EOL { my $self = shift;
1787 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1788 my $P = $self->{_pos};
1789 if ($P == @::ORIG or substr($::ORIG, $P, 1) =~ /^(?:\r\n|[\n\f\x0b\x{2028}\x{2029}])$/) {
1790 $self->cursor($P)->retm();
1791 }
1792 else {
1793 return ();
1794 }
1795 }
1796 sub _EOL_rev { $_[0]->_EOL }
1797  
1798 sub _RIGHTWB { my $self = shift;
1799 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1800 my $P = $self->{_pos};
1801 pos($::ORIG) = $P - 1;
1802 if ($::ORIG =~ /\w\b/) {
1803 $self->cursor($P)->retm();
1804 }
1805 else {
1806 return ();
1807 }
1808 }
1809 sub _RIGHTWB_rev { $_[0]->_RIGHTWB }
1810  
1811 sub _LEFTWB { my $self = shift;
1812 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1813 my $P = $self->{_pos};
1814 pos($::ORIG) = $P;
1815 if ($::ORIG =~ /\b(?=\w)/) {
1816 $self->cursor($P)->retm();
1817 }
1818 else {
1819 return ();
1820 }
1821 }
1822 sub _LEFTWB_rev { $_[0]->_LEFTWB }
1823  
1824 sub _LEFTRESULT { my $self = shift;
1825 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1826 my $P = $self->{_pos};
1827 pos($::ORIG) = $P;
1828 if ($::ORIG =~ /\b(?=\w)/) {
1829 $self->cursor($P)->retm();
1830 }
1831 else {
1832 return ();
1833 }
1834 }
1835 sub _LEFTRESULT_rev { $_[0]->_LEFTWB }
1836  
1837 sub _REDUCE { my $self = shift;
1838 my $S = shift;
1839 my $meth = shift;
1840 my $key = $meth;
1841 $key .= ' ' . $_[0] if @_;
1842  
1843 $self->{_reduced} = $key;
1844 $self->{_from} = $S;
1845 if ($::ACTIONS) {
1846 eval { $::ACTIONS->$meth($self, @_) };
1847 warn $@ if $@ and not $@ =~ /locate object method "\Q$meth/;
1848 }
1849 $self->deb("REDUCE $key from " . $S . " to " . $self->{_pos}) if DEBUG & DEBUG::matchers;
1850 $self;
1851 }
1852  
1853 sub _COMMITBRANCH { my $self = shift;
1854 my $xact = $self->xact;
1855 # $self->{LAST} = shift() if @_;
1856 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1857 my $P = $self->{_pos};
1858 while ($xact) {
1859 $xact->[-2] = 1;
1860 $self->deb("Commit $$xact[0] to $P") if DEBUG & DEBUG::matchers;
1861 return $self->cursor_xact("CB") if $xact->[0] =~ /^ALT/;
1862 $xact = $xact->[-1];
1863 }
1864 die "Not in an alternation, so can't commit to a branch";
1865 }
1866  
1867 sub _COMMITLTM { my $self = shift;
1868 my $xact = $self->xact;
1869 # $self->{LAST} = shift() if @_;
1870 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1871 my $P = $self->{_pos};
1872 while ($xact) {
1873 $xact->[-2] = 1;
1874 $self->deb("Commit $$xact[0] to $P") if DEBUG & DEBUG::matchers;
1875 return $self->cursor_xact("CL") if $xact->[0] =~ /^ALTLTM/;
1876 $xact = $xact->[-1];
1877 }
1878 die "Not in a longest token matcher, so can't commit to a longest token";
1879 }
1880  
1881 sub _COMMITRULE { my $self = shift;
1882 my $xact = $self->xact;
1883 # $self->{LAST} = shift() if @_;
1884 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1885 my $P = $self->{_pos};
1886 while ($xact) {
1887 $xact->[-2] = 1;
1888 $self->deb("Commit $$xact[0] to $P") if DEBUG & DEBUG::matchers;
1889 return $self->cursor_xact("CR") if $xact->[0] =~ /^RULE/;
1890 $xact = $xact->[-1];
1891 }
1892 die "Not in a rule, so can't commit to rule";
1893 }
1894  
1895 sub commit { my $self = shift;
1896 my $xact = $self->xact;
1897 # $self->{LAST} = shift() if @_;
1898 local $CTX = $self->callm if DEBUG & DEBUG::trace_call;
1899 my $P = $self->{_pos};
1900 while ($xact) {
1901 $xact->[-2] = 1;
1902 $self->deb("Commit $$xact[0] to $P") if DEBUG & DEBUG::matchers;
1903 return $self->cursor_xact("CM") if $xact->[0] =~ /^MATCH/;
1904 $xact = $xact->[-1];
1905 }
1906 die "Not in a match, so can't commit to match";
1907 }
1908  
1909 sub fail { my $self = shift;
1910 my $m = shift;
1911 return ();
1912 }
1913  
1914 sub bless { CORE::bless $_[1], $_[0]->WHAT }
1915  
1916 #############################################################
1917 # JIT lexer generator
1918 #############################################################
1919  
1920 ## NFA structure:
1921 ## array of (NFA node) ->
1922 ## 0: non extensible (imperative) flag
1923 ## 0: array of fate (array of fate element)
1924 ## array of (label) at odd index, new index at even
1925 ## DFA structure:
1926 ## each DFA node is array:
1927 ## 0: array of object fates
1928 ## 1: hash of specific cases (char => DFAnode)
1929 ## also carries some debug data
1930 ## 2n: reference to a uniprop hash
1931 ## 2n+1: DFAnode is that hash existed
1932 ## Labels: undef is epsilon link.
1933 ## otherwise list - 1 positive, 0+ negative
1934 ## each is: 1 character, else unicode prop in "Gc/L" form
1935 ## "DFA" lexer structure:
1936 ## {DFA} -> array of refs to all DFA nodes
1937 ## {DBA}, {FILE}, {NAME} same as "RE" lexer structure
1938 ## {S} -> ref to DFA root
1939 ## {NFA} -> NFA structure
1940 ## individual fates in the NFA end with a hook which can be 1 to stop adding
1941 ## fates on the end; it's not always possible to associate a unique fate with
1942 ## each NFA node, consider (a|b)*
1943 ## A NFA or DFA node is accepting if it has a nonempty list of fates
1944  
1945 #cycle breaker
1946 {
1947 package CursorBase::dfa;
1948 sub DESTROY {
1949 my $self = shift;
1950 for (@$self) { @$_ = (); }
1951 }
1952 }
1953  
1954 # Steal data from Perl5's Unicode maps
1955 my %unicode_map_cache;
1956 BEGIN {
1957 $unicode_map_cache{ALL} = [scalar("\377" x 128) x 1088, "ALL"] ;
1958 my $name = File::Spec->catfile($data_dir, "uniprops");
1959 open MAP, "<", "$name" or
1960 die "cannot open unicode maps from $name : $!\n";
1961  
1962 binmode MAP;
1963 while (defined (my $c = getc MAP)) {
1964 my $name = "";
1965 my $used;
1966 my $tile;
1967 read MAP, $name, ord($c);
1968 read MAP, $used, 136;
1969  
1970 $unicode_map_cache{$name} = [ (("") x 1088), $name ];
1971  
1972 for (my $i = 0; $i < 1088; $i++) {
1973 if (vec($used, $i, 1)) {
1974 read MAP, $tile, 128;
1975 $unicode_map_cache{$name}[$i] = $tile;
1976 }
1977 }
1978 }
1979 close MAP or die "cannot close unicode maps: $!";
1980 }
1981 sub _get_unicode_map {
1982 my $propname = shift;
1983 $unicode_map_cache{$propname} //
1984 die "Map $propname not found. Edit gen-unicode-table.pl and rerun.";
1985 }
1986  
1987 # This is the fast path handling for JIT DFA lexer generation (although it gets
1988 # short-circuited if the DFALEXERS entry exists, later). The lexer generation
1989 # process sometimes recurses to this, which is tracked using %::AUTOLEXED; if
1990 # the value is already set, we need to suppress recursion.
1991  
1992 our $fakepos = 1;
1993  
1994 sub _dump_nfa { my ($name, $nfa) = @_;
1995 print ::LOG "--- BEGIN NFA DUMP ($name) ---\n";
1996 for my $ix (0 .. @$nfa-1) {
1997 my @go;
1998 for (my $j = 2; $j < @{ $nfa->[$ix] }; $j += 2) {
1999 push @go, "[" . join("-",@{$nfa->[$ix][$j] // []}) . "] => " . $nfa->[$ix][$j+1];
2000 }
2001 my $l = sprintf "%4d: %-30s ", $ix, join(", ", @go);
2002 if ($nfa->[$ix][1]) {
2003 my @x = @{ $nfa->[$ix][1] };
2004 push @x, "..." if shift(@x);
2005 $l .= join(" ", "-->", @x);
2006 }
2007 print ::LOG $l, "\n";
2008 }
2009 print ::LOG "---- END NFA DUMP ----\n";
2010 }
2011  
2012 sub _dtree_dump { my ($ord, $dt) = @_;
2013 print ::LOG (" " x (2 + $ord));
2014 if (!defined $dt) {
2015 print ::LOG "END\n";
2016 } elsif (ref $dt ne 'ARRAY') {
2017 print ::LOG ($$dt)->[1]{ID}, "\n";
2018 } else {
2019 print ::LOG $dt->[2][-1], "?\n";
2020 _dtree_dump($ord+1, $dt->[1]);
2021 _dtree_dump($ord+1, $dt->[0]);
2022 }
2023 }
2024  
2025 sub _dfa_dump_node { my ($dfan) = @_;
2026 my @go;
2027 my @gor = %{ $dfan->[1] };
2028 while (my ($a, $b) = splice @gor, 0, 2) {
2029 next if $a eq 'DESC';
2030 next if $a eq 'ID';
2031 push @go, "'" . ::qm(chr $a) . "' => " . $b->[1]{ID};
2032 }
2033 printf ::LOG "%-30s %-30s\n", $dfan->[1]{DESC} . ":", join(", ", @go);
2034 _dtree_dump(0, $dfan->[2]);
2035 for (@{ $dfan->[0] }) {
2036 my @arr;
2037 for (my $fate = $_; $fate; $fate = $fate->[0]) {
2038 push @arr, $fate->[1], $fate->[2];
2039 }
2040 print ::LOG " --> ", join(" ", @arr), "\n";
2041 }
2042 }
2043  
2044 sub _elem_matches { # my ($char, $element) = @_;
2045 # Optimize for the common path
2046 return $_[0] eq $_[1] if length $_[1] == 1;
2047  
2048 my $i = ord $_[0];
2049 return vec(_get_unicode_map($_[1])->[$i >> 10], $i & 1023, 1);
2050 }
2051  
2052 my %boolean_tables = map { $_, 1 } qw/AHex Alpha BidiC BidiM CE CI CWCF CWCM
2053 CWKCF CWL CWT CWU Cased CompEx DI Dash Dep Dia Ext GrBase GrExt Hex Hyphen
2054 IDC IDS IDSB IDST Ideo JoinC Lower Math NChar NFDQC OAlpha ODI OGrExt OIDC
2055 OIDS OLower OMath OUpper PatSyn PatWS QMark Radical SD STerm Space Term
2056 UIdeo Upper VS XIDC XIDS/;
2057 sub _elem_excludes { my ($up1, $up2) = @_;
2058 my ($t1, $v1) = split "/", $up1;
2059 my ($t2, $v2) = split "/", $up2;
2060  
2061 return 0 if $t1 ne $t2;
2062 return 0 if $v1 eq $v2;
2063  
2064 return 1 if $boolean_tables{$t1};
2065 return 1 if $t1 eq 'Gc' && (length($v1) == length($v2)
2066 || substr($v1, 0, 1) ne substr($v2, 0, 1));
2067  
2068 return 0;
2069 }
2070  
2071 sub _elem_implies { my ($up1, $up2) = @_;
2072 my ($t1, $v1) = split "/", $up1;
2073 my ($t2, $v2) = split "/", $up2;
2074  
2075 return 0 if $t1 ne $t2;
2076 return 1 if $v1 eq $v2;
2077  
2078 return 1 if $t1 eq 'Gc' && substr($v1, 0, 1) eq $v2;
2079  
2080 return 0;
2081 }
2082  
2083 sub _elem_dich { my ($up1, $up2) = @_;
2084 my ($t1, $v1) = split "/", $up1;
2085 my ($t2, $v2) = split "/", $up2;
2086  
2087 return 0 if $t1 ne $t2;
2088 return 0 if $v1 eq $v2;
2089  
2090 return 1 if $boolean_tables{$t1};
2091 return 0;
2092 }
2093  
2094 sub _decision_tree { my ($thunk, @edges) = @_;
2095 my $branch;
2096  
2097 TERM: for (my $i = 0; $i < @edges; $i += 2) {
2098 for my $c (@{ $edges[$i] }) {
2099 next if $c eq 'ALL';
2100 $branch = $c;
2101 last TERM;
2102 }
2103 }
2104  
2105 if (defined $branch) {
2106 my @true;
2107 my @false;
2108  
2109 for (my $i = 0; $i < @edges; $i += 2) {
2110 my ($p, @n) = @{ $edges[$i] };
2111  
2112 if (!_elem_excludes($branch, $p) &&
2113 !(grep { _elem_implies($branch, $_) } @n)) {
2114 my $pp = _elem_implies($branch, $p) ? 'ALL' : $p;
2115 my @nn = grep { !_elem_excludes($branch, $_) } @n;
2116 push @true, [ $pp, @nn ], $edges[$i+1];
2117 }
2118  
2119 if (!_elem_implies($p, $branch) &&
2120 !(grep { _elem_dich($branch, $_) } @n)) {
2121 my $pp = _elem_dich($branch, $p) ? 'ALL' : $p;
2122 my @nn = grep { !_elem_implies($_, $branch) } @n;
2123 push @false, [ $pp, @nn ], $edges[$i+1];
2124 }
2125 }
2126  
2127 return [ _decision_tree($thunk, @false),
2128 _decision_tree($thunk, @true),
2129 _get_unicode_map($branch) ];
2130 } else {
2131 # all edges are labelled [ALL]
2132 my $bm = "";
2133 for (my $i = 1; $i < @edges; $i += 2) {
2134 vec($bm, $edges[$i], 1) = 1;
2135 }
2136 return ($bm ne '') ? (\ $thunk->($bm)) : undef;
2137 }
2138 }
2139  
2140 sub _tangle_edges { my ($our_edges, $thunk) = @_;
2141 my %used_chars;
2142 my %used_cats;
2143  
2144 for (my $i = 0; $i < @$our_edges; $i += 2) {
2145 next unless $our_edges->[$i];
2146 for (@{ $our_edges->[$i] }) {
2147 if (length($_) == 1) {
2148 $used_chars{$_} = 1;
2149 } else {
2150 $used_cats{$_} = 1;
2151 }
2152 }
2153 }
2154  
2155 # First, all specifically mentioned characters are floated to the initial
2156 # case
2157 my %next_1;
2158 my $edgelistref;
2159 for my $ch (keys %used_chars) {
2160 my $bm = "";
2161 EDGE: for (my $i = 0; $i < @$our_edges; $i += 2) {
2162 $edgelistref = $our_edges->[$i];
2163 if (length $edgelistref->[0] != 1) {
2164 my $o = ord $ch; # inlined from _elem_matches
2165 next unless vec(_get_unicode_map($edgelistref->[0])->[$o >> 10], $o & 1023, 1);
2166 } elsif ($edgelistref->[0] ne $ch) {
2167 next;
2168 }
2169 my @edgelist = @$edgelistref;
2170 for (my $j = 0; ++$j < @edgelist; ) {
2171 next EDGE if _elem_matches($ch, $edgelistref->[$j]);
2172 }
2173 vec($bm, $our_edges->[$i+1], 1) = 1;
2174 }
2175 $next_1{ord $ch} = $thunk->($bm);
2176 }
2177  
2178 # Now clean them out so the decision tree engine doesn't have to deal with
2179 # single characters
2180 $our_edges = [ @$our_edges ];
2181 for (my $i = 0; $i < @$our_edges; ) {
2182 if (!$our_edges->[$i] || length($our_edges->[$i][0]) == 1) {
2183 splice @$our_edges, $i, 2;
2184 } else {
2185 $our_edges->[$i] = [grep { length($_) > 1 } @{ $our_edges->[$i] }];
2186 $i += 2;
2187 }
2188 }
2189  
2190 \%next_1, _decision_tree($thunk, @$our_edges);
2191 }
2192  
2193 sub _jit_dfa_node { my ($lexer, $node) = @_;
2194 my $nfa2dfa = sub { my $nbm = shift;
2195 $lexer->{NFA2DFA}->{$nbm} //= do {
2196 my @node;
2197 $node[1] = { ID => scalar(@{ $lexer->{DFA} }), BITS => $nbm };
2198 push @{ $lexer->{DFA} }, \@node;
2199 \@node;
2200 }
2201 };
2202  
2203 my $bf = $node->[1]{BITS};
2204 my $id = $node->[1]{ID};
2205 my $nfa = $lexer->{NFA};
2206  
2207 my %black;
2208 my @nfixes = grep { vec($bf, $_, 1) } (0 .. length($bf)*8 - 1);
2209 my @grey = @nfixes;
2210 my @ouredges;
2211  
2212 while (@grey) {
2213 my $nix = pop @grey;
2214 next if $black{$nix};
2215 $black{$nix} = 1;
2216 my $nfn = $nfa->[$nix];
2217  
2218 push @{ $node->[0] }, $nfn->[1] if $nfn->[1];
2219 for (my $i = 2; $i < @$nfn; $i += 2) {
2220 if (!$nfn->[$i]) {
2221 push @grey, $nfn->[$i+1];
2222 } else {
2223 push @ouredges, $nfn->[$i], $nfn->[$i+1];
2224 }
2225 }
2226 }
2227  
2228 for my $fate (@{ $node->[0] }) {
2229 my @a = reverse @$fate;
2230 my $fo = undef;
2231 my $tb = "";
2232 for (my $i = 0; $i < @a - 1; $i += 3) {
2233 $tb = $a[$i] . $tb;
2234 $fo = [ $fo, $a[$i+2], $a[$i+1] ];
2235 }
2236 $fo = [ $tb, $fo ];
2237 $fate = $fo;
2238 }
2239 # Suppose two fates are different. They must have a first point of
2240 # divergence. This point cannot be a quantifier, because then both fates
2241 # would end at the quantifier; so it must be a LTM alternation. But any
2242 # LTM alternation would generate different sort keys for the sides.
2243 my @sfates = sort { $b->[0] cmp $a->[0] } @{ $node->[0] };
2244 @{ $node->[0] } = ();
2245 for (my $i = 0; $i < @sfates; $i++) {
2246 next if ($i < @sfates-1 && $sfates[$i][0] eq $sfates[$i+1][0]);
2247 push @{ $node->[0] }, $sfates[$i][1];
2248 }
2249  
2250 pop @$node;
2251 push @$node, _tangle_edges(\@ouredges, $nfa2dfa);
2252 $node->[1]{DESC} = $id . "{" . join(",", @nfixes) . "}";
2253 $node->[1]{ID} = $id;
2254  
2255 if (DEBUG & DEBUG::autolexer) {
2256 print ::LOG "JIT DFA node generation:\n";
2257 _dfa_dump_node($node);
2258 }
2259  
2260 $node->[0];
2261 }
2262  
2263 sub _scan_regexes { my ($class, $key) = @_;
2264 no strict 'refs';
2265 (${ $class . "::REGEXES" } //= do {
2266 my $stash = \ %{ $class . "::" };
2267 my %over;
2268 my %proto;
2269  
2270 for my $m (keys %$stash) {
2271 next if ref $stash->{$m}; # use constant
2272 next if !defined *{$stash->{$m}}{CODE};
2273 my ($meth, $p) = $m =~ /^(.*?)(__S_\d\d\d.*)?__PEEK$/ or next;
2274 #$self->deb("\tfound override for $meth in $m") if DEBUG & DEBUG::autolexer;
2275 $over{$meth} = 1;
2276 push @{$proto{$meth}}, $m if $p;
2277 }
2278  
2279 for (keys %proto) {
2280 @{$proto{$_}} = sort @{$proto{$_}};
2281 }
2282  
2283 $proto{ALL} = [ keys %over ];
2284 \%proto;
2285 })->{$key};
2286 }
2287  
2288 sub _AUTOLEXgenDFA { my ($self, $realkey, $key, $retree) = @_;
2289 local $::AUTOLEXED{$realkey} = $fakepos;
2290  
2291 my $lang = ref $self;
2292  
2293 $self->deb("AUTOLEXgen $key in $lang") if DEBUG & DEBUG::autolexer;
2294 my $ast = $retree->{$key};
2295  
2296 UP: {
2297 # Whenever possible, we want to share a lexer amongst as many grammars
2298 # as we can. So we try to float lexers up to superclasses.
2299  
2300 no strict 'refs';
2301 my $isa = \@{ $lang . "::ISA" };
2302  
2303 # We don't support multiple inheritance (can we?)
2304 if (@$isa != 1) {
2305 $self->deb("\tcannot reuse $key; multiply inherited") if DEBUG & DEBUG::autolexer;
2306 last;
2307 }
2308  
2309 my $super = $isa->[0];
2310  
2311 my $dic = $ast->{dic} //= do {
2312 my $i = 1; # skip _AUTOLEXpeek;
2313 my $pkg = 'CursorBase';
2314 $pkg = caller($i++) while $pkg eq 'CursorBase';
2315 #print STDERR "dic run: $pkg\n";
2316 $self->deb("\tdeclared in class $pkg") if DEBUG & DEBUG::autolexer;
2317 $pkg;
2318 };
2319  
2320 my $ar = ${ $lang . "::ALLROLES" } //= do {
2321 +{ map { $_->name, 1 } ($lang->meta, $lang->meta->calculate_all_roles) }
2322 };
2323  
2324 # It doesn't make sense to float a lexer above STD::Cursor, or (for 'has'
2325 # regexes), the class of definition.
2326 if ($ar->{$dic}) {
2327 $self->deb("\tcannot reuse $key; at top") if DEBUG & DEBUG::autolexer;
2328 last;
2329 }
2330  
2331 my $supercursor = $self->cursor_fresh($super);
2332 my $superlexer = eval {
2333 local %::AUTOLEXED;
2334 $supercursor->_AUTOLEXpeek($key, $retree)
2335 };
2336  
2337 if (!$superlexer) {
2338 $self->deb("\tcannot reuse $key; failed ($@)") if DEBUG & DEBUG::autolexer;
2339 last;
2340 }
2341  
2342 my $ml = _scan_regexes($lang, 'ALL');
2343  
2344 for my $meth (@$ml) {
2345 if ($superlexer->{USED_METHODS}{$meth}) {
2346 $self->deb("\tcannot reuse $key; $meth overriden/augmented")
2347 if DEBUG & DEBUG::autolexer;
2348 last UP;
2349 }
2350 }
2351  
2352 $self->deb("\treusing ($key, $realkey, $lang, $super).") if DEBUG & DEBUG::autolexer;
2353 return $superlexer;
2354 }
2355  
2356 my $dba = $ast->{dba};
2357  
2358 my $d = DEBUG & DEBUG::autolexer;
2359 print ::LOG "generating DFA lexer for $key -->\n" if $d;
2360 my $nfa;
2361  
2362 if ($key =~ /(.*):\*$/) {
2363 my $proto = $1;
2364 $dba = $proto;
2365 my $protopat = $1 . '__S_';
2366 my $protolen = length($protopat);
2367 my @alts;
2368 my $j = 0;
2369 my @stack = $lang;
2370  
2371 while (@stack) {
2372 no strict 'refs';
2373 my $class = pop @stack;
2374 push @stack, reverse @{ $class . "::ISA" };
2375 my @methods = @{ _scan_regexes($class, $proto) // [] };
2376 for my $method (@methods) {
2377 my $callname = $class . '::' . $method;
2378 $method = substr($method, 0, length($method)-6);
2379 my $peeklex = $self->$callname();
2380 die "$proto has no lexer!?" unless $peeklex->{NFAT};
2381  
2382 push @alts, ["${class}::$method", $peeklex->{NFAT}];
2383 }
2384 }
2385  
2386 $nfa = nfa::method($proto, nfa::ltm($proto, @alts));
2387 } elsif ($ast) {
2388 $nfa = $ast->nfa($self);
2389 } else {
2390 die "BAD KEY";
2391 }
2392  
2393 die "dba unspecified" unless $dba;
2394  
2395 local @::NFANODES;
2396 nfa::gnode($nfa, [0], undef);
2397 my $nfag = \@::NFANODES;
2398 my %usedmethods = map { $_, 1 } @{ $nfa->{m} };
2399 $nfa->{m} = [ sort keys %usedmethods ];
2400  
2401 _dump_nfa($key, $nfag) if $d;
2402 print ::LOG "used methods: ", join(" ", sort keys %usedmethods), "\n" if $d;
2403  
2404 my $dfa = CORE::bless [], 'CursorBase::dfa';
2405 push @$dfa, [ undef, { BITS => "\001", ID => 0 } ];
2406  
2407 { DBA => $dba, DFA => $dfa, NFA2DFA => { "\001" => $dfa->[0] },
2408 NFA => $nfag, NFAT => $nfa, S => $dfa->[0], USED_METHODS => \%usedmethods };
2409 }
2410  
2411 sub _AUTOLEXpeek { my $self = shift;
2412 my $key = shift;
2413 my $retree = shift;
2414  
2415 # protoregexes are identified by name
2416 my $realkey = refaddr($retree->{$key}) // $key;
2417  
2418 $self->deb("AUTOLEXpeek $key") if DEBUG & DEBUG::autolexer;
2419 die "Null key" if $key eq '';
2420 if ($::AUTOLEXED{$realkey}) { # no left recursion allowed in lexer!
2421 die "Left recursion in $key" if $fakepos == $::AUTOLEXED{$realkey};
2422 $self->deb("Suppressing lexer recursion on $key") if DEBUG & DEBUG::autolexer;
2423 return { USED_METHODS => {}, NFAT => $nfa::IMP }; # (but if we advanced just assume a :: here)
2424 }
2425 $key = 'termish' if $key eq 'EXPR';
2426 return $::LEXERS{ref $self}->{$realkey} //= do {
2427 $self->_AUTOLEXgenDFA($realkey, $key, $retree);
2428 };
2429 }
2430  
2431 #############################################################
2432 # Parser service routines
2433 #############################################################
2434  
2435 sub O {
2436 my $self = shift;
2437 my %args = @_;
2438 @$self{keys %args} = values %args;
2439 $self;
2440 }
2441  
2442 sub Opairs {
2443 my $self = shift;
2444 my $O = $self->{O} or return ();
2445 my @ret;
2446 for (my ($k,$v) = each %$O) {
2447 push @ret, $k, $v;
2448 }
2449 @ret;
2450 }
2451  
2452 sub gettrait {
2453 my $self = shift;
2454 my $traitname = shift;
2455 my $param = shift;
2456 my $text;
2457 if (@$param) {
2458 $text = $param->[0]->Str;
2459 $text =~ s/^<(.*)>$/$1/ or
2460 $text =~ s/^\((.*)\)$/$1/;
2461 }
2462 if ($traitname eq 'export') {
2463 if (defined $text) {
2464 $text =~ s/://g;
2465 }
2466 else {
2467 $text = 'DEFAULT';
2468 }
2469 $self->set_export($text);
2470 $text;
2471 }
2472 elsif (defined $text) {
2473 $text;
2474 }
2475 else {
2476 1;
2477 }
2478 }
2479  
2480 sub set_export {
2481 my $self = shift;
2482 my $text = shift;
2483 my $textpkg = $text . '::';
2484 my $name = $::DECLARAND->{name};
2485 my $xlex = $STD::ALL->{ ($::DECLARAND->{inlex})->[0] };
2486 $::DECLARAND->{export} = $text;
2487 my $sid = $::CURLEX->idref;
2488 my $x = $xlex->{'EXPORT::'} //= Stash::->new( 'PARENT::' => $sid, '!id' => [$sid->[0] . '::EXPORT'] );
2489 $x->{$textpkg} //= Stash::->new( 'PARENT::' => $x->idref, '!id' => [$sid->[0] . '::EXPORT::' . $text] );
2490 $x->{$textpkg}->{$name} = $::DECLARAND;
2491 $x->{$textpkg}->{'&'.$name} = $::DECLARAND
2492 if $name =~ /^\w/ and $::IN_DECL ne 'constant';
2493 $self;
2494 }
2495  
2496 sub mixin {
2497 my $self = shift;
2498 my $WHAT = ref($self)||$self;
2499 my @mixins = @_;
2500  
2501 my $NEWWHAT = $WHAT . '::';
2502 my @newmix;
2503 for my $mixin (@mixins) {
2504 my $ext = ref($mixin) || $mixin;
2505 push @newmix, $ext;
2506 $ext =~ s/(\w)\w*::/$1/g; # just looking for a "cache" key, really
2507 $NEWWHAT .= '_X_' . $ext;
2508 }
2509 $self->deb("mixin $NEWWHAT from $WHAT @newmix") if DEBUG & DEBUG::mixins;
2510 no strict 'refs';
2511 if (not exists &{$NEWWHAT.'::meta'}) { # never composed this one yet?
2512 # fake up mixin with MI, being sure to put "roles" in front
2513 my $eval = "package $NEWWHAT; use Moose ':all' => { -prefix => 'moose_' }; moose_extends('$WHAT'); moose_with(" . join(',', map {"'$_'"} @newmix) . ");our \$CATEGORY = '.';\n";
2514  
2515 $self->deb($eval) if DEBUG & DEBUG::mixins;
2516 local $SIG{__WARN__} = sub { die $_[0] unless $_[0] =~ /^Deep recursion/ };
2517 eval $eval;
2518 warn $@ if $@;
2519 }
2520 return $self->cursor_fresh($NEWWHAT);
2521 }
2522  
2523 sub tweak {
2524 my $self = shift;
2525 my $class = ref $self;
2526 no strict 'refs';
2527 for (;;) {
2528 my $retval = eval {
2529 $self->deb("Calling $class" . '::multitweak') if DEBUG & DEBUG::mixins;
2530 &{$class . '::multitweak'}($self,@_);
2531 };
2532 return $retval if $retval;
2533 die $@ unless $@ =~ /^NOMATCH|^Undefined subroutine/;
2534 last unless $class =~ s/(.*)::.*/$1/;
2535 }
2536 }
2537  
2538 # only used for error reporting
2539 sub clean_id { my $self = shift;
2540 my ($id,$name) = @_;
2541 my $file = $::FILE->{name};
2542  
2543 $id .= '::';
2544 $id =~ s/^MY:file<CORE.setting>.*?::/CORE::/;
2545 $id =~ s/^MY:file<\w+.setting>.*?::/SETTING::/;
2546 $id =~ s/^MY:file<\Q$file\E>$/UNIT/;
2547 $id =~ s/:pos\(\d+\)//;
2548 $id .= "<$name>";
2549 $id;
2550 }
2551  
2552 # remove consistent leading whitespace (mutates text nibbles in place)
2553  
2554 sub trim_heredoc { my $doc = shift;
2555 my ($stopper) = $doc->stopper or
2556 $doc->panic("Couldn't find delimiter for heredoc\n");
2557 my $ws = $stopper->{ws}->Str;
2558 return $stopper if $ws eq '';
2559  
2560 my $wsequiv = $ws;
2561 $wsequiv =~ s{^(\t+)}[' ' x (length($1) * ($::TABSTOP // 8))]xe;
2562  
2563 # We can't use ^^ after escapes, since the escape may be mid-line
2564 # and we'd get false positives. Use a fake newline instead.
2565 $doc->{nibbles}[0] =~ s/^/\n/;
2566  
2567 for (@{$doc->{nibbles}}) {
2568 next if ref $_; # next unless $_ =~ Str;
2569  
2570 # prefer exact match over any ws
2571 s{(?<=\n)(\Q$ws\E|[ \t]+)}{
2572 my $white = $1;
2573 if ($white eq $ws) {
2574 '';
2575 }
2576 else {
2577 $white =~ s[^ (\t+) ][ ' ' x (length($1) * ($::TABSTOP // 8)) ]xe;
2578 if ($white =~ s/^\Q$wsequiv\E//) {
2579 $white;
2580 }
2581 else {
2582 '';
2583 }
2584 }
2585 }eg;
2586 }
2587 $doc->{nibbles}[0] =~ s/^\n//; # undo fake newline
2588 $stopper;
2589 }
2590  
2591 sub add_categorical { my $lang = shift;
2592 my $name = shift;
2593 state $GEN = "500";
2594 $name =~ s/:<<(.*)>>$/:«$1»/;
2595 my $WHAT = ref $lang;
2596  
2597 # :() is a signature modifier, not an operator
2598 if ($name =~ /^\w+:\(/) {
2599 # XXX canonicalize sig here eventually
2600 $lang->add_my_name($name);
2601 return $lang;
2602 }
2603  
2604 if ($name =~ s/^(\w+):(?=[«<{[])/$1:sym/) {
2605 my $cat = $1;
2606 my ($sym) = $name =~ /:sym(.*)/;
2607 $sym =~ s/^<\s*(.*\S)\s*>$/<$1>/g;
2608 $sym =~ s/^\[\s*(.*\S)\s*\]$/$1/g;
2609 if ( $sym =~ s/\\x\[(.*)\]/\\x{$1}/g) {
2610 $sym = '«' . eval($sym) . '»';
2611 }
2612 elsif ($sym =~ s/\\c\[(.*)\]/\\N{$1}/g ) {
2613 $sym = '«' . eval("use charnames ':full'; $sym") . '»';
2614 }
2615  
2616 # unfortunately p5 doesn't understand q«...»
2617 if ($sym =~ s/^«\s*(.*\S)\s*»$/$1/) {
2618 my $ok = "'";
2619 for my $try (qw( ' / ! : ; | + - = )) {
2620 $ok = $try, last if index($sym,$try) < 0;
2621 }
2622 $sym = $ok . $sym . $ok;
2623 }
2624 {
2625 my $canon = substr($sym,1,length($sym)-2);
2626 $canon =~ s/([<>])/\\$1/g;
2627 my $canonname = $cat . ':<' . $canon . '>';
2628 $lang->add_my_name($canonname);
2629 }
2630 if ($sym =~ / /) {
2631 $sym = '[qw' . $sym . ']';
2632 }
2633 else {
2634 $sym = 'q' . $sym;
2635 }
2636  
2637 my $rule = "token $name { <sym> }";
2638  
2639 # produce p5 method name
2640 my $mangle = $name;
2641 $mangle =~ s/^(\w*):(sym)?//;
2642 my $category = $1;
2643 my @list;
2644 if ($mangle =~ s/^<(.*)>$/$1/ or
2645 $mangle =~ s/^«(.*)»$/$1/) {
2646 $mangle =~ s/\\(.)/$1/g;
2647 @list = $mangle =~ /(\S+)/g;
2648 }
2649 elsif ($mangle =~ s/^\[(.*)\]$/$1/ or
2650 $mangle =~ s/^\{(.*)\}$/$1/) {
2651 $mangle =~ s/\\x\[(.*)\]/\\x{$1}/g;
2652 @list = eval $mangle;
2653 }
2654 elsif ($mangle =~ m/^\(\"(.*)\"\)$/) {
2655 @list = eval $sym;
2656 }
2657 else {
2658 @list = $mangle;
2659 }
2660 $mangle = ::mangle(@list);
2661 $mangle = $category . '__S_' . sprintf("%03d",$GEN++) . $mangle;
2662  
2663 # XXX assuming no equiv specified, but to do that right,
2664 # this should be delayed till the block start is seen
2665 my $coercion = '';
2666 if ($name =~ /^infix:/) {
2667 $coercion = 'additive';
2668 }
2669 elsif ($name =~ /^prefix:/) {
2670 if ($sym =~ /^.\W/) {
2671 $coercion = 'symbolic_unary';
2672 }
2673 else {
2674 $coercion = 'named_unary';
2675 }
2676 }
2677 elsif ($name =~ /^postfix:/) {
2678 $coercion = 'methodcall';
2679 }
2680 elsif ($name =~ /^circumfix:/) {
2681 $coercion = 'term';
2682 }
2683 elsif ($name =~ /^postcircumfix:/) {
2684 $coercion = 'methodcall';
2685 }
2686 elsif ($name =~ /^term:/) {
2687 $coercion = 'term';
2688 }
2689  
2690 state $genpkg = 'ANON000';
2691 $genpkg++;
2692 my $e;
2693 if (@list == 1) {
2694 $e = <<"END";
2695 package $genpkg;
2696 use Moose ':all' => { -prefix => 'moose_' };
2697 moose_extends('$WHAT');
2698  
2699 # $rule
2700  
2701 my \$retree = {
2702 '$mangle' => bless({
2703 'dba' => '$category expression',
2704 'kind' => 'token',
2705 'min' => 12345,
2706 're' => bless({
2707 'a' => 0,
2708 'i' => 0,
2709 'min' => 12345,
2710 'name' => 'sym',
2711 'rest' => '',
2712 'sym' => $sym,
2713 }, 'RE_method'),
2714 }, 'RE_ast'),
2715 };
2716  
2717 our \$CATEGORY = '$category';
2718  
2719 sub ${mangle}__PEEK { \$_[0]->_AUTOLEXpeek('$mangle',\$retree) }
2720 sub $mangle {
2721 my \$self = shift;
2722 local \$CTX = \$self->callm() if \$::DEBUG & DEBUG::trace_call;
2723 my %args = \@_;
2724 my \$sym = \$args{sym} // $sym;
2725  
2726 my \$xact = ['RULE $mangle', 0, \$::XACT];
2727 local \$::XACT = \$xact;
2728  
2729 my \$S = \$self->{_pos};
2730 my \$C = \$self->cursor_xact("RULE $mangle \$S");
2731 # my \$xact = \$C->xact;
2732  
2733 \$C->{'sym'} = \$sym;
2734  
2735 \$self->_MATCHIFY(\$S, '$mangle',
2736 do {
2737 if (my (\$C) = (\$C->_SYM(\$sym, 0))) {
2738 \$C->_SUBSUMEr(['O'], sub {
2739 my \$C = shift;
2740 \$C->O(%STD::$coercion)
2741 });
2742 }
2743 else {
2744 ();
2745 }
2746 }
2747 );
2748 }
2749 1;
2750 END
2751 }
2752 else {
2753 for (@list) {
2754 if (/'/) {
2755 s/(.*)/"$1"/;
2756 }
2757 else {
2758 s/(.*)/'$1'/;
2759 }
2760 }
2761 my $starter = $list[0];
2762 my $stopper = $list[1];
2763  
2764 $e = <<"END";
2765 package $genpkg;
2766 use Moose ':all' => { -prefix => 'moose_' };
2767 moose_extends('$WHAT');
2768  
2769 # $rule
2770  
2771 my \$retree = {
2772 '$mangle' => bless({
2773 'dba' => '$category expression',
2774 'kind' => 'token',
2775 'min' => 12347,
2776 'pkg' => undef,
2777 're' => bless({
2778 'decl' => [],
2779 'a' => 0,
2780 'dba' => '$category expression',
2781 'i' => 0,
2782 'min' => 12347,
2783 'r' => 1,
2784 's' => 0,
2785 'zyg' => [
2786 bless({
2787 'a' => 0,
2788 'dba' => '$category expression',
2789 'i' => 0,
2790 'min' => 1,
2791 'r' => 1,
2792 's' => 0,
2793 'text' => $starter,
2794 }, 'RE_string'),
2795 bless({
2796 'a' => 0,
2797 'dba' => '$category expression',
2798 'i' => 0,
2799 'min' => 12345,
2800 'name' => 'semilist',
2801 'r' => 1,
2802 'rest' => '',
2803 's' => 0,
2804 }, 'RE_method'),
2805 bless({
2806 'decl' => [],
2807 'min' => 1,
2808 're' => bless({
2809 'a' => 0,
2810 'dba' => '$category expression',
2811 'i' => 0,
2812 'min' => 1,
2813 'r' => 1,
2814 's' => 0,
2815 'zyg' => [
2816 bless({
2817 'a' => 0,
2818 'dba' => '$category expression',
2819 'i' => 0,
2820 'min' => 1,
2821 'r' => 1,
2822 's' => 0,
2823 'text' => $stopper,
2824 }, 'RE_string'),
2825 bless({
2826 'min' => 0,
2827 'name' => 'FAILGOAL',
2828 'nobind' => 1,
2829 }, 'RE_method'),
2830 ],
2831 }, 'RE_first'),
2832 }, 'RE_bracket'),
2833 bless({
2834 'min' => 0,
2835 'name' => 'O',
2836 'rest' => '(|%term)',
2837 }, 'RE_method'),
2838 ],
2839 }, 'RE_sequence'),
2840 }, 'RE_ast'),
2841 };
2842  
2843 our \$CATEGORY = '$category';
2844  
2845 sub ${mangle}__PEEK { \$_[0]->_AUTOLEXpeek('$mangle',\$retree) }
2846 sub $mangle {
2847 no warnings 'recursion';
2848 my \$self = shift;
2849 local \$::CTX = \$self->callm() if \$::DEBUG & DEBUG::trace_call;
2850 my %args = \@_;
2851 local \$::sym = \$args{sym} // $sym;
2852 return () if \$::GOAL eq $starter;
2853  
2854 my \$C = \$self->cursor_xact("RULE $mangle");
2855 my \$xact = \$C->xact;
2856 my \$S = \$C->{'_pos'};
2857 \$C->{'sym'} = ref \$sym ? join(' ', \@\$sym) : \$sym;
2858  
2859 \$self->_MATCHIFYr(\$S, "$mangle",
2860 do {
2861 if (my (\$C) = (\$C->_EXACT($starter))) {
2862 do {
2863 if (my (\$C) = (((local \$::GOAL = $stopper , my \$goalpos = \$C), \$C->unbalanced($stopper))[-1])) {
2864 do {
2865 if (my (\$C) = (\$C->_SUBSUMEr(['semilist'], sub {
2866 my \$C = shift;
2867 \$C->semilist
2868 }))) {
2869 do {
2870 if (my (\$C) = (\$C->_BRACKETr(sub {
2871 my \$C=shift;
2872 do {
2873 my \$C = \$C->cursor_xact('ALT ||');
2874 my \$xact = \$C->xact;
2875 my \@gather;
2876 do {
2877 push \@gather, \$C->_EXACT($stopper)
2878 }
2879 or \$xact->[-2] or
2880 do {
2881 push \@gather, \$C->FAILGOAL($stopper , '$category expression',\$goalpos)};
2882 \@gather;
2883 }
2884 }))) {
2885 \$C->_SUBSUMEr(['O'], sub {
2886 my \$C = shift;
2887 \$C->O(%STD::$coercion)
2888 });
2889 }
2890 else {
2891 ();
2892 }
2893 };
2894 }
2895 else {
2896 ();
2897 }
2898 };
2899 }
2900 else {
2901 ();
2902 }
2903 };
2904 }
2905 else {
2906 ();
2907 }
2908 }
2909 );
2910 }
2911  
2912 1;
2913 END
2914 }
2915 $lang->deb("derive $genpkg from $WHAT adding $mangle") if DEBUG & DEBUG::mixins;
2916 eval $e or die "Can't create $name: $@\n";
2917 $::LANG{'MAIN'} = $lang->cursor_fresh($genpkg);
2918 }
2919 $lang;
2920 }
2921  
2922 sub add_enum { my $self = shift;
2923 my $type = shift;
2924 my $expr = shift;
2925 return unless $type;
2926 return unless $expr;
2927 my $typename = $type->Str;
2928 local $::IN_DECL = 'constant';
2929 # XXX complete kludge, really need to eval EXPR
2930 $expr =~ s/:(\w+)<\S+>/$1/g; # handle :name<string>
2931 for ($expr =~ m/([a-zA-Z_]\w*)/g) {
2932 $self->add_name($typename . "::$_");
2933 $self->add_name($_);
2934 }
2935 $self;
2936 }
2937  
2938 sub do_use { my $self = shift;
2939 my $module = shift;
2940 my $args = shift;
2941 my @imports;
2942  
2943 $self->do_need($module);
2944 $self->do_import($module,$args);
2945 $self;
2946 }
2947  
2948 sub do_need { my $self = shift;
2949 my $m = shift;
2950 my $module = $m->Str;
2951 my $topsym = $self->sys_load_modinfo($module);
2952 $self->add_my_name($module);
2953 $::DECLARAND->{really} = $topsym;
2954 $self;
2955 }
2956  
2957 sub do_import { my $self = shift;
2958 my $m = shift;
2959 my $args = shift;
2960 my @imports;
2961 my $module = $m->Str;
2962 if ($module =~ /(class|module|role|package)\s+(\S+)/) {
2963 $module = $2;
2964 }
2965  
2966 my $pkg = $self->find_stash($module);
2967 if ($pkg->{really}) {
2968 $pkg = $pkg->{really}->{UNIT};
2969 }
2970 else {
2971 $pkg = $self->find_stash($module . '::');
2972 }
2973 if ($args) {
2974 my $text = $args->Str;
2975 return $self unless $text;
2976 while ($text =~ s/^\s*:?(OUR|MY|STATE|HAS|AUGMENT|SUPERSEDE)?<(.*?)>,?//) {
2977 my $scope = lc($1 // 'my');
2978 my $imports = $2;
2979 local $::SCOPE = $scope;
2980 @imports = split ' ', $imports;
2981 for (@imports) {
2982 if ($pkg) {
2983 if ($_ =~ s/^://) {
2984 my @tagimports;
2985 eval { @tagimports = keys %{ $pkg->{'EXPORT::'}->{$_} }; };
2986 $self->do_import_aliases($pkg, @tagimports);
2987 }
2988 elsif ($pkg->{$_}{export}) {
2989 $self->add_my_name($_, $pkg->{$_});
2990 }
2991 elsif ($pkg->{'&'.$_}{export}) {
2992 $_ = '&' . $_;
2993 $self->add_my_name($_, $pkg->{$_});
2994 }
2995 elsif ($pkg->{$_}) {
2996 $self->worry("Can't import $_ because it's not exported by $module");
2997 next;
2998 }
2999 }
3000 else {
3001 $self->add_my_name($_);
3002 }
3003 }
3004 }
3005 }
3006 else {
3007 return $self unless $pkg;
3008 eval { @imports = keys %{ $pkg->{'EXPORT::'}->{'DEFAULT::'} }; };
3009 local $::SCOPE = 'my';
3010 $self->do_import_aliases($pkg, @imports);
3011 }
3012  
3013 $self;
3014 }
3015  
3016 sub do_import_aliases {
3017 my $self = shift;
3018 my $pkg = shift;
3019 # say "attempting to import @_";
3020 for (@_) {
3021 next if /^!/;
3022 next if /^PARENT::/;
3023 next if /^OUTER::/;
3024 $self->add_my_name($_, $pkg->{$_});
3025 }
3026 $self;
3027 }
3028  
3029 sub canonicalize_name { my $self = shift;
3030 my $name = shift;
3031 $name =~ s/^([\$\@\%\&])(\^|:(?!:))/$1/;
3032 $name =~ s/\b:[UD_]$//;
3033 return $name unless $name =~ /::/;
3034 $self->panic("Can't canonicalize a run-time name at compile time: $name") if $name =~ /::\(/;
3035 $name =~ s/^([\$\@%&][!*=?:^.]?)(.*::)(.*)$/$2<$1$3>/;
3036 my $vname;
3037 if ($name =~ s/::<(.*)>$//) {
3038 $vname = $1;
3039 }
3040 my @components = split(/(?<=::)/, $name, -1);
3041 shift(@components) while @components and $components[0] eq '';
3042 if (defined $vname) {
3043 $components[-1] .= '::' if @components and $components[-1] !~ /::$/;
3044 push(@components, $vname) if defined $vname;
3045 }
3046 @components;
3047 }
3048  
3049 sub lookup_dynvar { my $self = shift;
3050 my $name = shift;
3051 no strict 'refs';
3052 if ($name =~ s/^\$\?/::/) {
3053 return $$name if defined $$name;
3054 }
3055 elsif ($name =~ s/^\@\?/::/) {
3056 return \@$name if defined *$name{ARRAY};
3057 }
3058 elsif ($name =~ s/^\%\?/::/) {
3059 return \%$name if defined *$name{HASH};
3060 }
3061 return
3062 }
3063  
3064 sub mark_sinks { my $self = shift;
3065 my $statements = shift;
3066 return $self unless @$statements;
3067 my @s = @$statements;
3068 my $final = pop(@s);
3069 for my $s (@s) {
3070 if ($s->is_pure) {
3071 $self->worry("Useless use of " . $s->Str . " in sink context");
3072 }
3073 $s->{_pure} = 1; # nothing is pure :)
3074 $s->{_sink} = 1;
3075 }
3076 $self;
3077 }
3078  
3079 sub is_pure { my $self = shift;
3080 return 1 if $self->{_pure};
3081 # visit kids here?
3082 return 0;
3083 }
3084  
3085 sub check_old_cclass { my $self = shift;
3086 my $innards = shift;
3087  
3088 my $prev = substr($::ORIG,$self->{_pos}-length($innards)-4,2);
3089 return $self if $prev =~ /=\s*$/; # don't complain on $var = [\S] capture
3090  
3091 my $cclass = '';
3092 my $single = '';
3093 my $singleok = 1;
3094 my $double = '';
3095 my $doubleok = 1;
3096  
3097 my $last = '';
3098 my %seen;
3099  
3100 my $i = $innards;
3101 my $neg = '';
3102 $neg = '-' if $i =~ s/^\^//;
3103 my $digits = 0;
3104 $i =~ s/0-9/\\d/;
3105 while ($i ne '') {
3106 if ($i =~ s/^-(.)/$1/) {
3107 $singleok = $doubleok = 0;
3108 $cclass .= $last ? '..' : '\\-';
3109 $last = '';
3110 }
3111 elsif ($i =~ /^\|./ and $cclass ne '') {
3112 return $self; # probable alternation
3113 }
3114 elsif ($i =~ s/^\|//) {
3115 $last = '';
3116 $singleok = $doubleok = 0;
3117 $cclass .= '|';
3118 }
3119 elsif ($i =~ /^[*+?]/ and $cclass ne '') {
3120 return $self; # probable quantifier
3121 }
3122 elsif ($i =~ s/^\\?'//) {
3123 $last = "'";
3124 $single .= '\\' . $last;
3125 $double .= $last;
3126 $cclass .= $last;
3127 }
3128 elsif ($i =~ s/^\\?"//) {
3129 $last = '"';
3130 $single .= $last;
3131 $double .= '\\' . $last;
3132 $cclass .= $last;
3133 }
3134 elsif ($i =~ s/^(\\[btnrf0])//) {
3135 $last = eval '"' . $1 . '"';
3136 $single .= $last;
3137 $double .= $1;
3138 $cclass .= $1;
3139 }
3140 elsif ($i =~ s/(\\x\w\w)//) {
3141 $last = eval '"' . $1 . '"';
3142 $single .= $last;
3143 $double .= $1;
3144 $cclass .= $1;
3145 }
3146 elsif ($i =~ s/(\\0[0-7]{1,3})//) {
3147 $last = eval '"' . $1 . '"';
3148 $single .= $last;
3149 $double .= "\\o" . substr($1,1);
3150 $cclass .= "\\o" . substr($1,1);
3151 }
3152 elsif ($i =~ s/^(\\[sSwWdD])//) {
3153 $singleok = $doubleok = 0;
3154 $last = '';
3155 $cclass .= $1;
3156 }
3157 elsif ($i =~ s/^(\\?\t)//) {
3158 $last = "\t";
3159 $single .= $last;
3160 $double .= '\\t';
3161 $cclass .= '\\t';
3162 }
3163 elsif ($i =~ s/^(\\?\x20)//) {
3164 $last = ' ';
3165 $single .= $last;
3166 $double .= $last;
3167 $cclass .= '\\x20';
3168 }
3169 elsif ($i =~ s/^\.//) {
3170 $last = '.';
3171 $singleok = $doubleok = 0;
3172 $cclass .= '.';
3173 }
3174 elsif ($i =~ s/^\\(.)//) {
3175 $last = $1;
3176 $single .= $last;
3177 $double .= '\\' . $last;
3178 $cclass .= '\\' . $last;
3179 }
3180 elsif ($i =~ s/^(.)//s) {
3181 $last = $1;
3182 $cclass .= $last;
3183 $single .= $last;
3184 $double .= $last;
3185 }
3186 else {
3187 die "can't happen";
3188 }
3189  
3190 if ($last ne '' and $seen{$last}++) {
3191 return $self; # dup likely indicates not a character class
3192 }
3193 }
3194  
3195 my $common = "[$innards] appears to be an old-school character class;";
3196  
3197 # XXX not Unicodey yet
3198 if ($neg) {
3199 return $self->worry("$common non-digits should be matched with \\D instead") if $cclass eq '\\d';
3200 return $self->worry("$common non-newlines should be matched with \\N instead") if $cclass eq '\\n';
3201 if ($singleok) {
3202 return $self->worry("$common non-(horizontal whitespace) should be matched with \\H instead") if $single =~ /\A[ \t\b\r]*\z/;
3203 return $self->worry("$common non-(vertical whitespace) should be matched with \\V instead") if $single =~ /\A[\n\f]*\z/;
3204 return $self->worry("$common non-whitespace should be matched with \\S instead") if $single =~ /\A[ \t\b\r\n\f]*\z/;
3205 return $self->worry("$common please use <-[$cclass]> if you mean a character class");
3206 }
3207 elsif ($doubleok) {
3208 return $self->worry("$common please use <-[$cclass]> if you mean a character class");
3209 }
3210 }
3211 else {
3212 return $self->worry("$common digits should be matched with \\d instead") if $cclass eq '\\d';
3213 if ($singleok) {
3214 return $self->worry("$common horizontal whitespace should be matched with \\h instead") if $single =~ /\A[ \t\b\r]*\z/;
3215 return $self->worry("$common vertical whitespace should be matched with \\v instead") if $single =~ /\A[\n\f]*\z/;
3216 return $self->worry("$common whitespace should be matched with \\s instead") if $single =~ /\A[ \t\b\r\n\f]*\z/;
3217 }
3218 if ($singleok and $single eq $double) {
3219 return $self->worry("$common please use <[$cclass]> if you\n mean a character class, or quote it like '$single' to match\n string as a unit");
3220 }
3221 elsif ($doubleok) {
3222 return $self->worry("$common please use <[$cclass]> if you\n mean a character class, or quote it like \"$double\" to match\n string as a unit");
3223 }
3224 }
3225 if ($::FATALS) {
3226 return $self->worry("$common please use <${neg}[$cclass]> if you mean a character class");
3227 }
3228 else {
3229 return $self->worry("$common please use <${neg}[$cclass]> if you\n mean a character class, or put whitespace inside like [ $innards ] to disable\n this warning");
3230 }
3231 $self;
3232 }
3233  
3234 ## vim: expandtab sw=4