lib/LaTeXML/MathParser.pm untested

filter: Statementstmt 0.0% 0 / 1058 Branchbran 0.0% 0 / 514 Conditioncond 0.0% 0 / 366 MC/DCmcdc 0.0% 0 / 212 Subroutinesub 0.0% 0 / 84 Podpod 0.0% 0 / 68 totaltotal 0.0% 0 / 2302 CC 258 SCAR 45.8
CC
258
Cov
0%
filter_hints
380.0
NewScript
306.0
parse_rec
272.0
CRAP 97.3
1 # /=====================================================================\ #
2 # | LaTeXML::MathParser | #
3 # | Parse Math | #
4 # |=====================================================================| #
5 # | Part of LaTeXML: | #
6 # | Public domain software, produced as part of work done by the | #
7 # | United States Government & not subject to copyright in the US. | #
8 # |---------------------------------------------------------------------| #
9 # | Bruce Miller <bruce.miller@nist.gov> #_# | #
10 # | http://dlmf.nist.gov/LaTeXML/ (o o) | #
11 # \=========================================================ooo==U==ooo=/ #
12 # ================================================================================
13 # LaTeXML::MathParser Math Parser for LaTeXML using Parse::RecDescent.
14 # Parse the intermediate representation generated by the TeX processor.
15 # ================================================================================
16 package LaTeXML::MathParser;
17 use strict;
18 use warnings;
19 use Parse::RecDescent;
20 use LaTeXML::Global;
21 use LaTeXML::Common::Object;
22 use LaTeXML::Common::Error;
23 use LaTeXML::Core::Token;
24 use LaTeXML::Common::Font;
25 use LaTeXML::Common::XML;
26 use List::Util qw(min max);
27 use base (qw(Exporter));
28  
29 our @EXPORT_OK = (qw(&Lookup &New &Absent &Apply &ApplyNary &recApply &CatSymbols
30 &Annotate &InvisibleTimes &InvisibleComma &MorphVertbar
31 &TwoPartRelop &NewFormulae &NewFormula &NewList
32 &ApplyDelimited &NewScript &DecorateOperator &InterpretDelimited &NewEvalAt
33 &LeftRec
34 &Arg &MaybeFunction
35 &SawNotation &IsNotationAllowed
36 &isMatchingClose &Fence));
37 our %EXPORT_TAGS = (constructors
38 => [qw(&Lookup &New &Absent &Apply &ApplyNary &recApply &CatSymbols
39 &Annotate &InvisibleTimes &InvisibleComma &MorphVertbar
40 &TwoPartRelop &NewFormulae &NewFormula &NewList
41 &ApplyDelimited &NewScript &DecorateOperator &InterpretDelimited &NewEvalAt
42 &LeftRec
43 &Arg &MaybeFunction
44 &SawNotation &IsNotationAllowed
45 &isMatchingClose &Fence)]);
46  
47 DebuggableFeature('recdescent', "Trace Parse::RecDescent");
48 our $MATHPARSE_PROGRESS_QUANTUM = 100;
49  
50 # ================================================================================
51 sub new {
52 my ($class, %options) = @_;
53 require LaTeXML::MathGrammar;
54  
55 my $internalparser = LaTeXML::MathGrammar->new();
56 Fatal("expected", "MathGrammar", undef,
57 "Compilation of Math Parser grammar failed") unless $internalparser;
58 my $self = bless { internalparser => $internalparser, lexematize => $options{lexematize} }, $class;
59 return $self; }
60  
61 sub parseMath {
62 my ($self, $document, %options) = @_;
63 local $LaTeXML::MathParser::DOCUMENT = $document;
64 $self->clear; # Not reentrant!
65 $self->cleanupScripts($document);
66 if (my @math = $document->findnodes('descendant-or-self::ltx:XMath[not(ancestor::ltx:XMath)]')) {
67 my $proc = "Math Parsing " . scalar(@math) . " formulae ...";
68 ProgressSpinup($proc);
69 #### SEGFAULT TEST
70 #### $document->doctest("before parse",1);
71 foreach my $math (@math) {
72 $self->parse($math, $document); }
73 ProgressSpindown($proc);
74  
75 NoteLog("Math parsing succeeded:"
76 . join('', map { "\n $_: "
77 . colorizeString($$self{passed}{$_} . "/" . ($$self{passed}{$_} + $$self{failed}{$_}), ($$self{failed}{$_} == 0 ? 'success' : 'warning')) }
78 grep { $$self{passed}{$_} + $$self{failed}{$_} }
79 keys %{ $$self{passed} }));
80 if (my @unk = keys %{ $$self{unknowns} }) {
81 NoteLog("Symbols assumed as simple identifiers (with # of occurences):\n "
82 . join(', ', map { "'" . colorizeString("$_", 'warning') . "' ($$self{unknowns}{$_})" } sort @unk));
83 if (!$STATE->lookupValue('MATHPARSER_SPECULATE')) {
84 NoteLog("Set MATHPARSER_SPECULATE to speculate on possible notations."); } }
85 if (my @funcs = keys %{ $$self{maybe_functions} }) {
86 NoteLog("Possibly used as functions?\n "
87 . join(', ', map { "'$_' ($$self{maybe_functions}{$_}/$$self{unknowns}{$_} usages)" }
88 sort @funcs)); }
89 }
90 #### SEGFAULT TEST
91 #### $document->doctest("IN scope",1);
92 #### SEGFAULT TEST
93 #### $document->doctest("OUT of scope",1);
94 return $document; }
95  
96 # This is a rather peculiar cleanup that needs to be done to manage ids & idrefs
97 # Before parsing, sub/superscripts are represented by an operator-less XMApp with the script
98 # itself as the only child. Ideally, upon parsing these get merged, combined and disappear into
99 # proper XMApp of an appropriate operator on the base and scripts. Unless there is a parse
100 # failure, in which case they remain.
101 # The problem comes from various XMDual constructs where an XMRef refers to the script XMApp.
102 # It can occur that one branch parses and the other fails: This can leave a reference to
103 # the script XMApp which no longer exists!
104 # To solve this, we find & replace all references to such script XMApps by an explicit XMApp
105 # with the XMRef refering to the script itself, not the XMApp. (make sense?)
106 sub cleanupScripts {
107 my ($self, $document) = @_;
108 foreach my $app ($document->findnodes(
109 'descendant-or-self::ltx:XMApp[@xml:id and contains(@role,"SCRIPT")]')) {
110 my $role = $app->getAttribute('role');
111 my $appid = $app->getAttribute('xml:id');
112 if ($role =~ /^(?:PRE|POST|FLOAT)(:?SUB|SUPER)SCRIPT$/) {
113 my @refs = $document->findnodes("descendant-or-self::ltx:XMRef[\@idref = '$appid']");
114 if (scalar(@refs)) {
115 my $script = $app->firstChild;
116 my ($scriptref) = LaTeXML::Package::createXMRefs($document, $script);
117 $document->unRecordID($appid); # no longer refers to the app
118 $app->removeAttribute('xml:id');
119 # Copy all attributes, EXCEPT xml:id
120 my %attr = map { (getQName($_) => $_->getValue) }
121 grep { $_->nodeType == XML_ATTRIBUTE_NODE } $app->attributes;
122 # Now, replace each ref to the script application by an application to a ref to the script.
123 foreach my $ref (@refs) {
124 $document->replaceTree(['ltx:XMApp', {%attr}, $scriptref], $ref); }
125 } } }
126 return; }
127  
128 sub getQName {
129 my ($node) = @_;
130 if (ref $node eq 'ARRAY') {
131 return $$node[0]; }
132 else {
133 return $LaTeXML::MathParser::DOCUMENT->getModel->getNodeQName($node); } }
134  
135 sub realizeXMNode {
136 my ($node) = @_;
137 my $doc = $LaTeXML::MathParser::DOCUMENT;
138 my $idref;
139 if (!defined $node) {
140 return; }
141 elsif (ref $node eq 'ARRAY') {
142 $idref = $$node[1]{idref} if $$node[0] eq 'ltx:XMRef'; }
143 elsif (ref $node eq 'XML::LibXML::Element') {
144 $idref = $node->getAttribute('idref')
145 if $doc->getModel->getNodeQName($node) eq 'ltx:XMRef'; }
146 if ($idref) {
147 # Can it happen that $realnode is, itself, an XMRef? Then we should recurse!
148 if (my $realnode = $doc->lookupID($idref)) {
149 return $realnode; }
150 else {
151 Error("expected", 'id', undef, "Cannot find a node with xml:id='$idref'",
152 ($LaTeXML::MathParser::IDREFS{$idref}
153 ? "Previously bound to " . ToString($LaTeXML::MathParser::IDREFS{$idref})
154 : ()));
155 return ['ltx:ERROR', {}, "Missing XMRef idref=$idref"]; } }
156 else {
157 return $node; } }
158  
159 # ================================================================================
160 sub clear {
161 my ($self) = @_;
162 $$self{passed} = { 'ltx:XMath' => 0, 'ltx:XMArg' => 0, 'ltx:XMWrap' => 0 };
163 $$self{failed} = { 'ltx:XMath' => 0, 'ltx:XMArg' => 0, 'ltx:XMWrap' => 0 };
164 $$self{unknowns} = {};
165 $$self{maybe_functions} = {};
166 $$self{n_parsed} = 0;
167 $$self{progress} = 0;
168 return; }
169  
170 our %EXCLUDED_PRETTYNAME_ATTRIBUTES = (
171 fontsize => 1, opacity => 1,
172 color => 1, backgroundcolor => 1);
173  
174 sub token_prettyname {
175 my ($node) = @_;
176 my $name = $node->getAttribute('name');
177 if (defined $name) { }
178 elsif ($name = $node->textContent) {
179 my $font = $LaTeXML::MathParser::DOCUMENT->getNodeFont($node);
180 my %attr = $font->relativeTo(LaTeXML::Common::Font->textDefault);
181 my $desc = join(' ', map { ToString($attr{$_}{value}) }
182 (grep { !$EXCLUDED_PRETTYNAME_ATTRIBUTES{$_} } (sort keys %attr)));
183 $name .= "{$desc}" if $desc; }
184 else {
185 $name = Stringify($node); } # what else ????
186 return $name; }
187  
188 sub note_unknown {
189 my ($self, $node) = @_;
190 my $name = token_prettyname($node);
191 $$self{unknowns}{$name}++;
192 return; }
193  
194 # debugging utility, should be somewhere handy.
195 sub printNode {
196 my ($node) = @_;
197 if (ref $node eq 'ARRAY') {
198 my ($tag, $attr, @children) = @$node;
199 my @keys = sort keys %$attr;
200 return "<$tag"
201 . (@keys ? ' ' . join(' ', map { "$_='" . ($$attr{$_} || '') . "'" } @keys) : '')
202 . (@children
203 ? ">\n" . join('', map { printNode($_) } @children) . "</$tag>"
204 : '/>')
205 . "\n"; }
206 else {
207 return ToString($node); } }
208  
209 # ================================================================================
210 # Some more XML utilities, but math specific (?)
211  
212 # Get the Token's meaning, else name, else content, else role
213 sub getTokenMeaning {
214 my ($node) = @_;
215 my $x;
216 $node = realizeXMNode($node);
217 return (defined($x = p_getAttribute($node, 'meaning')) ? $x
218 : (defined($x = p_getAttribute($node, 'name')) ? $x
219 : (($x = (ref $node eq 'ARRAY' ? '' : $node->textContent)) ne '' ? $x
220 : (defined($x = p_getAttribute($node, 'role')) ? $x
221 : undef)))); }
222  
223 sub node_location {
224 my ($node) = @_;
225 my $n = $node;
226 while ($n && (ref $n !~ /^XML::LibXML::Document/) # Sometimes DocuementFragment ???
227 && !$n->getAttribute('refnum') && !$n->getAttribute('labels')) {
228 $n = $n->parentNode; }
229 if ($n && (ref $n !~ /^XML::LibXML::Document/)) {
230 my ($r, $l) = ($n->getAttribute('refnum'), $n->getAttribute('labels'));
231 return ($r && $l ? "$r ($l)" : $r || $l); }
232 else {
233 return 'Unknown'; } }
234  
235 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
236 # Parser
237 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
238 # Top-level per-formula parse.
239 # We do a depth-first traversal of the content of the XMath element,
240 # since various sub-elements (XMArg & XMWrap) act as containers of
241 # nominally complete subexpressions.
242 # We do these first for two reasons.
243 # Firstly, since after parsing, the parent will be rebuilt from the result,
244 # we lose the node "identity"; ie. we can't find the child to replace it!
245 # Secondly, in principle (although this isn't used yet), parsing the
246 # child could reveal something interesting about it; say, it's effective role.
247 # Then, this information could be used when parsing the parent.
248 # In fact, this could work the other way too; parsing the parent could tell
249 # us something about what the child must be....
250 sub parse {
251 my ($self, $xnode, $document) = @_;
252 local $LaTeXML::MathParser::STRICT = 1;
253 local $LaTeXML::MathParser::WARNED = 0;
254 local $LaTeXML::MathParser::UNPARSED = 0;
255 local $LaTeXML::MathParser::XNODE = $xnode;
256 local $LaTeXML::MathParser::PUNCTUATION = {};
257 local $LaTeXML::MathParser::LOSTNODES = {};
258 local %LaTeXML::MathParser::IDREFS = ();
259 # This bit for debugging....
260 foreach my $n ($document->findnodes("descendant-or-self::*[\@xml:id]", $xnode)) {
261 my $id = $n->getAttribute('xml:id');
262 $LaTeXML::MathParser::IDREFS{$id} = $n; }
263 if ($$self{lexematize}) {
264 my $lexeme_form = $self->node_to_lexeme_full($xnode);
265 $lexeme_form =~ s/^\s+//;
266 $lexeme_form =~ s/\s+$//;
267 $lexeme_form =~ s/\s+/ /g; # normalize internal whitespaces
268 $xnode->parentNode->setAttribute('lexemes', $lexeme_form);
269 }
270 if (my $result = $self->parse_rec($xnode, 'Anything,', $document)) {
271 # Add text representation to the containing Math element.
272 my $p = $xnode->parentNode;
273 # This is a VERY screwy situation? How can the parent be a document fragment??
274 # This has got to be a LibXML bug???
275 if ($p->nodeType == XML_DOCUMENT_FRAG_NODE) {
276 my @n = $p->childNodes;
277 if (scalar(@n) == 1) {
278 $p = $n[0]; }
279 else {
280 Fatal('malformed', '<XMath>', $xnode, "XMath node has DOCUMENT_FRAGMENT for parent!"); } }
281 # HACK: replace XMRef's to stray trailing punctution
282 foreach my $id (keys %$LaTeXML::MathParser::PUNCTUATION) {
283 my $r = $$LaTeXML::MathParser::PUNCTUATION{$id}->cloneNode;
284 $r->removeAttribute('xml:id');
285 foreach my $n ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']", $p)) {
286 $document->replaceTree($r, $n); } }
287 foreach my $id (keys %$LaTeXML::MathParser::LOSTNODES) {
288 my $repid = $$LaTeXML::MathParser::LOSTNODES{$id};
289 # but the replacement my have been replaced as well!
290 while (my $reprepid = $$LaTeXML::MathParser::LOSTNODES{$repid}) {
291 $repid = $reprepid; }
292 if ($document->findnodes("descendant-or-self::*[\@xml:id='$id']")
293 && !$document->findnodes("descendant-or-self::*[\@xml:id='$repid']")) {
294 # Do nothing if the node never actually got replaced (parse ultimately failed?)
295 }
296 else {
297 foreach my $n ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']", $p)) {
298 $document->setAttribute($n, idref => $repid); } } }
299 $p->setAttribute('text', text_form($result)); }
300 if ($LaTeXML::MathParser::UNPARSED && is_genuinely_unparsed($xnode, $document)) {
301 my $maybe_math = $xnode->parentNode;
302 if ($document->getNodeQName($maybe_math) eq 'ltx:Math') {
303 $maybe_math->setAttribute('class', 'ltx_math_unparsed'); } }
304 return; }
305  
306 my %TAG_FEEDBACK = ('ltx:XMArg' => 'a', 'ltx:XMWrap' => 'w'); # [CONSTANT]
307 # Recursively parse a node with some internal structure
308 # by first parsing any structured children, then it's content.
309 sub parse_rec {
310 my ($self, $node, $rule, $document) = @_;
311 $self->parse_children($node, $document);
312  
313 # This will only handle 1 layer nesting (successfully?)
314 # Note that this would have been found by the top level xpath,
315 # but we've got to worry about node identity: the parent is being rebuilt
316 foreach my $nested ($document->findnodes('descendant::ltx:XMath', $node)) {
317 $self->parse($nested, $document); }
318 my $tag = getQName($node);
319 if (my $requested_rule = $node->getAttribute('rule')) {
320 $rule = $requested_rule; }
321 if ($rule eq 'kludge') {
322 $self->parse_kludge($node, $document);
323 return; }
324 elsif (my $result = $self->parse_single($node, $document, $rule)) {
325 $$self{passed}{$tag}++;
326 if ($tag eq 'ltx:XMath') { # Replace the content of XMath with parsed result
327 ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0;
328 map { $document->unRecordNodeIDs($_) } element_nodes($node);
329 # unbindNode followed by (append|replace)Tree (which removes ID's) should be safe
330 map { $_->unbindNode() } $node->childNodes;
331 $document->appendTree($node, $result);
332 $result = [element_nodes($node)]->[0]; }
333 else { # Replace the whole node for XMArg, XMWrap; preserve some attributes
334 ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0;
335 # Copy all attributes
336 my $resultid = p_getAttribute($result, 'xml:id');
337 my %attr = map { (getQName($_) => $_->getValue) }
338 grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes;
339 # add to result, even allowing modification of xml node, since we're committed.
340 # [Annotate converts node to array which messes up clearing the id!]
341 my $isarr = ref $result eq 'ARRAY';
342 my $rtag = ($isarr ? $$result[0] : $document->getNodeQName($result));
343 # Make sure font is "Appropriate", if we're creating a new token (yuck)
344 if ($isarr && $attr{_font} && ($rtag eq 'ltx:XMTok')) {
345 my $content = join('', @$result[2 .. $#$result]);
346 if ((!defined $content) || ($content eq '')) {
347 delete $attr{_font}; } # No font needed
348 elsif (my $font = $document->decodeFont($attr{_font})) {
349 delete $attr{_font};
350 $attr{font} = $font->specialize($content); } }
351 else {
352 delete $attr{_font}; }
353 foreach my $key (keys %attr) {
354 next unless ($key =~ /^_/) || $document->canHaveAttribute($rtag, $key);
355 my $value = $attr{$key};
356 if ($key eq 'xml:id') { # Since we're moving the id...bookkeeping
357 $document->unRecordID($value);
358 $node->removeAttribute('xml:id'); }
359 if ($isarr) { $$result[1]{$key} = $value; }
360 else { $document->setAttribute($result, $key => $value); } }
361 $result = $document->replaceTree($result, $node);
362 my $newid = $attr{'xml:id'};
363 # Danger: the above code replaced the id on the parsed result with the one from XMArg,..
364 # If there are any references to $resultid, we need to point them to $newid!
365 if ($resultid && $newid && ($resultid ne $newid)) {
366 foreach my $ref ($document->findnodes("//*[\@idref='$resultid']")) {
367 $ref->setAttribute(idref => $newid); } }
368 }
369 return $result; }
370 else {
371 $self->parse_kludge($node, $document);
372 ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0;
373 $$self{failed}{$tag}++;
374 return; } }
375  
376 # Depth first parsing of XMArg nodes.
377 sub parse_children {
378 my ($self, $node, $document) = @_;
379 foreach my $child (element_nodes($node)) {
380 my $tag = getQName($child);
381 if ($tag eq 'ltx:XMArg') {
382 $self->parse_rec($child, 'Anything', $document); }
383 elsif ($tag eq 'ltx:XMWrap') {
384 local $LaTeXML::MathParser::STRICT = 0;
385 $self->parse_rec($child, 'Anything', $document); }
386 ### A nice evolution would be to use the Kludge parser for
387 ### the presentation form in XMDual
388 ### This would avoid silly "parses" of non-semantic stuff; eg assuming times between tokens!
389 ### However, it needs some experimentation to match DLMF's enhancements
390 #### $self->parse_children($child,$document);
391 #### $self->parse_kludge($child,$document); }
392 elsif ($tag =~ /^ltx:(XMApp|XMArray|XMRow|XMCell)$/) {
393 $self->parse_children($child, $document); }
394 elsif ($tag eq 'ltx:XMDual') {
395 $self->parse_children($child, $document); }
396 ### Some day this may be needed? but not yet...
397 # elsif ($tag eq 'ltx:XMRef') {
398 # # We might be referencing something outside the current tree.
399 # # but we'd like it's contents to be parsed before we start using it at this level.
400 # # NOTE: Do we have to worry about things being parsed twice? (ie. when the other tree's done)
401 # # if so, we probably want to mark things as having already been parsed.
402 # my $refnode = realizeXMNode($child);
403 # if(getQName($refnode) ne 'ltx:XMTok'){ # Anything referenced with structure...
404 # Debug("PRE-PARSING Ref'd node: ".ToString($child)." == ".ToString($refnode));
405 # ## $self->parse_rec($refnode,'Anything',$document);
406 # $self->parse_children($refnode,$document);
407 # Debug("NOW REFERENCES: ".ToString(realizeXMNode($child)));
408 # } }
409 }
410 return; }
411  
412 my $HINT_PUNCT_THRESHOLD = 10.0; # \quad or bigger becomes punctuation ? [CONSTANT]
413  
414 sub filter_hints {
415 my ($self, $document, @nodes) = @_;
416 my @prefiltered = ();
417 my $prev = undef;
418 my $pending_comments = '';
419 my $pending_space = 0.0;
420 my $pending_phantom = undef;
421 # Filter the nodes, watching for XMHint's and Comments.
422 foreach my $node (@nodes) {
423 my $type = $node->nodeType;
424 if ($type == XML_TEXT_NODE) { # text? better be (ignorable) whitespace!
425 my $string = $node->textContent;
426 if ($string =~ /\S/) {
427 Warn('unexpected', 'text', $node, "Unexpected text in Math tree"); } }
428 elsif ($type == XML_COMMENT_NODE) { # Comment!
429 my $comment = $node->getData;
430 if ($prev) { # Append to previous element's comments
431 my $c = $prev->getAttribute('_comment');
432 $prev->setAttribute(_comment => ($c ? $c . "\n" . $comment : $comment)); }
433 else { # Or save for first
434 $pending_comments = ($pending_comments ? $pending_comments . "\n" . $comment : $comment); } }
435 elsif ($type != XML_ELEMENT_NODE) {
436 Warn('unexpected', 'node', $node, "Unexpected item in Math tree"); }
437 elsif (getQName($node) eq 'ltx:XMHint') { # If a Hint node?
438 if (my $width = $node->getAttribute('width')) {
439 if (my $pts = getXMHintSpacing($width)) {
440 my $ph = ($node->getAttribute('name') || '') =~ /phantom$/;
441 # If previous token (and not open paren), add the space to the right of it
442 if ($prev && (($prev->getAttribute('role') || '') ne 'OPEN')) {
443 my $s = $prev->getAttribute('_space') || 0.0;
444 my $p = $prev->getAttribute('_phantom');
445 $prev->setAttribute(_space => $s + $pts);
446 $prev->setAttribute(_phantom => $p || $ph); }
447 else { # Else save it for the next token.
448 $pending_space += $pts;
449 $pending_phantom = $ph; } } }
450 # If XMHint is referenced??
451 # Really shouldn't happen, but if XMHint is in XMWrap, it may end up with id....?
452 if (my $id = $node->getAttribute('xml:id')) {
453 if ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']")) {
454 push(@prefiltered, $node); $prev = $node; } } }
455 # Don't store stuff on APPLYOP, since they tend to disappear.
456 elsif ((getQName($node) eq 'ltx:XMTok') && (($node->getAttribute('role') || '') eq 'APPLYOP')) {
457 ### push(@prefiltered, $node); $prev = $node; } # Keep it.
458 push(@prefiltered, $node); } # Keep it, but don't step $prev
459 else { # Some other element.
460 if ($pending_comments) {
461 $node->setAttribute(_pre_comment => $pending_comments);
462 $pending_comments = ''; }
463 if ($pending_space) {
464 $node->setAttribute(lpadding =>
465 LaTeXML::Common::Dimension::attributeformat($pending_space * 65536));
466 $node->setAttribute(_phantom => $pending_phantom);
467 $pending_space = 0.0;
468 $pending_phantom = undef; }
469 push(@prefiltered, $node); $prev = $node; } # Keep it.
470 }
471 # if $pending_space > 0, should store as rpadding on last element?
472 my @filtered = ();
473 # Filter through the pre-filtered nodes looking for large rpadding to convert to virtual PUNCT.
474 # But we'll assume that phantoms do NOT get converted to PUNCT ?
475 foreach my $node (@prefiltered) {
476 push(@filtered, $node);
477 if (my $s = $node->getAttribute('_space')) {
478 $node->removeAttribute('_space');
479 if ((!$node->getAttribute('_phantom'))
480 && ($s >= $HINT_PUNCT_THRESHOLD) && (($node->getAttribute('role') || '') ne 'PUNCT')) {
481 # Treat the XMHint as Punctuation (role=PUNCT)
482 # I'm leary that this is a safe way to create an XML node that's NOT in the tree, but...
483 my $p = $node->parentNode;
484 my $punct = $document->openElementAt($p, 'ltx:XMHint', role => 'PUNCT', width => $s . 'pt',
485 name => ('q' x int($s / 10)) . 'uad');
486 $p->removeChild($punct); # But don't actually leave it in the tree!!!!
487 push(@filtered, $punct); }
488 else {
489 $node->setAttribute(rpadding =>
490 LaTeXML::Common::Dimension::attributeformat($s * 65536)); } } }
491 return @filtered; }
492  
493 # Given a width attribute on an XMHint, return the pts, if any
494 sub getXMHintSpacing {
495 my ($width) = @_;
496 if ($width && ($width =~ /^$LaTeXML::Common::Glue::GLUE_re$/)) {
497 return ($2 eq 'mu' ? $1 / 1.8 : $1); }
498 else {
499 return 0; } }
500  
501 # We've pretty much builtin the assumption that the target XML is "As If" 10 pts,
502 # so we'll assume that 1em is 10 pts.
503 my $POINTS_PER_EM = 10.0; # [CONSTANT]
504 # Convert spacing, given as a number of points, to a string of appropriate spacing chars
505 sub spacingToString {
506 my ($points) = @_;
507 my $spacing = '';
508 my $ems = $points / $POINTS_PER_EM;
509 my $n = int($ems);
510 if ($n > 0) { $spacing .= ("\x{2003}" x $n); $ems -= $n; }
511 if ($ems > 0.500) { $spacing .= "\x{2002}"; $ems -= 0.500; }
512 if ($ems > 0.333) { $spacing .= "\x{2003}"; $ems -= 0.333; }
513 if ($ems > 0.250) { $spacing .= "\x{2005}"; $ems -= 0.250; }
514 if ($ems > 0.166) { $spacing .= "\x{2006}"; $ems -= 0.166; }
515 return $spacing; }
516  
517 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
518 # Low-Level hack parsing when "real" parsing fails;
519 # Two issues cause generated Presentation MathML to be really bad:
520 # (1) not having mrow/mfenced structures wrapping OPEN...CLOSE sequences
521 # throws off MathML's stretchiness treatment of the fences
522 # (they're all the same size; big)
523 # (2) un-attached sub/superscripts won't position correctly,
524 # unless they're attached to something plausible.
525 # NOTE: we should be able to optionally switch this off.
526 # Especially, when we want to try alternative parse strategies.
527 sub parse_kludge {
528 my ($self, $mathnode, $document) = @_;
529 my @nodes = $self->filter_hints($document, $mathnode->childNodes);
530 # the 1st array in stack accumlates the nodes within the current fenced row.
531 # When there's only a single array, it's single entry will be the complete row.
532 my @stack = ([], []);
533 my @pairs = map { [$_, $self->getGrammaticalRole($_)] } @nodes;
534 while (scalar(@pairs) || (scalar(@stack) > 1)) {
535 my $pair = shift(@pairs);
536 my $role = ($pair ? $$pair[1] : 'CLOSE');
537 if ($role eq 'OPEN') {
538 unshift(@stack, [$pair]); } # Start new fenced row;
539 elsif ($role eq 'CLOSE') { # Close the current row
540 my $row = shift(@stack); # get the current list of items
541 push(@$row, $pair) if $pair; # Put the close (if any) into it
542  
543 my @kludged = $self->parse_kludgeScripts_rec(@$row); # handle scripts
544 # wrap, if needed.
545 $row = [(scalar(@kludged) > 1 ? ['ltx:XMWrap', {}, @kludged] : $kludged[0]), 'FENCED'];
546 push(@{ $stack[0] }, $row); } # and put this constructed row at end of containing row.
547 else {
548 push(@{ $stack[0] }, $pair); } } # Otherwise, just put this item into current row.
549  
550 # If we got to here, remove the nodes and replace them by the kludged structure.
551 map { $document->unRecordNodeIDs($_) } element_nodes($mathnode);
552 # unbindNode followed by (append|replace)Tree (which removes ID's) should be safe
553 map { $_->unbindNode() } $mathnode->childNodes;
554 # We're hoping for a single list on the stack,
555 # But extra CLOSEs will leave extra junk behind, so process all the stacked lists.
556 my @replacements = ();
557 foreach my $pair (@{ $stack[0] }) {
558 my $kludge = $$pair[0];
559 push(@replacements, (ref $kludge eq 'ARRAY') && ($$kludge[0] eq 'ltx:XMWrap')
560 ? @$kludge[2 .. $#$kludge] : ($kludge)); }
561 $document->appendTree($mathnode, @replacements);
562 p_setAttribute($mathnode, '_unparsed', '1'); # But mark as unparsed
563 return; }
564  
565 sub parse_kludgeScripts_rec {
566 no warnings 'recursion';
567 my ($self, $x, $y, @more) = @_;
568 my @acc = ();
569 while (defined $y) {
570 if ($$x[1] =~ /^FLOAT(SUB|SUPER)SCRIPT$/) { # a Floating script? maybe pre-script
571 if ($$y[1] =~ /^POST(SUB|SUPER)SCRIPT$/) { # followed by another script?
572 if (@more) { # followed by a something? => Combined pre sub & super
573 my ($base, @rest) = $self->parse_kludgeScripts_rec(@more);
574 return (@acc, NewScript(NewScript($base, $$y[0], 'pre'), $$x[0], 'pre'), @rest); }
575 else { # else floating sub & super
576 return (@acc, NewScript(NewScript(Absent(), $$y[0], 'post'), $$x[0], 'post')); } }
577 else { # else just prescript on whatever follows
578 my ($base, @rest) = $self->parse_kludgeScripts_rec($y, @more);
579 return (@acc, NewScript($base, $$x[0], 'pre'), @rest); } }
580 elsif ($$y[1] =~ /^POST(SUB|SUPER)SCRIPT$/) {
581 # or a postscript is applied to the preceding thing.
582 return (@acc, $self->parse_kludgeScripts_rec([NewScript($$x[0], $$y[0]), ''], @more)); }
583 else { # else skip over a and continue
584 push(@acc, $$x[0]);
585 ($x, $y, @more) = ($y, @more); } }
586 return (@acc, $$x[0]); }
587  
588 # OBSOLETE EXPLORATORY CODE; but there are some interesting ideas....
589 # sub parse_kludge {
590 # my($self,$mathnode,$document)=@_;
591 # my @nodes = $self->filter_hints($document,$mathnode->childNodes);
592 # map { $mathnode->removeChild($_) } @nodes;
593 # my @result=();
594 # while(@nodes){
595 # @nodes = $self->parse_kludge_rec(@nodes);
596 # push(@result,shift(@nodes)); }
597 # $document->appendTree($mathnode,@result); }
598  
599 # # Kludge Parse the next thing, and then add any following scripts to it.
600 # sub parse_kludge_rec {
601 # my($self,@more)=@_;
602 # my($item,$open,$close,$seps,$arg);
603 # ($item,@more) = $self->parse_kludge_reca(@more);
604 # while(@more && (($self->getGrammaticalRole($more[0])||'') =~ /^POST(SUB|SUPER)SCRIPT$/)){
605 # $item = NewScript($item,shift(@more)); }
606 # if(@more && (($self->getGrammaticalRole($more[0])||'') eq 'APPLYOP')){
607 # shift(@more);
608 # if(@more && (($self->getGrammaticalRole($more[0])||'') eq 'OPEN')){
609 # ($open,$close,$seps,$arg,@more)=$self->parse_kludge_fence(@more);
610 # $item = Apply(Annotate($item,argopen=>$open, argclose=>$close, separators=>$seps),@$arg); }
611 # else {
612 # ($arg,@more)=$self->parse_kludge_rec(@more);
613 # $item = Apply($item,$arg); }}
614 # ($item,@more); }
615  
616 # sub parse_kludge_reca {
617 # my($self,$next,@more)=@_;
618 # my $role = $self->getGrammaticalRole($next);
619 # if($role =~ /^FLOAT(SUB|SUPER)SCRIPT$/){
620 # my($base,@rest) = $self->parse_kludge_rec(@more);
621 # (NewScript($base,$next),@rest); }
622 # elsif($role eq 'OPEN'){
623 # my($open,$close,$seps,$list,@more)=$self->parse_kludge_fence($next,@more);
624 # (Apply(Annotate(New(undef,undef,role=>'FENCED'),
625 # argopen=>$open, argclose=>$close, separators=>$seps),@$list), @more); }
626 # else {
627 # ($next,@more); }}
628  
629 # sub parse_kludge_fence {
630 # my($self,$next,@more)=@_;
631 # my($open,$close,$punct,$r,$item,@list)=($next,undef,'',undef);
632 # while(@more){
633 # my @i=();
634 # while(@more && (($r=($self->getGrammaticalRole($more[0])||'')) !~ /^(CLOSE|PUNCT)$/)){
635 # ($item,@more)=$self->parse_kludge_rec(@more);
636 # push(@i,$item); }
637 # push(@list,(scalar(@i > 1) ? ['ltx:XMWrap',{},@i] : $i[0]));
638 # if($r eq 'CLOSE'){
639 # $close=shift(@more); last; }
640 # else {
641 # $punct .= ($punct ? ' ':''). p_getValue(shift(@more)); }} # Delimited by SINGLE SPACE!
642 # ($open,$close,$punct,[@list],@more); }
643  
644 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
645 # Low-level Parser: parse a single expression
646 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
647 # Convert to textual form for processing by MathGrammar
648 sub parse_single {
649 my ($self, $mathnode, $document, $rule) = @_;
650 my @nodes = $self->filter_hints($document, $mathnode->childNodes);
651  
652 my ($punct, $result, $unparsed);
653 my @punct = ();
654 # Extract trailing punctuation, if rule allows it.
655 if ($rule =~ s/,$//) {
656 # Collect ALL trailing PUNCT|PERIOD's...
657 my ($x, $r);
658 while (($x = $nodes[-1]) && ($x = realizeXMNode($x))
659 && ($r = $x->getAttribute('role')) && (($r eq 'PUNCT') || ($r eq 'PERIOD'))) {
660 my $p = pop(@nodes);
661 # Special case hackery, in case this thing is XMRef'd!!!
662 # We could just stick it on the end of the presentation,
663 # but it doesn't belong in the content at all!?!?
664 if (my $id = $p->getAttribute('xml:id')) {
665 $$LaTeXML::MathParser::PUNCTUATION{$id} = $p; }
666 unshift(@punct, $p); }
667 }
668  
669 if ($LaTeXML::DEBUG{recdescent}) {
670 $::RD_TRACE = 1; # Turn on MathGrammar tracing
671 # my $box = $document->getNodeBox($LaTeXML::MathParser::XNODE);
672 my $box = $document->getNodeBox($mathnode);
673 Debug(('=' x 60) .
674 "\nParsing formula \"" . ToString($box) . "\""
675 . "\n from " . ToString($box->getLocator)
676 . "\n == \"" . join(' ', map { node_string($_, $document) } @nodes) . "\""
677 . "\n == " . ToString($mathnode)); }
678  
679 if (scalar(@nodes) < 2) { # Too few nodes? What's to parse?
680 $result = $nodes[0] || Absent(); }
681 else {
682 # Now do the actual parse.
683 ($result, $unparsed) = $self->parse_internal($rule, @nodes);
684 }
685 # Failure? No result or uparsed lexemes remain.
686 # NOTE: Should do script hack??
687 if ((!defined $result) || $unparsed) {
688 $LaTeXML::MathParser::UNPARSED = 1;
689 $self->failureReport($document, $mathnode, $rule, $unparsed, @nodes);
690 return; }
691 # Success!
692 else {
693 if (@punct) { # create a trivial XMDual to treat the punctuation as presentation
694 $result = ['ltx:XMDual', {},
695 LaTeXML::Package::createXMRefs($document, $result),
696 ['ltx:XMWrap', {}, $result, @punct]]; } # or perhaps: Apply, punctuated???
697 if ($LaTeXML::DEBUG{recdescent}) {
698 Debug("=>" . printNode($result) . "\n" . ('=' x 60)); }
699 return $result; } }
700  
701 sub node_to_lexeme {
702 my ($self, $node) = @_;
703 my $qname = getQName($node);
704 my $is_inked = 1;
705 my $lexeme = ($qname eq 'ltx:XMTok' or $qname eq 'ltx:text') && $node->textContent;
706 if (!defined($lexeme) || length($lexeme) == 0) {
707 $is_inked = 0;
708 $lexeme = getTokenMeaning($node); }
709 my $document = $LaTeXML::MathParser::DOCUMENT;
710 return unless defined $lexeme;
711 if ($is_inked) {
712 if (my $font = $node->getAttribute('_font')) {
713 my $font_spec = $document->decodeFont($font);
714 if (my %declarations = $font_spec && $font_spec->relativeTo(LaTeXML::Common::Font->textDefault)) {
715 my %to_add = ();
716 my $font_pending = $declarations{font} || {};
717 my $properties_pending = $$font_pending{properties} || {};
718 foreach my $attr (qw(family series shape)) {
719 if (my $value = $$properties_pending{$attr}) {
720 $to_add{$value} = 1; } }
721 if (%to_add) {
722 my $prefix = join("_", sort(keys(%to_add))) . "_";
723 $lexeme = join(" ", map { $prefix . $_ } split(' ', $lexeme)); }
724 } }
725 if ($lexeme =~ /^\p{L}+$/) {
726 $lexeme = join(" ", map { 'roman_' . $_ } split(' ', $lexeme)); } }
727 $lexeme =~ s/\s+$//;
728 return $lexeme; }
729  
730 sub node_to_lexeme_full {
731 my ($self, $unrealized_node) = @_;
732 my $node = realizeXMNode($unrealized_node);
733 my $tag = getQName($node);
734 if ($tag eq 'ltx:XMHint') { return ""; } # just skip XMHints, they don't contain lexemes
735 my $role = p_getAttribute($node, 'role');
736 if ($tag eq 'ltx:XMTok' || ($role && ($tag !~ 'ltx:XM(Dual|App|Arg|Array|Wrap|ath)'))) {
737 # Elements that directly represent a lexeme, or intended operation with a syntactic role (such as a postscript),
738 # can proceed to building the lexeme from the leaf node.
739 my $lexeme = $self->node_to_lexeme($node);
740 if ($role && $role =~ /^(UNDER|OVER)ACCENT$/) {
741 # over¯ and under¯ are the lexeme names of choice for \overline and \underline
742 $lexeme = lc($1) . $lexeme; }
743 return $lexeme; }
744 # Elements that do not have a role and are intermediate "may" need an argument wrapper, so that arguments
745 # remain unambiguous. For instance a `\frac{a}{b}` has clear argument structure to be preserved.
746 my ($mark_start, $mark_end) = ('', '');
747 if ($tag ne 'ltx:XMath') {
748 if ($role) {
749 $mark_start = "start_$role ";
750 $mark_end = " end_$role";
751 } elsif ($tag =~ '^ltx:XM(Arg|Row|Cell)') {
752 my $tag_role = uc($1);
753 $mark_start = "start_$tag_role ";
754 $mark_end = " end_$tag_role";
755 }
756 }
757 my $lexemes = $mark_start;
758 if ($tag eq 'ltx:XMDual') {
759 $lexemes .= $self->node_to_lexeme_full($LaTeXML::MathParser::DOCUMENT->getSecondChildElement($node)); }
760 elsif ($tag eq 'ltx:XMText') {
761 # can be either a single node XMText, we're looking at a leaf text node
762 # or \text{}-like construct, with multiple math formulas and interleaved text
763 foreach my $child ($node->childNodes) {
764 if (ref($child) eq 'XML::LibXML::Text') {
765 $lexemes .= $child->textContent() . ' '; }
766 else {
767 my $child_lexeme = $self->node_to_lexeme_full($child);
768 if (defined $child_lexeme && length($child_lexeme) > 0) {
769 $lexemes .= $child_lexeme . ' '; } } } }
770 elsif ($tag eq 'ltx:text') { # could recurse in from ltx:XMText
771 return $self->node_to_lexeme($node); }
772 else {
773 my @child_elements = element_nodes($node);
774 # skip through single child wrappers (don't serialize)
775 while (scalar(@child_elements) == 1 && getQName($child_elements[0]) =~ /^ltx:XM(Arg|Wrap)$/) {
776 @child_elements = element_nodes($child_elements[0]);
777 }
778 foreach my $child (@child_elements) {
779 my $child_lexeme = $self->node_to_lexeme_full($child);
780 if (defined $child_lexeme && length($child_lexeme) > 0) {
781 $lexemes .= $child_lexeme . ' ';
782 } } }
783 $lexemes .= $mark_end;
784 return $lexemes; }
785  
786 sub parse_internal {
787 my ($self, $rule, @nodes) = @_;
788 #------------
789 # Generate a textual token for each node; The parser operates on this encoded string.
790 local $LaTeXML::MathParser::LEXEMES = {};
791 my $i = 0;
792 my $lexemes = '';
793  
794 foreach my $node (@nodes) {
795 # This is a parser-specific lexeme, but it is not (yet) identical to the serialized lexeme by
796 # ->node_to_lexeme, which is currently experimental
797 my $role = $self->getGrammaticalRole($node);
798 my $text = getTokenMeaning($node);
799 $text = 'Unknown' unless defined $text;
800 my $lexeme = $role . ":" . $text . ":" . ++$i;
801 $lexeme =~ s/\s//g;
802  
803 $$LaTeXML::MathParser::LEXEMES{$lexeme} = $node;
804 $lexemes .= ' ' . $lexeme; }
805  
806 #------------
807 # apply the parser to the textified sequence.
808 local $LaTeXML::MathParser::PARSER = $self;
809 local %LaTeXML::MathParser::SEEN_NOTATIONS = ();
810 local %LaTeXML::MathParser::DISALLOWED_NOTATIONS = ();
811 local $LaTeXML::MathParser::MAX_ABS_DEPTH = 1;
812 # "Speculative" mode is disabled by default (set MATHPARSER_SPECULATE).
813 # It causes the parser to explore possible undeclared notatations,
814 # like possible functions (the only one, so far)
815 # These extra grammar rules can be costly in corner cases.
816 # This should be disabled when speed (but feedback) is essential
817 if ($STATE->lookupValue('MATHPARSER_SPECULATE')) {
818 $LaTeXML::MathParser::DISALLOWED_NOTATIONS{MaybeFunctions} = 0; }
819 else {
820 $LaTeXML::MathParser::DISALLOWED_NOTATIONS{MaybeFunctions} = 1; }
821 my $unparsed = $lexemes;
822 my $result = $$self{internalparser}->$rule(\$unparsed);
823 if (((!defined $result) || $unparsed) # If parsing Failed
824 && $LaTeXML::MathParser::SEEN_NOTATIONS{QM}) { # & Saw some QM stuff.
825 $LaTeXML::MathParser::DISALLOWED_NOTATIONS{QM} = 1; # Retry w/o QM notations
826 $unparsed = $lexemes;
827 $result = $$self{internalparser}->$rule(\$unparsed); }
828 while (((!defined $result) || $unparsed) # If parsing Failed
829 && ($LaTeXML::MathParser::SEEN_NOTATIONS{AbsFail}) # & Attempted deeper abs nesting?
830 && ($LaTeXML::MathParser::MAX_ABS_DEPTH < 3)) { # & Not ridiculously deep
831 delete $LaTeXML::MathParser::SEEN_NOTATIONS{AbsFail};
832 ++$LaTeXML::MathParser::MAX_ABS_DEPTH; # Try deeper.
833 $unparsed = $lexemes;
834 $result = $$self{internalparser}->$rule(\$unparsed); }
835  
836 # If still failed, try other strategies?
837  
838 return ($result, $unparsed); }
839  
840 sub getGrammaticalRole {
841 my ($self, $node) = @_;
842 $node = realizeXMNode($node);
843 my $role = p_getAttribute($node, 'role');
844 if (!defined $role) {
845 my $tag = getQName($node);
846 if ($tag eq 'ltx:XMTok') {
847 $role = 'UNKNOWN'; }
848 elsif ($tag eq 'ltx:XMDual') {
849 my ($content, $presentation) = element_nodes($node);
850 $role = p_getAttribute($content, 'role') || p_getAttribute($presentation, 'role'); }
851 $role = 'ATOM' unless defined $role; }
852 $self->note_unknown($node) if ($role eq 'UNKNOWN') && $LaTeXML::MathParser::STRICT;
853 return $role; }
854  
855 # How many tokens before & after the failure point to report in the Warning message.
856 my $FAILURE_PRETOKENS = 3; # [CONSTANT]
857 my $FAILURE_POSTTOKENS = 1; # [CONSTANT]
858  
859 sub failureReport {
860 my ($self, $document, $mathnode, $rule, $unparsed, @nodes) = @_;
861 if ($LaTeXML::MathParser::STRICT || (($LaTeXML::Common::Error::VERBOSITY || 0) > 1)) {
862 my $loc = "";
863 # If we haven't already done it for this formula, show the original TeX.
864 if (!$LaTeXML::MathParser::WARNED) {
865 $LaTeXML::MathParser::WARNED = 1;
866 my $box = $document->getNodeBox($LaTeXML::MathParser::XNODE);
867 $loc = "In \"" . UnTeX($box) . "\""; }
868 $unparsed =~ s/^\s*//;
869 my @rest = split(/ /, $unparsed);
870 my $pos = scalar(@nodes) - scalar(@rest);
871 # Break up the input at the point where the parse failed.
872 my $max = 50;
873 my $parsed = join(' ', ($pos > $max ? ('...') : ()),
874 (map { node_string($_, $document) } @nodes[max(0, $pos - 50) .. $pos - 1]));
875 my $toparse = join(' ',
876 (map { node_string($_, $document) } @nodes[$pos .. min($pos + 50, $#nodes)]),
877 ($#nodes > $pos + 50 ? ('...') : ()));
878 my $parsefail
879 = join('.', map { $self->getGrammaticalRole($_) }
880 @nodes[($pos - $FAILURE_PRETOKENS >= 0
881 ? $pos - $FAILURE_PRETOKENS : 0) .. $pos - 1])
882 . ">"
883 . join('.', map { $self->getGrammaticalRole($_) }
884 @nodes[$pos .. ($pos + $FAILURE_POSTTOKENS - 1 < $#nodes
885 ? $pos + $FAILURE_POSTTOKENS - 1 : $#nodes)]);
886 my $lexeme = node_location($nodes[$pos] || $nodes[$pos - 1] || $mathnode);
887 my $indent = length($parsed) - 2; $indent = 8 if $indent > 8;
888 Warn('not_parsed', $parsefail, $mathnode,
889 "MathParser failed to match rule '$rule'",
890 ($loc ? ($loc) : ()),
891 ($parsed
892 ? ($parsed, (' ' x $indent) . "> " . $toparse)
893 : ("> " . $toparse)));
894 }
895 return; }
896  
897 # used for debugging & failure reporting.
898 sub node_string {
899 my ($node, $document) = @_;
900 my $role = $node->getAttribute('role') || 'UNKNOWN';
901 my $box = $document->getNodeBox($node);
902 return ($box ? ToString($box) : text_form($node)) . "[[$role]]"; }
903  
904 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
905 # Conversion to a less ambiguous, mostly-prefix form.
906 # Mostly for debugging information?
907 # Note that the nodes are true libXML nodes, already absorbed into the document
908 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
909 sub text_form {
910 my ($node) = @_;
911 return textrec($node, undef); }
912  
913 my %PREFIX_ALIAS = ( # [CONSTANT]
914 SUPERSCRIPTOP => '^', SUBSCRIPTOP => '_', times => '*',
915 'equals' => '=', 'less-than' => '<', 'greater-than' => '>',
916 'less-than-or-equals' => '<=', 'greater-than-or-equals' => '>=',
917 'much-less-than' => '<<', 'much-greater-than' => '>>',
918 'plus' => '+', 'minus' => '-', 'divide' => '/');
919 # Put infix, along with `binding power'
920 my %IS_INFIX = (METARELOP => 1, # [CONSTANT]
921 RELOP => 2, ARROW => 2,
922 ADDOP => 10, MULOP => 100, FRACOP => 100,
923 SUPERSCRIPTOP => 1000, SUBSCRIPTOP => 1000);
924  
925 sub textrec {
926 no warnings 'recursion';
927 my ($node, $outer_bp, $outer_name) = @_;
928 return '[missing]' unless defined $node;
929 $node = realizeXMNode($node);
930 my $tag = getQName($node);
931 $outer_bp = 0 unless defined $outer_bp;
932 $outer_name = '' unless defined $outer_name;
933 # If node has meaning, that's the text form.
934 if (my $meaning = p_getAttribute($node, 'meaning') || p_getAttribute($node, 'name')) {
935 return $PREFIX_ALIAS{$meaning} || $meaning; }
936 elsif ($tag eq 'ltx:XMApp') {
937 my $app_role = $node->getAttribute('role');
938 my ($op, @args) = element_nodes($node);
939 $op = realizeXMNode($op);
940 if ($app_role && $app_role =~ /^FLOAT(SUB|SUPER)SCRIPT$/) {
941 return ($1 eq 'SUPER' ? '^' : '_') . textrec($op); }
942 else {
943 my $name = ((getQName($op) eq 'ltx:XMTok') && getTokenMeaning($op)) || 'unknown';
944 my ($bp, $string) = textrec_apply($name, $op, @args);
945 return (($bp < $outer_bp) || (($bp == $outer_bp) && ($name ne $outer_name))
946 ? '(' . $string . ')' : $string); } }
947 elsif ($tag eq 'ltx:XMDual') {
948 my ($content, $presentation) = element_nodes($node);
949 my $text = textrec($content, $outer_bp, $outer_name); # Just send out the semantic form.
950 # Fall back to presentation, if content has poor semantics (eg. from replacement patterns)
951 return ($text =~ /^\(*Unknown/ ? textrec($presentation, $outer_bp, $outer_name) : $text); }
952 elsif ($tag eq 'ltx:XMTok') {
953 my $name = getTokenMeaning($node);
954 $name = 'Unknown' unless defined $name;
955 return $PREFIX_ALIAS{$name} || $name; }
956 elsif (($tag eq 'ltx:XMWrap') || ($tag eq 'ltx:XMCell')) {
957 # ??
958 return join('@', map { textrec($_) } element_nodes($node)); }
959 elsif ($tag eq 'ltx:XMArray') {
960 return textrec_array($node); }
961 else {
962 return '[' . (p_getValue($node) || '') . ']'; } }
963  
964 sub textrec_apply {
965 no warnings 'recursion';
966 my ($name, $op, @args) = @_;
967 my $role = ((ref $op ne 'ARRAY') && $op->getAttribute('role')) || 'Unknown';
968 if (($role =~ /^(SUB|SUPER)SCRIPTOP$/) && (($op->getAttribute('scriptpos') || '') =~ /^pre\d+$/)) {
969 # Note that this will likely get parenthesized due to high bp
970 return (5000, textrec($op) . " " . textrec($args[1]) . " " . textrec($args[0])); }
971 elsif (my $bp = $IS_INFIX{$role}) {
972 # A sub/superscript with a meaning probably should be prefix
973 if (($role =~ /^(SUB|SUPER)SCRIPTOP$/) && $op->getAttribute('meaning')) {
974 return (500, textrec($op, 10000, $name) . '@(' . join(', ', map { textrec($_) } @args) . ')') }
975 else { # Format as infix.
976 return ($bp, (scalar(@args) == 1 # unless a single arg; then prefix.
977 ? textrec($op) . ' ' . textrec($args[0], $bp, $name)
978 : join(' ' . textrec($op) . ' ', map { textrec($_, $bp, $name) } @args))); } }
979 elsif ($role eq 'POSTFIX') {
980 return (10000, textrec($args[0], 10000, $name) . textrec($op)); }
981 elsif ($name eq 'multirelation') {
982 return (2, join(' ', map { textrec($_, 2, $name) } @args)); }
983 else {
984 return (500, textrec($op, 10000, $name) . '@(' . join(', ', map { textrec($_) } @args) . ')'); } }
985  
986 sub textrec_array {
987 my ($node) = @_;
988 my $name = $node->getAttribute('meaning') || $node->getAttribute('name') || 'Array';
989 my @rows = ();
990 foreach my $row (element_nodes($node)) {
991 push(@rows, '[' . join(', ', map { ($_->firstChild ? textrec($_->firstChild) : '') } element_nodes($row)) . ']'); }
992 return $name . '[' . join(', ', @rows) . ']'; }
993  
994 sub is_genuinely_unparsed {
995 my ($node, $document) = @_;
996 # any unparsed fragment should be considered legitimate with one exception
997 # author-provided ungrammatical snippets in the presentation branches of XMDual
998 # are allowed to fail the parse process.
999 #
1000 # For now a reliable way of if we are in that case is to descend the formula through
1001 # the content branch of XMDual and check if any node has an "unparsed" mark.
1002 # Then only genuine parse failures will be detected.
1003 my $tag = $document->getNodeQName($node);
1004 if (($tag eq 'ltx:XMWrap') || ($tag eq 'ltx:XMArg') || $node->hasAttribute('_unparsed')) {
1005 return 1; }
1006 elsif (($tag eq 'ltx:XMTok') || ($tag eq 'ltx:XMText') || ($tag eq 'ltx:XMHint')) {
1007 return 0; }
1008 elsif ($tag eq 'ltx:XMRef') {
1009 # avoid infinite loops on malformed XMRefs that don't point anywhere
1010 if (!$node->getAttribute('idref')) {
1011 return 1; }
1012 else {
1013 my $real_node = realizeXMNode($node);
1014 # Note that some parses fail with an ARRAY ref [ltx:Error,...]
1015 # so realizeXMNode may fail!
1016 return 1 if (ref $real_node) !~ /^XML::LibXML/;
1017 return is_genuinely_unparsed($real_node, $document); } }
1018 elsif ($tag eq 'ltx:XMDual') {
1019 my ($content, $presentation) = element_nodes($node);
1020 return is_genuinely_unparsed($content, $document); }
1021 else {
1022 foreach my $child (element_nodes($node)) {
1023 return 1 if is_genuinely_unparsed($child, $document); }
1024 return 0; } }
1025  
1026 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
1027 # Cute! Were it NOT for Sub/Superscripts, the whole parsing process only
1028 # builds a new superstructure around the sequence of token nodes in the input.
1029 # Thus, any internal structure is unchanged.
1030 # They get re-parented, but if the parse fails, we've only got to put them
1031 # BACK into the original node, to recover the original arrangment!!!
1032 # Thus, we don't have to clone, and deal with namespace duplication.
1033 # ...
1034 # EXCEPT, as I said, for sub/superscripts!!!!
1035 #
1036  
1037 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
1038 # Constructors used in grammar
1039 # All the tree construction in the grammar should come through these operations.
1040 # We avoid mucking with the actual XML nodes (both to avoid modifying the original
1041 # tree until we have a successful parse, and to avoid XML::LibXML cloning nightmares)
1042 # We are converting XML nodes to array representation: [$tag, {%attr},@children]
1043 # This means any inspection of nodes has to recognize that
1044 # * node may be in XML vs ARRAY representation
1045 # * node may be an XMRef to another node whose properties are the ones we should use.
1046 #
1047 # Also, when we are examining a node's properties (roles, fences, script positioning, etc)
1048 # we should be careful to check for XMRef indirection and examine the properties
1049 # of the node that was referred to.
1050 # HOWEVER, we should construct our parse tree using (a clone of) the XMRef node,
1051 # rather than (a clone of) the referred to node, so as to preserve identity.
1052 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
1053 # We're currently keeping the id's on the nodes as they get cloned,
1054 # since they'll (maybe) replace the unparsed nodes.
1055 # However, if we consider multiple parses or preserving both parsed & unparsed,
1056 # we may have to do some adaptation and id shifting.
1057 # ================================================================================
1058  
1059 # ================================================================================
1060 # Low-level accessors
1061 sub Lookup {
1062 my ($lexeme) = @_;
1063 return $$LaTeXML::MathParser::LEXEMES{$lexeme}; }
1064  
1065 # The following accessors work on both the LibXML and ARRAY representations
1066 # but they do NOT automatically dereference XMRef!
1067 sub p_getValue {
1068 my ($node) = @_;
1069 if (!defined $node) {
1070 return; }
1071 elsif (ref $node eq 'XML::LibXML::Element') {
1072 my $x;
1073 return (($x = $node->textContent) ne '' ? $x # get content, or fall back to name
1074 : (defined($x = $node->getAttribute('name')) ? $x
1075 : undef)); }
1076 elsif (ref $node eq 'ARRAY') {
1077 my ($op, $attr, @args) = @$node;
1078 if (@args) {
1079 return join('', grep { defined $_ } map { p_getValue($_) } @args); }
1080 else {
1081 return $$node[1]{name}; } }
1082 elsif (ref $node eq 'XML::LibXML::Text') {
1083 return $node->textContent; }
1084 else { # ????
1085 return $node; } }
1086  
1087 sub p_getTokenMeaning {
1088 my ($item) = @_;
1089 if (!defined $item) {
1090 return; }
1091 elsif (ref $item eq 'ARRAY') {
1092 my ($op, $attr, @args) = @$item;
1093 return $$attr{meaning} || $$attr{name} || $args[0] || $$attr{role}; }
1094 elsif (ref $item eq 'XML::LibXML::Element') {
1095 return getTokenMeaning($item); } }
1096  
1097 sub p_getAttribute {
1098 my ($item, $key) = @_;
1099 if (!defined $item) {
1100 return; }
1101 elsif (ref $item eq 'ARRAY') {
1102 return $$item[1]{$key}; }
1103 elsif (ref $item eq 'XML::LibXML::Element') {
1104 return $item->getAttribute($key); } }
1105  
1106 sub p_setAttribute {
1107 my ($item, $key, $value) = @_;
1108 if (!defined $item) {
1109 return; }
1110 elsif (ref $item eq 'ARRAY') {
1111 return $$item[1]{$key} = $value; }
1112 elsif (ref $item eq 'XML::LibXML::Element') {
1113 return $item->setAttribute($key, $value); } }
1114  
1115 sub p_element_nodes {
1116 my ($item) = @_;
1117 if (!defined $item) {
1118 return (); }
1119 elsif (ref $item eq 'ARRAY') {
1120 my ($op, $attr, @args) = @$item;
1121 return grep { ref $_ } @args; }
1122 elsif (ref $item eq 'XML::LibXML::Element') {
1123 return element_nodes($item); } }
1124  
1125 sub p_getQName {
1126 my ($item) = @_;
1127 if (!defined $item) {
1128 return; }
1129 elsif (ref $item eq 'ARRAY') {
1130 return $$item[0]; }
1131 elsif (ref $item eq 'XML::LibXML::Element') {
1132 return getQName($item); } }
1133  
1134 # Make a new Token node with given name, content, and attributes.
1135 # $content is an array of nodes (which may need to be cloned if still attached)
1136 sub New {
1137 my ($meaning, $content, %attributes) = @_;
1138 my %attr = ();
1139 $attr{meaning} = $meaning if $meaning;
1140 foreach my $key (sort keys %attributes) {
1141 my $value = p_getValue($attributes{$key});
1142 $attr{$key} = $value if defined $value; }
1143 if (!$attr{font}) {
1144 $attr{font} = ($content && $content =~ /\S/
1145 ? LaTeXML::Common::Font->textDefault->specialize($content)
1146 : LaTeXML::Common::Font->new()); }
1147 return ['ltx:XMTok', {%attr}, ($content ? ($content) : ())]; }
1148  
1149 # Like New(), but concatenating several symbols into one new one.
1150 # This accounts for the symbols "disappearing", being replaced by one,
1151 # and thus any references to the old one have to be adjusted ... or removed
1152 # I'm a little concerned that we should be defering this till a successful parse?
1153 sub CatSymbols {
1154 my ($symbol1, $symbol2, $meaning, $content, %attributes) = @_;
1155 my $new = New($meaning, $content, %attributes);
1156 my $repl;
1157 my $symbols = [$symbol1, $symbol2];
1158 my $doc = $LaTeXML::MathParser::DOCUMENT;
1159 foreach my $symbol (@$symbols) {
1160 if (my $id = p_getAttribute(realizeXMNode($symbol), 'xml:id')) {
1161 if (!$repl) {
1162 my $newid = $doc->modifyID($id);
1163 $$new[1]{'xml:id'} = $newid;
1164 ReplacedBy($symbol, $new);
1165 $repl = 1; }
1166 else {
1167 foreach my $r ($doc->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']")) {
1168 $doc->removeNode($r); } } # ? Hopefully this is safe.
1169 } }
1170 return $new; }
1171  
1172 # Some handy shorthands.
1173 sub Absent {
1174 return New('absent'); }
1175  
1176 sub InvisibleTimes {
1177 return New('times', "\x{2062}", role => 'MULOP'); }
1178  
1179 sub InvisibleComma {
1180 return New(undef, "\x{2063}", role => 'PUNCT'); }
1181  
1182 # Get n-th arg of an XMApp.
1183 # However, this is really only used to get the script out of a sub/super script
1184 sub Arg {
1185 my ($node, $n) = @_;
1186 my $onode = $node;
1187 $node = realizeXMNode($node);
1188 if (ref $node eq 'ARRAY') {
1189 return $$node[$n + 2]; }
1190 else {
1191 my @args = element_nodes($node);
1192 my $nth = $args[$n];
1193 # Tricky case: if $node is an XMRef, we'll want to reference the SUB node too
1194 # and not just use it directly; else that node will be duplicated in both branches of XMDual
1195 if ($nth && !$node->isSameNode($onode)) {
1196 return LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $nth); }
1197 else {
1198 return $args[$n]; } } } # will get cloned if/when needed.
1199  
1200 # Add more attributes to a node.
1201 # Values can be strings or nodes whose text content is used.
1202 # Note that it avoids changing the underlying XML, so it may return a new object!
1203 sub Annotate {
1204 my ($node, %attributes) = @_;
1205 my %attrib = ();
1206 # first scan & convert any attributes, to make sure there really are any values.
1207 foreach my $attr (keys %attributes) {
1208 my $value = p_getValue($attributes{$attr});
1209 $attrib{$attr} = $value if defined $value; }
1210 # Hmm.... maybe we need to merge some things, open close?
1211 if (keys %attrib) { # Any attributes to assign?
1212 # If we've gotten a real XML node, convert to array representation
1213 # We do NOT want to modify any of the original XML!!!!
1214 if (ref $node ne 'ARRAY') {
1215 $node = [getQName($node),
1216 { map { (getQName($_) => $_->getValue) }
1217 grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes },
1218 $node->childNodes]; }
1219 my $qname = $$node[0];
1220 # Remove any attributes that aren't allowed!!!
1221 foreach my $k (keys %attrib) {
1222 delete $attrib{$k} unless $k =~ /^_/ || $LaTeXML::MathParser::DOCUMENT->canHaveAttribute($qname, $k); }
1223 # Make sure font is "Appropriate", if we're creating a new token
1224 if ($attrib{_font} && ($qname eq 'ltx:XMTok')) {
1225 my $content = join('', @$node[2 .. $#$node]);
1226 if ((!defined $content) || ($content eq '')) {
1227 delete $attrib{_font}; } # No font needed
1228 elsif (my $font = $LaTeXML::MathParser::DOCUMENT->decodeFont($attrib{_font})) {
1229 delete $attrib{_font};
1230 $attrib{font} = $font->specialize($content); } }
1231 map { $$node[1]{$_} = $attrib{$_} } keys %attrib; } # Now add them.
1232 return $node; }
1233  
1234 sub UTF {
1235 my ($code) = @_;
1236 return pack('U', $code); }
1237  
1238 # Convert a token with role=VERTBAR to a different role,
1239 # possibly changing the content to the appropriate Unicode.
1240 my %vertbar_category = (
1241 OPEN => 'delimiter', CLOSE => 'delimiter', MIDDLE => 'delimiter',
1242 PUNCT => 'operator');
1243 my %vertbar_glyph = (
1244 delimiter => { '|' => UTF(0x7C), '||' => "\x{2016}", '|||' => "\x{2980}",
1245 "\x{2225}" => "\x{2016}" },
1246 operator => { '|' => "\x{2223}", '||' => "\x{2225}", '|||' => "\x{2AF4}",
1247 "\x{2016}" => "\x{2225}" },
1248 );
1249  
1250 sub MorphVertbar {
1251 my ($node, $role) = @_;
1252 #return $node;
1253 my $rnode = realizeXMNode($node);
1254 my $oldrole = p_getAttribute($rnode, 'role');
1255 my $oldchar = $rnode->textContent;
1256 my $category = $vertbar_category{$role};
1257 my $char = ($oldchar && $category && $vertbar_glyph{$category}{$oldchar}) || $oldchar;
1258 my $newnode = $node;
1259 # Need to modify the token.
1260 if ($category && (($char ne $oldchar) || ($role ne $oldrole))) {
1261 # Collect up the attributes
1262 my %attrib = ();
1263 if (ref $node eq 'ARRAY') {
1264 %attrib = %{ $$node[2] }; }
1265 else {
1266 %attrib = map { (getQName($_) => $_->getValue) }
1267 grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes; }
1268 $attrib{role} = $role;
1269 $newnode = [getQName($node), {%attrib}, $char]; }
1270 return $newnode; }
1271  
1272 # ================================================================================
1273 # Mid-level constructors
1274  
1275 # Apply $op to the list of arguments
1276 # args may be array-rep or lexemes (or nodes?)
1277 sub Apply {
1278 my ($op, @args) = @_;
1279 my $font = p_getAttribute($op, '_font');
1280 return ['ltx:XMApp', { ($font ? (_font => $font) : ()) }, $op, @args]; }
1281  
1282 # Apply $op to a `delimited' list of arguments of the form
1283 # open, expr (punct expr)* close
1284 # after extracting the opening and closing delimiters, and the separating punctuation
1285 # Generate an XMDual, so that any styling of delimiters & punctuation is preserved.
1286 sub ApplyDelimited {
1287 my ($op, @stuff) = @_;
1288 my $open = $stuff[0];
1289 my $close = $stuff[-1];
1290 my ($seps, @args) = extract_separators(@stuff[1 .. $#stuff - 1]);
1291 return ['ltx:XMDual', {},
1292 Apply(LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $op),
1293 LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @args)),
1294 Apply($op, ['ltx:XMWrap', {}, @stuff])]; }
1295  
1296 # This is similar, but "interprets" a delimited list as being the
1297 # application of some operator to the items in the list.
1298 sub InterpretDelimited {
1299 my ($op, @stuff) = @_;
1300 my $open = $stuff[0];
1301 my $close = $stuff[-1];
1302 my ($seps, @args) = extract_separators(@stuff[1 .. $#stuff - 1]);
1303 return ['ltx:XMDual', {},
1304 Apply($op, LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @args)),
1305 ['ltx:XMWrap', {}, @stuff]]; }
1306  
1307 # Given a sequence of operators, form the nested application op(op(...(arg)))
1308 sub recApply {
1309 my (@ops) = @_;
1310 return (scalar(@ops) > 1 ? Apply(shift(@ops), recApply(@ops)) : $ops[0]); }
1311  
1312 # Given alternating expressions & separators (punctuation,...)
1313 # extract the separators as a concatenated string,
1314 # returning (separators, args...)
1315 # But note that the separators are never used for anything!?
1316 sub extract_separators {
1317 my (@stuff) = @_;
1318 my ($punct, @args);
1319 if (@stuff) {
1320 push(@args, shift(@stuff)); # Grab 1st expression
1321 while (@stuff) { # Expecting pairs of punct, expression
1322 my $p = realizeXMNode(shift(@stuff));
1323 $punct .=
1324 ($punct ? ' ' : '') # Delimited by SINGLE SPACE!
1325 . spacingToString(getXMHintSpacing(p_getAttribute($p, 'lpadding')))
1326 . (p_getValue($p) // ' ')
1327 . spacingToString(getXMHintSpacing(p_getAttribute($p, 'rpadding')));
1328 push(@args, shift(@stuff)); } } # Collect the next expression.
1329 @args = grep { $_ } @args; # drop all undef args, trailing punct could have added an undef
1330 return ($punct, @args); }
1331  
1332 # ================================================================================
1333 # Some special cases
1334  
1335 # This specifies the "meaning" of things within a pair
1336 # of open/close delimiters, depending on the number of things.
1337 # Really should be customizable?
1338 # Note that these are all Best Guesses, but really can have
1339 # alternate interpretations depending on context, field, etc.
1340 # Question: Is there enough context to guess better?
1341 # For example, whether (a,b) is an interval or list?
1342 # (both could reasonably be preceded by \in )
1343 my %balanced = ( # [CONSTANT]
1344 '(' => ')', '[' => ']', '{' => '}',
1345 '|' => '|', '||' => '||', "\x{2016}" => "\x{2016}", "\x{2980}" => "\x{2980}",
1346 "\x{230A}" => "\x{230B}", # lfloor, rfloor
1347 "\x{2308}" => "\x{2309}", # lceil, rceil
1348 "\x{2329}" => "\x{232A}", # angle brackets; NOT mathematical, but balance in case they show up.
1349 "\x{27E8}" => "\x{27E9}", # angle brackets (preferred)
1350 "\x{2225}" => "\x{2225}", # lVert, rVert
1351 );
1352 # For enclosing a single object
1353 # Note that the default here is just to put open/closed attributes on the single object
1354 my %enclose1 = ( # [CONSTANT]
1355 '{@}' => 'set', # alternatively, just variant parentheses
1356 '|@|' => 'absolute-value',
1357 '||@||' => 'norm', "\x{2016}@\x{2016}" => 'norm',
1358 "\x{230A}@\x{230B}" => 'floor',
1359 "\x{2308}@\x{2309}" => 'ceiling',
1360 '<@>' => 'expectation', # or just average?
1361 '<@|' => 'bra', '|@>' => 'ket');
1362 # For enclosing more than 2 objects; the punctuation is significant too
1363 my %enclose2 = ( # [CONSTANT]
1364 '(@,@)' => 'open-interval', # alternatively, just a list
1365 '[@,@]' => 'closed-interval',
1366 '(@,@]' => 'open-closed-interval', '[@,@)' => 'closed-open-interval',
1367 '{@,@}' => 'set', # alternatively, just a list ?
1368 );
1369 # For enclosing more than 2 objects.
1370 # assume 1st punct? or should we check all are same?
1371 my %encloseN = ( # [CONSTANT]
1372 '(@,@)' => 'vector', '{@,@}' => 'set',);
1373  
1374 sub isMatchingClose {
1375 my ($open, $close) = @_;
1376 my $oname = p_getValue(realizeXMNode($open));
1377 my $cname = p_getValue(realizeXMNode($close));
1378 my $expect = $oname && $balanced{$oname};
1379 return (defined $expect) && (defined $cname) && ($expect eq $cname); }
1380  
1381 # Given a delimited sequence: open expr (punct expr)* close
1382 # (OR, an empty sequence open close)
1383 # Convert it into the appropriate thing, depending on the specific open & close used.
1384 # Generate an XMDual to preserve any styling of delimiters and punctuation.
1385 sub Fence {
1386 my (@stuff) = @_;
1387 # Peak at delimiters to guess what kind of construct this is.
1388 my $nargs = scalar(@stuff);
1389 Error("expected", "arguments", undef,
1390 "Even number of arguments to Fence(); should be of form open,expr,(punct,expr)*,close",
1391 "got " . join(' ', map { ToString($_) } @stuff))
1392 if ($nargs != 2) && (($nargs % 2) == 0); # either empty or odd number
1393 my ($open, $close) = (realizeXMNode($stuff[0]), realizeXMNode($stuff[-1]));
1394 my $o = p_getValue($open);
1395 my $c = p_getValue($close);
1396 my $n = int(($nargs - 2 + 1) / 2);
1397 my @p = map { p_getValue(realizeXMNode(@stuff[2 * $_])) } 1 .. $n - 1;
1398 my $op = ($n == 0
1399 ? 'list' # ?
1400 : ($n == 1
1401 ? $enclose1{ $o . '@' . $c }
1402 : ($n == 2
1403 ? ($enclose2{ $o . '@' . $p[0] . '@' . $c } || 'list')
1404 : ($encloseN{ $o . '@' . $p[0] . '@' . $c } || 'list'))));
1405 $op = 'delimited-' . $o . $c unless defined $op;
1406 my $decl_id = p_getAttribute($open, 'decl_id');
1407 if (($n == 1) && ($op eq 'delimited-()')) { # Hopefully, can just ignore the parens?
1408 return ['ltx:XMDual', {},
1409 LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $stuff[1]),
1410 ['ltx:XMWrap', {}, @stuff]]; }
1411 else {
1412 return InterpretDelimited(New($op, undef, ($decl_id ? (decl_id => $decl_id) : ())), @stuff); } }
1413  
1414 # Compose a complex relational operator from two tokens, such as >=, >>
1415 # (similar to CatSymbols, but specialized to relops)
1416 sub TwoPartRelop {
1417 my ($op1, $op2) = @_;
1418 $op1 = Lookup($op1);
1419 $op2 = Lookup($op2);
1420 my $m1 = p_getTokenMeaning($op1);
1421 my $m2 = p_getTokenMeaning($op2);
1422 my $font = p_getAttribute($op1, '_font');
1423 my $meaning;
1424 if ($m1 eq $m2) {
1425 $meaning = "much-$m1"; }
1426 else {
1427 $meaning = "$m1-or-$m2"; }
1428 my $content = $op1->textContent . $op2->textContent;
1429 return ['ltx:XMTok', { role => "RELOP", meaning => $meaning, ($font ? (_font => $font) : ()) },
1430 $content]; }
1431  
1432 # NOTE: It might be best to separate the multiple Formulae into separate XMath's???
1433 # but only at the top level!
1434 sub NewFormulae {
1435 my (@stuff) = @_;
1436 if (scalar(@stuff) == 1) {
1437 return $stuff[0]; }
1438 else {
1439 my ($seps, @formula) = extract_separators(@stuff);
1440 return ['ltx:XMDual', {},
1441 Apply(New('formulae'),
1442 LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @formula)),
1443 ['ltx:XMWrap', {}, @stuff]]; } }
1444  
1445 # A Formula is an alternation of expr (relationalop expr)*
1446 # It presumably would be equivalent to (expr1 relop1 expr2) AND (expr2 relop2 expr3) ...
1447 # But, I haven't figured out the ideal prefix form that can easily be converted to presentation.
1448 sub NewFormula {
1449 my (@args) = @_;
1450 my $n = scalar(@args);
1451 if ($n == 1) {
1452 return $args[0]; }
1453 elsif ($n == 3) {
1454 return Apply($args[1], $args[0], $args[2]); }
1455 else {
1456 return Apply(New('multirelation'), @args); } }
1457  
1458 sub NewList {
1459 my (@stuff) = @_;
1460 # drop placeholder token for missing trailing punct, if any
1461 if (scalar(@stuff) > 1 && ((p_getTokenMeaning($stuff[-1]) || '') eq 'absent')) {
1462 pop(@stuff); }
1463 if (@stuff == 1) {
1464 return $stuff[0]; }
1465 else {
1466 my ($seps, @items) = extract_separators(@stuff);
1467 return ['ltx:XMDual', {},
1468 Apply(New('list'),
1469 LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @items)),
1470 ['ltx:XMWrap', {}, @stuff]]; } }
1471  
1472 # Given alternation of expr (addop expr)*, compose the tree (left recursive),
1473 # flattenning portions that have the same operator
1474 # ie. a + b + c - d => (- (+ a b c) d)
1475 sub LeftRec {
1476 no warnings 'recursion';
1477 my ($arg1, @more) = @_;
1478 if (@more) {
1479 my $op = shift(@more);
1480 my $opname = p_getTokenMeaning(realizeXMNode($op));
1481 my @args = ($arg1, shift(@more));
1482 while (@more && isSameExpr($op, $more[0])) {
1483 ReplacedBy($more[0], $op, 1);
1484 shift(@more);
1485 push(@args, shift(@more)); }
1486 return LeftRec(Apply($op, @args), @more); }
1487 else {
1488 return $arg1; } }
1489  
1490 # Like apply($op,$arg1,$arg2), but if $op is 'same' as the operator in $arg1,
1491 # then combine as an nary apply of $op to $arg1's arguments and $arg2.
1492 sub ApplyNary {
1493 my ($op, $arg1, $arg2) = @_;
1494 my $rop = realizeXMNode($op);
1495 my $opname = p_getTokenMeaning($rop) || '__undef_meaning__';
1496 my $opcontent = p_getValue($rop) || '__undef_content__';
1497 my @args = ();
1498 if (p_getQName($arg1) eq 'ltx:XMApp') {
1499 my ($op1, @args1) = p_element_nodes($arg1);
1500 my $rop1 = realizeXMNode($op1);
1501 if (isSameExpr($rop, $rop1)
1502 # Check that arg1 isn't wrapped, fenced or enclosed in some restrictive way
1503 # Especially an ID! (but really only important if the id is referenced somewhere?)
1504 && !(grep { p_getAttribute(realizeXMNode($arg1), $_) } qw(enclose xml:id))) {
1505 # Note that $op1 GOES AWAY!!!
1506 ReplacedBy($op1, $rop, 1);
1507 push(@args, @args1); }
1508 else {
1509 push(@args, $arg1); } }
1510 else {
1511 push(@args, $arg1); }
1512 return Apply($op, @args, $arg2); }
1513  
1514 # Usually we just expect to compare a token + to another.
1515 # but want (to some extent) to deal with embellished operators (eg. sub, sup...)
1516 # Rather involved if we want to try to do it "Right".
1517 sub isSameExpr {
1518 my ($op1, $op2) = @_;
1519 $op1 = realizeXMNode($op1);
1520 $op2 = realizeXMNode($op2);
1521 my $tag1 = getQName($op1);
1522 my $tag2 = getQName($op2);
1523 # Either both are tokens,
1524 # OR both have structure, but then we need to compare the children!!!!
1525 # First check, same top-level and critical attributes
1526 return unless
1527 ($tag1 eq $tag2)
1528 && ((p_getTokenMeaning($op1) || '__undef_meaning__')
1529 eq (p_getTokenMeaning($op2) || '__undef_meaning__'))
1530 && ((p_getValue($op1) || '__undef_content__')
1531 eq (p_getValue($op2) || '__undef_content__'))
1532 # Check that ops are used in same way.
1533 && ((p_getAttribute($op1, 'mathstyle') || '<none>')
1534 eq (p_getAttribute($op2, 'mathstyle') || '<none>'));
1535 if ($tag1 eq 'ltx:XMTok') { return 1; } # If tokens, they match
1536 else {
1537 my @ch1 = p_element_nodes($op1);
1538 my @ch2 = p_element_nodes($op2);
1539 my $n = scalar(@ch1);
1540 return unless $n == scalar(@ch2);
1541 foreach my $i (0 .. $n - 1) {
1542 return unless isSameExpr($ch1[$i], $ch1[$i]); }
1543 return 1; } }
1544  
1545 # There are several cases where parsing a formula will rearrange nodes
1546 # such that some nodes will no-longer be used. For example, when
1547 # converting a nested set of infix + into a single n-ary sum.
1548 # In effect, all those excess +'s are subsumed by the single first one.
1549 # It may be, however, that those lost nodes are referenced (XMRef) from the
1550 # other branch of an XMDual, and those references should be updated to refer
1551 # to the single node replacing the lost ones.
1552 # This function records that replacement, and the top-level parser fixes up the tree.
1553 # NOTE: There may be cases (in the Grammar, eg) where punctuation & ApplyOp's
1554 # get lost completely? Watch out for this!
1555 # If $isdup, assume the two trees are equivalent structure,
1556 # and so mark corresponding internal nodes as "replaced" as well.
1557 sub ReplacedBy {
1558 my ($lostnode, $keepnode, $isdup) = @_;
1559 return unless ref $lostnode && ref $keepnode;
1560 if (my $lostid = p_getAttribute($lostnode, 'xml:id')) {
1561 # Could be we want to generate an id for $keepnode, here?
1562 if (my $keepid = p_getAttribute($keepnode, 'xml:id')) {
1563 $$LaTeXML::MathParser::LOSTNODES{$lostid} = $keepid; }
1564 else {
1565 Warn('expected', 'id', undef, "LOST $lostid but no replacement!\n"); } }
1566  
1567 # The following recurses into the two trees,
1568 # This is on the assumption that they are "equivalent" trees
1569 if ($isdup) {
1570 my @lostkids = p_element_nodes($lostnode);
1571 my @keepkids = p_element_nodes($keepnode);
1572 my $n = scalar(@lostkids);
1573 my $m = scalar(@keepkids);
1574 if ($n != $m) {
1575 Error("unexpected", "nodes", undef, "Nodes aren't the same structure ($n vs $m)",
1576 "Old: " . printNode($lostnode),
1577 "New: " . printNode($keepnode)
1578 );
1579 $n = $m if $m < $n; }
1580 foreach my $i (0 .. $n - 1) {
1581 ReplacedBy($lostkids[$i], $keepkids[$i], $isdup); } }
1582 return; }
1583  
1584 # ================================================================================
1585 # Construct an appropriate application of sub/superscripts
1586 # This accounts for script positioning:
1587 # Whether it precedes (float), is over/under (if base requests),
1588 # or follows (normal case), along with whether sub/super.
1589 # the alignment of multiple sub/superscripts derived from the binding level when created.
1590 # scriptpos = (pre|mod|post) number; where number is the binding-level.
1591 # If $pos is given (pre|mid|post), it overrides the position implied by the script
1592 sub NewScript {
1593 my ($base, $script, $pos) = @_;
1594 my $role;
1595 my $rbase = realizeXMNode($base);
1596 my $rscript = realizeXMNode($script);
1597 my $ibase = $rbase;
1598 # Get "inner" (content) base, if the base is a dual it may be more relevant
1599 if (p_getQName($rbase) eq 'ltx:XMDual') {
1600 ($ibase) = p_element_nodes($rbase); }
1601 elsif (p_getQName($rbase) eq 'ltx:XMApp') {
1602 my ($op, $a1, @rest) = p_element_nodes($rbase);
1603 if ((p_getQName($op) eq 'ltx:XMTok')
1604 && ((p_getAttribute($op, 'role') || '') =~ /^(FLOAT|POST)?(SUB|SUPER)SCRIPT(OP)?$/)) {
1605 $ibase = $op; } }
1606 my @bsp = grep { $_ } (p_getAttribute($rbase, 'scriptpos'),
1607 p_getAttribute($rbase, 'scriptpos'),
1608 p_getAttribute($ibase, 'scriptpos'),
1609 'post');
1610 my ($bx) = grep { $_ } map { /^(pre|mid|post)?(\d+)?$/ && $1; } @bsp;
1611 my ($bl) = grep { $_ } map { /^(pre|mid|post)?(\d+)?$/ && $2; } @bsp;
1612 my ($sx, $sl) = (p_getAttribute($rscript, 'scriptpos') || 'post') =~ /^(pre|mid|post)?(\d+)?$/;
1613 $bl = ($sl || 1) unless $bl;
1614 $sl = ($bl || 1) unless $sl;
1615 $sx = ($bl == $sl ? $bx : 'post') unless $sx;
1616 my ($mode, $y) = p_getAttribute($rscript, 'role') =~ /^(FLOAT|POST)?(SUB|SUPER)SCRIPT$/;
1617 my $x = ($pos ? $pos : ($mode eq 'FLOAT' ? 'pre'
1618 : ($bl == $sl ? $bx : $sx) || 'post'));
1619 my $lpad = ($x eq 'pre') && p_getAttribute($rscript, 'lpadding');
1620 my $rpad = ($x ne 'pre') && p_getAttribute($rscript, 'rpadding');
1621 my $t;
1622 my $l = $sl || $bl ||
1623 (($t = $LaTeXML::MathParser::DOCUMENT->getNodeBox($script))
1624 && ($t->getProperty('scriptlevel'))) || 0;
1625 # If the INNER script was a floating script (ie. {}^{x})
1626 # we'll NOT want this one to stack over it so bump the level.
1627 my $bumped;
1628 if (p_getAttribute($rbase, '_wasfloat')) {
1629 $l++; $bumped = 1 }
1630 elsif (my $innerl = p_getAttribute($rbase, '_bumplevel')) {
1631 $l = $innerl; }
1632 my $app = Apply(New(undef, undef, role => $y . 'SCRIPTOP', scriptpos => "$x$l"),
1633 $base, Arg($script, 0));
1634 # Record whether this script was a floating one
1635 $$app[1]{_wasfloat} = 1 if $mode eq 'FLOAT';
1636 $$app[1]{_bumplevel} = $l if $bumped;
1637 $$app[1]{lpadding} = $lpad if $lpad && !$$app[1]{lpadding}; # better to add?
1638 $$app[1]{rpadding} = $rpad if $rpad && !$$app[1]{rpadding}; # better to add?
1639 return $app; }
1640  
1641 # Basically, like NewScript, but decorates an operator with sub/superscripts
1642 # (with vague unknown implications for meaning?)
1643 # but which will preserve the role (& meaning?)
1644 sub DecorateOperator {
1645 my ($op, $script) = @_;
1646 my $decop = NewScript($op, $script);
1647 my $rop = realizeXMNode($op);
1648 my $role = p_getAttribute($rop, 'role');
1649 return Annotate($decop, role => $role); }
1650  
1651 sub NewEvalAt {
1652 my ($base, $vertbar, $lower, $upper) = @_;
1653 my $pres = ['ltx:XMWrap', {}, $base, Annotate($vertbar, role => 'CLOSE')];
1654 $pres = NewScript($pres, $lower) if $lower;
1655 $pres = NewScript($pres, $upper) if $upper;
1656 return ['ltx:XMDual', {},
1657 ['ltx:XMApp', {},
1658 ['ltx:XMTok', { meaning => 'evaluated-at' }],
1659 LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT,
1660 $base,
1661 ($lower ? (Arg($lower, 0)) : ()),
1662 ($upper ? (Arg($upper, 0)) : ()))],
1663 $pres]; }
1664  
1665 # ================================================================================
1666 # A "notation" is a language construct or set thereof.
1667  
1668 # Called from the grammar to record the fact that a notation was seen.
1669 sub SawNotation {
1670 my ($notation, $node) = @_;
1671 $LaTeXML::MathParser::SEEN_NOTATIONS{$notation} = 1;
1672 return 1; }
1673  
1674 # Called by the grammar to determine whether we should try productions
1675 # which involve the given notation.
1676 sub IsNotationAllowed {
1677 my ($notation) = @_;
1678 return ($LaTeXML::MathParser::DISALLOWED_NOTATIONS{$notation} ? undef : 1); }
1679  
1680 # ================================================================================
1681  
1682 # Note that an UNKNOWN token may have been used as a function.
1683 # For simplicity in the grammar, we accept a token that has sub|super scripts applied.
1684 sub MaybeFunction {
1685 my ($token) = @_;
1686 $token = realizeXMNode($token);
1687 my $self = $LaTeXML::MathParser::PARSER;
1688 while (p_getQName($token) eq 'ltx:XMApp') {
1689 $token = Arg($token, 1); }
1690 my $name = token_prettyname($token);
1691 # DANGER!!
1692 # We want to be using Annotate here, but we're screwed up by the
1693 # potential "embellishing" of the function token.
1694 # (ie. the descent above past all XMApp's)
1695 $token->setAttribute('possibleFunction', 'yes');
1696 $$self{maybe_functions}{$name}++
1697 if $LaTeXML::MathParser::STRICT && !$$self{suspicious_tokens}{$token};
1698 $$self{suspicious_tokens}{$token} = 1;
1699 return; }
1700  
1701 # ================================================================================
1702 1;
1703  
1704 __END__