| 1 | # /=====================================================================\ # | ||
| 2 | # | LaTeXML::MathParser | # | ||
| 3 | # | Parse Math | # | ||
| 4 | # |=====================================================================| # | ||
| 5 | # | Part of LaTeXML: | # | ||
| 6 | # | Public domain software, produced as part of work done by the | # | ||
| 7 | # | United States Government & not subject to copyright in the US. | # | ||
| 8 | # |---------------------------------------------------------------------| # | ||
| 9 | # | Bruce Miller <bruce.miller@nist.gov> #_# | # | ||
| 10 | # | http://dlmf.nist.gov/LaTeXML/ (o o) | # | ||
| 11 | # \=========================================================ooo==U==ooo=/ # | ||
| 12 | # ================================================================================ | ||
| 13 | # LaTeXML::MathParser Math Parser for LaTeXML using Parse::RecDescent. | ||
| 14 | # Parse the intermediate representation generated by the TeX processor. | ||
| 15 | # ================================================================================ | ||
| 16 | package LaTeXML::MathParser; | ||
| 17 | use strict; | ||
| 18 | use warnings; | ||
| 19 | use Parse::RecDescent; | ||
| 20 | use LaTeXML::Global; | ||
| 21 | use LaTeXML::Common::Object; | ||
| 22 | use LaTeXML::Common::Error; | ||
| 23 | use LaTeXML::Core::Token; | ||
| 24 | use LaTeXML::Common::Font; | ||
| 25 | use LaTeXML::Common::XML; | ||
| 26 | use List::Util qw(min max); | ||
| 27 | use base (qw(Exporter)); | ||
| 28 | |||
| 29 | our @EXPORT_OK = (qw(&Lookup &New &Absent &Apply &ApplyNary &recApply &CatSymbols | ||
| 30 | &Annotate &InvisibleTimes &InvisibleComma &MorphVertbar | ||
| 31 | &TwoPartRelop &NewFormulae &NewFormula &NewList | ||
| 32 | &ApplyDelimited &NewScript &DecorateOperator &InterpretDelimited &NewEvalAt | ||
| 33 | &LeftRec | ||
| 34 | &Arg &MaybeFunction | ||
| 35 | &SawNotation &IsNotationAllowed | ||
| 36 | &isMatchingClose &Fence)); | ||
| 37 | our %EXPORT_TAGS = (constructors | ||
| 38 | => [qw(&Lookup &New &Absent &Apply &ApplyNary &recApply &CatSymbols | ||
| 39 | &Annotate &InvisibleTimes &InvisibleComma &MorphVertbar | ||
| 40 | &TwoPartRelop &NewFormulae &NewFormula &NewList | ||
| 41 | &ApplyDelimited &NewScript &DecorateOperator &InterpretDelimited &NewEvalAt | ||
| 42 | &LeftRec | ||
| 43 | &Arg &MaybeFunction | ||
| 44 | &SawNotation &IsNotationAllowed | ||
| 45 | &isMatchingClose &Fence)]); | ||
| 46 | |||
| 47 | DebuggableFeature('recdescent', "Trace Parse::RecDescent"); | ||
| 48 | our $MATHPARSE_PROGRESS_QUANTUM = 100; | ||
| 49 | |||
| 50 | # ================================================================================ | ||
| 51 | sub new { | ||
| 52 | my ($class, %options) = @_; | ||
| 53 | require LaTeXML::MathGrammar; | ||
| 54 | |||
| 55 | my $internalparser = LaTeXML::MathGrammar->new(); | ||
| 56 | Fatal("expected", "MathGrammar", undef, | ||
| 57 | "Compilation of Math Parser grammar failed") unless $internalparser; | ||
| 58 | my $self = bless { internalparser => $internalparser, lexematize => $options{lexematize} }, $class; | ||
| 59 | return $self; } | ||
| 60 | |||
| 61 | sub parseMath { | ||
| 62 | my ($self, $document, %options) = @_; | ||
| 63 | local $LaTeXML::MathParser::DOCUMENT = $document; | ||
| 64 | $self->clear; # Not reentrant! | ||
| 65 | $self->cleanupScripts($document); | ||
| 66 | if (my @math = $document->findnodes('descendant-or-self::ltx:XMath[not(ancestor::ltx:XMath)]')) { | ||
| 67 | my $proc = "Math Parsing " . scalar(@math) . " formulae ..."; | ||
| 68 | ProgressSpinup($proc); | ||
| 69 | #### SEGFAULT TEST | ||
| 70 | #### $document->doctest("before parse",1); | ||
| 71 | foreach my $math (@math) { | ||
| 72 | $self->parse($math, $document); } | ||
| 73 | ProgressSpindown($proc); | ||
| 74 | |||
| 75 | NoteLog("Math parsing succeeded:" | ||
| 76 | . join('', map { "\n $_: " | ||
| 77 | . colorizeString($$self{passed}{$_} . "/" . ($$self{passed}{$_} + $$self{failed}{$_}), ($$self{failed}{$_} == 0 ? 'success' : 'warning')) } | ||
| 78 | grep { $$self{passed}{$_} + $$self{failed}{$_} } | ||
| 79 | keys %{ $$self{passed} })); | ||
| 80 | if (my @unk = keys %{ $$self{unknowns} }) { | ||
| 81 | NoteLog("Symbols assumed as simple identifiers (with # of occurences):\n " | ||
| 82 | . join(', ', map { "'" . colorizeString("$_", 'warning') . "' ($$self{unknowns}{$_})" } sort @unk)); | ||
| 83 | if (!$STATE->lookupValue('MATHPARSER_SPECULATE')) { | ||
| 84 | NoteLog("Set MATHPARSER_SPECULATE to speculate on possible notations."); } } | ||
| 85 | if (my @funcs = keys %{ $$self{maybe_functions} }) { | ||
| 86 | NoteLog("Possibly used as functions?\n " | ||
| 87 | . join(', ', map { "'$_' ($$self{maybe_functions}{$_}/$$self{unknowns}{$_} usages)" } | ||
| 88 | sort @funcs)); } | ||
| 89 | } | ||
| 90 | #### SEGFAULT TEST | ||
| 91 | #### $document->doctest("IN scope",1); | ||
| 92 | #### SEGFAULT TEST | ||
| 93 | #### $document->doctest("OUT of scope",1); | ||
| 94 | return $document; } | ||
| 95 | |||
| 96 | # This is a rather peculiar cleanup that needs to be done to manage ids & idrefs | ||
| 97 | # Before parsing, sub/superscripts are represented by an operator-less XMApp with the script | ||
| 98 | # itself as the only child. Ideally, upon parsing these get merged, combined and disappear into | ||
| 99 | # proper XMApp of an appropriate operator on the base and scripts. Unless there is a parse | ||
| 100 | # failure, in which case they remain. | ||
| 101 | # The problem comes from various XMDual constructs where an XMRef refers to the script XMApp. | ||
| 102 | # It can occur that one branch parses and the other fails: This can leave a reference to | ||
| 103 | # the script XMApp which no longer exists! | ||
| 104 | # To solve this, we find & replace all references to such script XMApps by an explicit XMApp | ||
| 105 | # with the XMRef refering to the script itself, not the XMApp. (make sense?) | ||
| 106 | sub cleanupScripts { | ||
| 107 | my ($self, $document) = @_; | ||
| 108 | foreach my $app ($document->findnodes( | ||
| 109 | 'descendant-or-self::ltx:XMApp[@xml:id and contains(@role,"SCRIPT")]')) { | ||
| 110 | my $role = $app->getAttribute('role'); | ||
| 111 | my $appid = $app->getAttribute('xml:id'); | ||
| 112 | if ($role =~ /^(?:PRE|POST|FLOAT)(:?SUB|SUPER)SCRIPT$/) { | ||
| 113 | my @refs = $document->findnodes("descendant-or-self::ltx:XMRef[\@idref = '$appid']"); | ||
| 114 | if (scalar(@refs)) { | ||
| 115 | my $script = $app->firstChild; | ||
| 116 | my ($scriptref) = LaTeXML::Package::createXMRefs($document, $script); | ||
| 117 | $document->unRecordID($appid); # no longer refers to the app | ||
| 118 | $app->removeAttribute('xml:id'); | ||
| 119 | # Copy all attributes, EXCEPT xml:id | ||
| 120 | my %attr = map { (getQName($_) => $_->getValue) } | ||
| 121 | grep { $_->nodeType == XML_ATTRIBUTE_NODE } $app->attributes; | ||
| 122 | # Now, replace each ref to the script application by an application to a ref to the script. | ||
| 123 | foreach my $ref (@refs) { | ||
| 124 | $document->replaceTree(['ltx:XMApp', {%attr}, $scriptref], $ref); } | ||
| 125 | } } } | ||
| 126 | return; } | ||
| 127 | |||
| 128 | sub getQName { | ||
| 129 | my ($node) = @_; | ||
| 130 | if (ref $node eq 'ARRAY') { | ||
| 131 | return $$node[0]; } | ||
| 132 | else { | ||
| 133 | return $LaTeXML::MathParser::DOCUMENT->getModel->getNodeQName($node); } } | ||
| 134 | |||
| 135 | sub realizeXMNode { | ||
| 136 | my ($node) = @_; | ||
| 137 | my $doc = $LaTeXML::MathParser::DOCUMENT; | ||
| 138 | my $idref; | ||
| 139 | if (!defined $node) { | ||
| 140 | return; } | ||
| 141 | elsif (ref $node eq 'ARRAY') { | ||
| 142 | $idref = $$node[1]{idref} if $$node[0] eq 'ltx:XMRef'; } | ||
| 143 | elsif (ref $node eq 'XML::LibXML::Element') { | ||
| 144 | $idref = $node->getAttribute('idref') | ||
| 145 | if $doc->getModel->getNodeQName($node) eq 'ltx:XMRef'; } | ||
| 146 | if ($idref) { | ||
| 147 | # Can it happen that $realnode is, itself, an XMRef? Then we should recurse! | ||
| 148 | if (my $realnode = $doc->lookupID($idref)) { | ||
| 149 | return $realnode; } | ||
| 150 | else { | ||
| 151 | Error("expected", 'id', undef, "Cannot find a node with xml:id='$idref'", | ||
| 152 | ($LaTeXML::MathParser::IDREFS{$idref} | ||
| 153 | ? "Previously bound to " . ToString($LaTeXML::MathParser::IDREFS{$idref}) | ||
| 154 | : ())); | ||
| 155 | return ['ltx:ERROR', {}, "Missing XMRef idref=$idref"]; } } | ||
| 156 | else { | ||
| 157 | return $node; } } | ||
| 158 | |||
| 159 | # ================================================================================ | ||
| 160 | sub clear { | ||
| 161 | my ($self) = @_; | ||
| 162 | $$self{passed} = { 'ltx:XMath' => 0, 'ltx:XMArg' => 0, 'ltx:XMWrap' => 0 }; | ||
| 163 | $$self{failed} = { 'ltx:XMath' => 0, 'ltx:XMArg' => 0, 'ltx:XMWrap' => 0 }; | ||
| 164 | $$self{unknowns} = {}; | ||
| 165 | $$self{maybe_functions} = {}; | ||
| 166 | $$self{n_parsed} = 0; | ||
| 167 | $$self{progress} = 0; | ||
| 168 | return; } | ||
| 169 | |||
| 170 | our %EXCLUDED_PRETTYNAME_ATTRIBUTES = ( | ||
| 171 | fontsize => 1, opacity => 1, | ||
| 172 | color => 1, backgroundcolor => 1); | ||
| 173 | |||
| 174 | sub token_prettyname { | ||
| 175 | my ($node) = @_; | ||
| 176 | my $name = $node->getAttribute('name'); | ||
| 177 | if (defined $name) { } | ||
| 178 | elsif ($name = $node->textContent) { | ||
| 179 | my $font = $LaTeXML::MathParser::DOCUMENT->getNodeFont($node); | ||
| 180 | my %attr = $font->relativeTo(LaTeXML::Common::Font->textDefault); | ||
| 181 | my $desc = join(' ', map { ToString($attr{$_}{value}) } | ||
| 182 | (grep { !$EXCLUDED_PRETTYNAME_ATTRIBUTES{$_} } (sort keys %attr))); | ||
| 183 | $name .= "{$desc}" if $desc; } | ||
| 184 | else { | ||
| 185 | $name = Stringify($node); } # what else ???? | ||
| 186 | return $name; } | ||
| 187 | |||
| 188 | sub note_unknown { | ||
| 189 | my ($self, $node) = @_; | ||
| 190 | my $name = token_prettyname($node); | ||
| 191 | $$self{unknowns}{$name}++; | ||
| 192 | return; } | ||
| 193 | |||
| 194 | # debugging utility, should be somewhere handy. | ||
| 195 | sub printNode { | ||
| 196 | my ($node) = @_; | ||
| 197 | if (ref $node eq 'ARRAY') { | ||
| 198 | my ($tag, $attr, @children) = @$node; | ||
| 199 | my @keys = sort keys %$attr; | ||
| 200 | return "<$tag" | ||
| 201 | . (@keys ? ' ' . join(' ', map { "$_='" . ($$attr{$_} || '') . "'" } @keys) : '') | ||
| 202 | . (@children | ||
| 203 | ? ">\n" . join('', map { printNode($_) } @children) . "</$tag>" | ||
| 204 | : '/>') | ||
| 205 | . "\n"; } | ||
| 206 | else { | ||
| 207 | return ToString($node); } } | ||
| 208 | |||
| 209 | # ================================================================================ | ||
| 210 | # Some more XML utilities, but math specific (?) | ||
| 211 | |||
| 212 | # Get the Token's meaning, else name, else content, else role | ||
| 213 | sub getTokenMeaning { | ||
| 214 | my ($node) = @_; | ||
| 215 | my $x; | ||
| 216 | $node = realizeXMNode($node); | ||
| 217 | return (defined($x = p_getAttribute($node, 'meaning')) ? $x | ||
| 218 | : (defined($x = p_getAttribute($node, 'name')) ? $x | ||
| 219 | : (($x = (ref $node eq 'ARRAY' ? '' : $node->textContent)) ne '' ? $x | ||
| 220 | : (defined($x = p_getAttribute($node, 'role')) ? $x | ||
| 221 | : undef)))); } | ||
| 222 | |||
| 223 | sub node_location { | ||
| 224 | my ($node) = @_; | ||
| 225 | my $n = $node; | ||
| 226 | while ($n && (ref $n !~ /^XML::LibXML::Document/) # Sometimes DocuementFragment ??? | ||
| 227 | && !$n->getAttribute('refnum') && !$n->getAttribute('labels')) { | ||
| 228 | $n = $n->parentNode; } | ||
| 229 | if ($n && (ref $n !~ /^XML::LibXML::Document/)) { | ||
| 230 | my ($r, $l) = ($n->getAttribute('refnum'), $n->getAttribute('labels')); | ||
| 231 | return ($r && $l ? "$r ($l)" : $r || $l); } | ||
| 232 | else { | ||
| 233 | return 'Unknown'; } } | ||
| 234 | |||
| 235 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 236 | # Parser | ||
| 237 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 238 | # Top-level per-formula parse. | ||
| 239 | # We do a depth-first traversal of the content of the XMath element, | ||
| 240 | # since various sub-elements (XMArg & XMWrap) act as containers of | ||
| 241 | # nominally complete subexpressions. | ||
| 242 | # We do these first for two reasons. | ||
| 243 | # Firstly, since after parsing, the parent will be rebuilt from the result, | ||
| 244 | # we lose the node "identity"; ie. we can't find the child to replace it! | ||
| 245 | # Secondly, in principle (although this isn't used yet), parsing the | ||
| 246 | # child could reveal something interesting about it; say, it's effective role. | ||
| 247 | # Then, this information could be used when parsing the parent. | ||
| 248 | # In fact, this could work the other way too; parsing the parent could tell | ||
| 249 | # us something about what the child must be.... | ||
| 250 | sub parse { | ||
| 251 | my ($self, $xnode, $document) = @_; | ||
| 252 | local $LaTeXML::MathParser::STRICT = 1; | ||
| 253 | local $LaTeXML::MathParser::WARNED = 0; | ||
| 254 | local $LaTeXML::MathParser::UNPARSED = 0; | ||
| 255 | local $LaTeXML::MathParser::XNODE = $xnode; | ||
| 256 | local $LaTeXML::MathParser::PUNCTUATION = {}; | ||
| 257 | local $LaTeXML::MathParser::LOSTNODES = {}; | ||
| 258 | local %LaTeXML::MathParser::IDREFS = (); | ||
| 259 | # This bit for debugging.... | ||
| 260 | foreach my $n ($document->findnodes("descendant-or-self::*[\@xml:id]", $xnode)) { | ||
| 261 | my $id = $n->getAttribute('xml:id'); | ||
| 262 | $LaTeXML::MathParser::IDREFS{$id} = $n; } | ||
| 263 | if ($$self{lexematize}) { | ||
| 264 | my $lexeme_form = $self->node_to_lexeme_full($xnode); | ||
| 265 | $lexeme_form =~ s/^\s+//; | ||
| 266 | $lexeme_form =~ s/\s+$//; | ||
| 267 | $lexeme_form =~ s/\s+/ /g; # normalize internal whitespaces | ||
| 268 | $xnode->parentNode->setAttribute('lexemes', $lexeme_form); | ||
| 269 | } | ||
| 270 | if (my $result = $self->parse_rec($xnode, 'Anything,', $document)) { | ||
| 271 | # Add text representation to the containing Math element. | ||
| 272 | my $p = $xnode->parentNode; | ||
| 273 | # This is a VERY screwy situation? How can the parent be a document fragment?? | ||
| 274 | # This has got to be a LibXML bug??? | ||
| 275 | if ($p->nodeType == XML_DOCUMENT_FRAG_NODE) { | ||
| 276 | my @n = $p->childNodes; | ||
| 277 | if (scalar(@n) == 1) { | ||
| 278 | $p = $n[0]; } | ||
| 279 | else { | ||
| 280 | Fatal('malformed', '<XMath>', $xnode, "XMath node has DOCUMENT_FRAGMENT for parent!"); } } | ||
| 281 | # HACK: replace XMRef's to stray trailing punctution | ||
| 282 | foreach my $id (keys %$LaTeXML::MathParser::PUNCTUATION) { | ||
| 283 | my $r = $$LaTeXML::MathParser::PUNCTUATION{$id}->cloneNode; | ||
| 284 | $r->removeAttribute('xml:id'); | ||
| 285 | foreach my $n ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']", $p)) { | ||
| 286 | $document->replaceTree($r, $n); } } | ||
| 287 | foreach my $id (keys %$LaTeXML::MathParser::LOSTNODES) { | ||
| 288 | my $repid = $$LaTeXML::MathParser::LOSTNODES{$id}; | ||
| 289 | # but the replacement my have been replaced as well! | ||
| 290 | while (my $reprepid = $$LaTeXML::MathParser::LOSTNODES{$repid}) { | ||
| 291 | $repid = $reprepid; } | ||
| 292 | if ($document->findnodes("descendant-or-self::*[\@xml:id='$id']") | ||
| 293 | && !$document->findnodes("descendant-or-self::*[\@xml:id='$repid']")) { | ||
| 294 | # Do nothing if the node never actually got replaced (parse ultimately failed?) | ||
| 295 | } | ||
| 296 | else { | ||
| 297 | foreach my $n ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']", $p)) { | ||
| 298 | $document->setAttribute($n, idref => $repid); } } } | ||
| 299 | $p->setAttribute('text', text_form($result)); } | ||
| 300 | if ($LaTeXML::MathParser::UNPARSED && is_genuinely_unparsed($xnode, $document)) { | ||
| 301 | my $maybe_math = $xnode->parentNode; | ||
| 302 | if ($document->getNodeQName($maybe_math) eq 'ltx:Math') { | ||
| 303 | $maybe_math->setAttribute('class', 'ltx_math_unparsed'); } } | ||
| 304 | return; } | ||
| 305 | |||
| 306 | my %TAG_FEEDBACK = ('ltx:XMArg' => 'a', 'ltx:XMWrap' => 'w'); # [CONSTANT] | ||
| 307 | # Recursively parse a node with some internal structure | ||
| 308 | # by first parsing any structured children, then it's content. | ||
| 309 | sub parse_rec { | ||
| 310 | my ($self, $node, $rule, $document) = @_; | ||
| 311 | $self->parse_children($node, $document); | ||
| 312 | |||
| 313 | # This will only handle 1 layer nesting (successfully?) | ||
| 314 | # Note that this would have been found by the top level xpath, | ||
| 315 | # but we've got to worry about node identity: the parent is being rebuilt | ||
| 316 | foreach my $nested ($document->findnodes('descendant::ltx:XMath', $node)) { | ||
| 317 | $self->parse($nested, $document); } | ||
| 318 | my $tag = getQName($node); | ||
| 319 | if (my $requested_rule = $node->getAttribute('rule')) { | ||
| 320 | $rule = $requested_rule; } | ||
| 321 | if ($rule eq 'kludge') { | ||
| 322 | $self->parse_kludge($node, $document); | ||
| 323 | return; } | ||
| 324 | elsif (my $result = $self->parse_single($node, $document, $rule)) { | ||
| 325 | $$self{passed}{$tag}++; | ||
| 326 | if ($tag eq 'ltx:XMath') { # Replace the content of XMath with parsed result | ||
| 327 | ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0; | ||
| 328 | map { $document->unRecordNodeIDs($_) } element_nodes($node); | ||
| 329 | # unbindNode followed by (append|replace)Tree (which removes ID's) should be safe | ||
| 330 | map { $_->unbindNode() } $node->childNodes; | ||
| 331 | $document->appendTree($node, $result); | ||
| 332 | $result = [element_nodes($node)]->[0]; } | ||
| 333 | else { # Replace the whole node for XMArg, XMWrap; preserve some attributes | ||
| 334 | ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0; | ||
| 335 | # Copy all attributes | ||
| 336 | my $resultid = p_getAttribute($result, 'xml:id'); | ||
| 337 | my %attr = map { (getQName($_) => $_->getValue) } | ||
| 338 | grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes; | ||
| 339 | # add to result, even allowing modification of xml node, since we're committed. | ||
| 340 | # [Annotate converts node to array which messes up clearing the id!] | ||
| 341 | my $isarr = ref $result eq 'ARRAY'; | ||
| 342 | my $rtag = ($isarr ? $$result[0] : $document->getNodeQName($result)); | ||
| 343 | # Make sure font is "Appropriate", if we're creating a new token (yuck) | ||
| 344 | if ($isarr && $attr{_font} && ($rtag eq 'ltx:XMTok')) { | ||
| 345 | my $content = join('', @$result[2 .. $#$result]); | ||
| 346 | if ((!defined $content) || ($content eq '')) { | ||
| 347 | delete $attr{_font}; } # No font needed | ||
| 348 | elsif (my $font = $document->decodeFont($attr{_font})) { | ||
| 349 | delete $attr{_font}; | ||
| 350 | $attr{font} = $font->specialize($content); } } | ||
| 351 | else { | ||
| 352 | delete $attr{_font}; } | ||
| 353 | foreach my $key (keys %attr) { | ||
| 354 | next unless ($key =~ /^_/) || $document->canHaveAttribute($rtag, $key); | ||
| 355 | my $value = $attr{$key}; | ||
| 356 | if ($key eq 'xml:id') { # Since we're moving the id...bookkeeping | ||
| 357 | $document->unRecordID($value); | ||
| 358 | $node->removeAttribute('xml:id'); } | ||
| 359 | if ($isarr) { $$result[1]{$key} = $value; } | ||
| 360 | else { $document->setAttribute($result, $key => $value); } } | ||
| 361 | $result = $document->replaceTree($result, $node); | ||
| 362 | my $newid = $attr{'xml:id'}; | ||
| 363 | # Danger: the above code replaced the id on the parsed result with the one from XMArg,.. | ||
| 364 | # If there are any references to $resultid, we need to point them to $newid! | ||
| 365 | if ($resultid && $newid && ($resultid ne $newid)) { | ||
| 366 | foreach my $ref ($document->findnodes("//*[\@idref='$resultid']")) { | ||
| 367 | $ref->setAttribute(idref => $newid); } } | ||
| 368 | } | ||
| 369 | return $result; } | ||
| 370 | else { | ||
| 371 | $self->parse_kludge($node, $document); | ||
| 372 | ProgressStep() if ($$self{progress}++ % $MATHPARSE_PROGRESS_QUANTUM) == 0; | ||
| 373 | $$self{failed}{$tag}++; | ||
| 374 | return; } } | ||
| 375 | |||
| 376 | # Depth first parsing of XMArg nodes. | ||
| 377 | sub parse_children { | ||
| 378 | my ($self, $node, $document) = @_; | ||
| 379 | foreach my $child (element_nodes($node)) { | ||
| 380 | my $tag = getQName($child); | ||
| 381 | if ($tag eq 'ltx:XMArg') { | ||
| 382 | $self->parse_rec($child, 'Anything', $document); } | ||
| 383 | elsif ($tag eq 'ltx:XMWrap') { | ||
| 384 | local $LaTeXML::MathParser::STRICT = 0; | ||
| 385 | $self->parse_rec($child, 'Anything', $document); } | ||
| 386 | ### A nice evolution would be to use the Kludge parser for | ||
| 387 | ### the presentation form in XMDual | ||
| 388 | ### This would avoid silly "parses" of non-semantic stuff; eg assuming times between tokens! | ||
| 389 | ### However, it needs some experimentation to match DLMF's enhancements | ||
| 390 | #### $self->parse_children($child,$document); | ||
| 391 | #### $self->parse_kludge($child,$document); } | ||
| 392 | elsif ($tag =~ /^ltx:(XMApp|XMArray|XMRow|XMCell)$/) { | ||
| 393 | $self->parse_children($child, $document); } | ||
| 394 | elsif ($tag eq 'ltx:XMDual') { | ||
| 395 | $self->parse_children($child, $document); } | ||
| 396 | ### Some day this may be needed? but not yet... | ||
| 397 | # elsif ($tag eq 'ltx:XMRef') { | ||
| 398 | # # We might be referencing something outside the current tree. | ||
| 399 | # # but we'd like it's contents to be parsed before we start using it at this level. | ||
| 400 | # # NOTE: Do we have to worry about things being parsed twice? (ie. when the other tree's done) | ||
| 401 | # # if so, we probably want to mark things as having already been parsed. | ||
| 402 | # my $refnode = realizeXMNode($child); | ||
| 403 | # if(getQName($refnode) ne 'ltx:XMTok'){ # Anything referenced with structure... | ||
| 404 | # Debug("PRE-PARSING Ref'd node: ".ToString($child)." == ".ToString($refnode)); | ||
| 405 | # ## $self->parse_rec($refnode,'Anything',$document); | ||
| 406 | # $self->parse_children($refnode,$document); | ||
| 407 | # Debug("NOW REFERENCES: ".ToString(realizeXMNode($child))); | ||
| 408 | # } } | ||
| 409 | } | ||
| 410 | return; } | ||
| 411 | |||
| 412 | my $HINT_PUNCT_THRESHOLD = 10.0; # \quad or bigger becomes punctuation ? [CONSTANT] | ||
| 413 | |||
| 414 | sub filter_hints { | ||
| 415 | my ($self, $document, @nodes) = @_; | ||
| 416 | my @prefiltered = (); | ||
| 417 | my $prev = undef; | ||
| 418 | my $pending_comments = ''; | ||
| 419 | my $pending_space = 0.0; | ||
| 420 | my $pending_phantom = undef; | ||
| 421 | # Filter the nodes, watching for XMHint's and Comments. | ||
| 422 | foreach my $node (@nodes) { | ||
| 423 | my $type = $node->nodeType; | ||
| 424 | if ($type == XML_TEXT_NODE) { # text? better be (ignorable) whitespace! | ||
| 425 | my $string = $node->textContent; | ||
| 426 | if ($string =~ /\S/) { | ||
| 427 | Warn('unexpected', 'text', $node, "Unexpected text in Math tree"); } } | ||
| 428 | elsif ($type == XML_COMMENT_NODE) { # Comment! | ||
| 429 | my $comment = $node->getData; | ||
| 430 | if ($prev) { # Append to previous element's comments | ||
| 431 | my $c = $prev->getAttribute('_comment'); | ||
| 432 | $prev->setAttribute(_comment => ($c ? $c . "\n" . $comment : $comment)); } | ||
| 433 | else { # Or save for first | ||
| 434 | $pending_comments = ($pending_comments ? $pending_comments . "\n" . $comment : $comment); } } | ||
| 435 | elsif ($type != XML_ELEMENT_NODE) { | ||
| 436 | Warn('unexpected', 'node', $node, "Unexpected item in Math tree"); } | ||
| 437 | elsif (getQName($node) eq 'ltx:XMHint') { # If a Hint node? | ||
| 438 | if (my $width = $node->getAttribute('width')) { | ||
| 439 | if (my $pts = getXMHintSpacing($width)) { | ||
| 440 | my $ph = ($node->getAttribute('name') || '') =~ /phantom$/; | ||
| 441 | # If previous token (and not open paren), add the space to the right of it | ||
| 442 | if ($prev && (($prev->getAttribute('role') || '') ne 'OPEN')) { | ||
| 443 | my $s = $prev->getAttribute('_space') || 0.0; | ||
| 444 | my $p = $prev->getAttribute('_phantom'); | ||
| 445 | $prev->setAttribute(_space => $s + $pts); | ||
| 446 | $prev->setAttribute(_phantom => $p || $ph); } | ||
| 447 | else { # Else save it for the next token. | ||
| 448 | $pending_space += $pts; | ||
| 449 | $pending_phantom = $ph; } } } | ||
| 450 | # If XMHint is referenced?? | ||
| 451 | # Really shouldn't happen, but if XMHint is in XMWrap, it may end up with id....? | ||
| 452 | if (my $id = $node->getAttribute('xml:id')) { | ||
| 453 | if ($document->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']")) { | ||
| 454 | push(@prefiltered, $node); $prev = $node; } } } | ||
| 455 | # Don't store stuff on APPLYOP, since they tend to disappear. | ||
| 456 | elsif ((getQName($node) eq 'ltx:XMTok') && (($node->getAttribute('role') || '') eq 'APPLYOP')) { | ||
| 457 | ### push(@prefiltered, $node); $prev = $node; } # Keep it. | ||
| 458 | push(@prefiltered, $node); } # Keep it, but don't step $prev | ||
| 459 | else { # Some other element. | ||
| 460 | if ($pending_comments) { | ||
| 461 | $node->setAttribute(_pre_comment => $pending_comments); | ||
| 462 | $pending_comments = ''; } | ||
| 463 | if ($pending_space) { | ||
| 464 | $node->setAttribute(lpadding => | ||
| 465 | LaTeXML::Common::Dimension::attributeformat($pending_space * 65536)); | ||
| 466 | $node->setAttribute(_phantom => $pending_phantom); | ||
| 467 | $pending_space = 0.0; | ||
| 468 | $pending_phantom = undef; } | ||
| 469 | push(@prefiltered, $node); $prev = $node; } # Keep it. | ||
| 470 | } | ||
| 471 | # if $pending_space > 0, should store as rpadding on last element? | ||
| 472 | my @filtered = (); | ||
| 473 | # Filter through the pre-filtered nodes looking for large rpadding to convert to virtual PUNCT. | ||
| 474 | # But we'll assume that phantoms do NOT get converted to PUNCT ? | ||
| 475 | foreach my $node (@prefiltered) { | ||
| 476 | push(@filtered, $node); | ||
| 477 | if (my $s = $node->getAttribute('_space')) { | ||
| 478 | $node->removeAttribute('_space'); | ||
| 479 | if ((!$node->getAttribute('_phantom')) | ||
| 480 | && ($s >= $HINT_PUNCT_THRESHOLD) && (($node->getAttribute('role') || '') ne 'PUNCT')) { | ||
| 481 | # Treat the XMHint as Punctuation (role=PUNCT) | ||
| 482 | # I'm leary that this is a safe way to create an XML node that's NOT in the tree, but... | ||
| 483 | my $p = $node->parentNode; | ||
| 484 | my $punct = $document->openElementAt($p, 'ltx:XMHint', role => 'PUNCT', width => $s . 'pt', | ||
| 485 | name => ('q' x int($s / 10)) . 'uad'); | ||
| 486 | $p->removeChild($punct); # But don't actually leave it in the tree!!!! | ||
| 487 | push(@filtered, $punct); } | ||
| 488 | else { | ||
| 489 | $node->setAttribute(rpadding => | ||
| 490 | LaTeXML::Common::Dimension::attributeformat($s * 65536)); } } } | ||
| 491 | return @filtered; } | ||
| 492 | |||
| 493 | # Given a width attribute on an XMHint, return the pts, if any | ||
| 494 | sub getXMHintSpacing { | ||
| 495 | my ($width) = @_; | ||
| 496 | if ($width && ($width =~ /^$LaTeXML::Common::Glue::GLUE_re$/)) { | ||
| 497 | return ($2 eq 'mu' ? $1 / 1.8 : $1); } | ||
| 498 | else { | ||
| 499 | return 0; } } | ||
| 500 | |||
| 501 | # We've pretty much builtin the assumption that the target XML is "As If" 10 pts, | ||
| 502 | # so we'll assume that 1em is 10 pts. | ||
| 503 | my $POINTS_PER_EM = 10.0; # [CONSTANT] | ||
| 504 | # Convert spacing, given as a number of points, to a string of appropriate spacing chars | ||
| 505 | sub spacingToString { | ||
| 506 | my ($points) = @_; | ||
| 507 | my $spacing = ''; | ||
| 508 | my $ems = $points / $POINTS_PER_EM; | ||
| 509 | my $n = int($ems); | ||
| 510 | if ($n > 0) { $spacing .= ("\x{2003}" x $n); $ems -= $n; } | ||
| 511 | if ($ems > 0.500) { $spacing .= "\x{2002}"; $ems -= 0.500; } | ||
| 512 | if ($ems > 0.333) { $spacing .= "\x{2003}"; $ems -= 0.333; } | ||
| 513 | if ($ems > 0.250) { $spacing .= "\x{2005}"; $ems -= 0.250; } | ||
| 514 | if ($ems > 0.166) { $spacing .= "\x{2006}"; $ems -= 0.166; } | ||
| 515 | return $spacing; } | ||
| 516 | |||
| 517 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 518 | # Low-Level hack parsing when "real" parsing fails; | ||
| 519 | # Two issues cause generated Presentation MathML to be really bad: | ||
| 520 | # (1) not having mrow/mfenced structures wrapping OPEN...CLOSE sequences | ||
| 521 | # throws off MathML's stretchiness treatment of the fences | ||
| 522 | # (they're all the same size; big) | ||
| 523 | # (2) un-attached sub/superscripts won't position correctly, | ||
| 524 | # unless they're attached to something plausible. | ||
| 525 | # NOTE: we should be able to optionally switch this off. | ||
| 526 | # Especially, when we want to try alternative parse strategies. | ||
| 527 | sub parse_kludge { | ||
| 528 | my ($self, $mathnode, $document) = @_; | ||
| 529 | my @nodes = $self->filter_hints($document, $mathnode->childNodes); | ||
| 530 | # the 1st array in stack accumlates the nodes within the current fenced row. | ||
| 531 | # When there's only a single array, it's single entry will be the complete row. | ||
| 532 | my @stack = ([], []); | ||
| 533 | my @pairs = map { [$_, $self->getGrammaticalRole($_)] } @nodes; | ||
| 534 | while (scalar(@pairs) || (scalar(@stack) > 1)) { | ||
| 535 | my $pair = shift(@pairs); | ||
| 536 | my $role = ($pair ? $$pair[1] : 'CLOSE'); | ||
| 537 | if ($role eq 'OPEN') { | ||
| 538 | unshift(@stack, [$pair]); } # Start new fenced row; | ||
| 539 | elsif ($role eq 'CLOSE') { # Close the current row | ||
| 540 | my $row = shift(@stack); # get the current list of items | ||
| 541 | push(@$row, $pair) if $pair; # Put the close (if any) into it | ||
| 542 | |||
| 543 | my @kludged = $self->parse_kludgeScripts_rec(@$row); # handle scripts | ||
| 544 | # wrap, if needed. | ||
| 545 | $row = [(scalar(@kludged) > 1 ? ['ltx:XMWrap', {}, @kludged] : $kludged[0]), 'FENCED']; | ||
| 546 | push(@{ $stack[0] }, $row); } # and put this constructed row at end of containing row. | ||
| 547 | else { | ||
| 548 | push(@{ $stack[0] }, $pair); } } # Otherwise, just put this item into current row. | ||
| 549 | |||
| 550 | # If we got to here, remove the nodes and replace them by the kludged structure. | ||
| 551 | map { $document->unRecordNodeIDs($_) } element_nodes($mathnode); | ||
| 552 | # unbindNode followed by (append|replace)Tree (which removes ID's) should be safe | ||
| 553 | map { $_->unbindNode() } $mathnode->childNodes; | ||
| 554 | # We're hoping for a single list on the stack, | ||
| 555 | # But extra CLOSEs will leave extra junk behind, so process all the stacked lists. | ||
| 556 | my @replacements = (); | ||
| 557 | foreach my $pair (@{ $stack[0] }) { | ||
| 558 | my $kludge = $$pair[0]; | ||
| 559 | push(@replacements, (ref $kludge eq 'ARRAY') && ($$kludge[0] eq 'ltx:XMWrap') | ||
| 560 | ? @$kludge[2 .. $#$kludge] : ($kludge)); } | ||
| 561 | $document->appendTree($mathnode, @replacements); | ||
| 562 | p_setAttribute($mathnode, '_unparsed', '1'); # But mark as unparsed | ||
| 563 | return; } | ||
| 564 | |||
| 565 | sub parse_kludgeScripts_rec { | ||
| 566 | no warnings 'recursion'; | ||
| 567 | my ($self, $x, $y, @more) = @_; | ||
| 568 | my @acc = (); | ||
| 569 | while (defined $y) { | ||
| 570 | if ($$x[1] =~ /^FLOAT(SUB|SUPER)SCRIPT$/) { # a Floating script? maybe pre-script | ||
| 571 | if ($$y[1] =~ /^POST(SUB|SUPER)SCRIPT$/) { # followed by another script? | ||
| 572 | if (@more) { # followed by a something? => Combined pre sub & super | ||
| 573 | my ($base, @rest) = $self->parse_kludgeScripts_rec(@more); | ||
| 574 | return (@acc, NewScript(NewScript($base, $$y[0], 'pre'), $$x[0], 'pre'), @rest); } | ||
| 575 | else { # else floating sub & super | ||
| 576 | return (@acc, NewScript(NewScript(Absent(), $$y[0], 'post'), $$x[0], 'post')); } } | ||
| 577 | else { # else just prescript on whatever follows | ||
| 578 | my ($base, @rest) = $self->parse_kludgeScripts_rec($y, @more); | ||
| 579 | return (@acc, NewScript($base, $$x[0], 'pre'), @rest); } } | ||
| 580 | elsif ($$y[1] =~ /^POST(SUB|SUPER)SCRIPT$/) { | ||
| 581 | # or a postscript is applied to the preceding thing. | ||
| 582 | return (@acc, $self->parse_kludgeScripts_rec([NewScript($$x[0], $$y[0]), ''], @more)); } | ||
| 583 | else { # else skip over a and continue | ||
| 584 | push(@acc, $$x[0]); | ||
| 585 | ($x, $y, @more) = ($y, @more); } } | ||
| 586 | return (@acc, $$x[0]); } | ||
| 587 | |||
| 588 | # OBSOLETE EXPLORATORY CODE; but there are some interesting ideas.... | ||
| 589 | # sub parse_kludge { | ||
| 590 | # my($self,$mathnode,$document)=@_; | ||
| 591 | # my @nodes = $self->filter_hints($document,$mathnode->childNodes); | ||
| 592 | # map { $mathnode->removeChild($_) } @nodes; | ||
| 593 | # my @result=(); | ||
| 594 | # while(@nodes){ | ||
| 595 | # @nodes = $self->parse_kludge_rec(@nodes); | ||
| 596 | # push(@result,shift(@nodes)); } | ||
| 597 | # $document->appendTree($mathnode,@result); } | ||
| 598 | |||
| 599 | # # Kludge Parse the next thing, and then add any following scripts to it. | ||
| 600 | # sub parse_kludge_rec { | ||
| 601 | # my($self,@more)=@_; | ||
| 602 | # my($item,$open,$close,$seps,$arg); | ||
| 603 | # ($item,@more) = $self->parse_kludge_reca(@more); | ||
| 604 | # while(@more && (($self->getGrammaticalRole($more[0])||'') =~ /^POST(SUB|SUPER)SCRIPT$/)){ | ||
| 605 | # $item = NewScript($item,shift(@more)); } | ||
| 606 | # if(@more && (($self->getGrammaticalRole($more[0])||'') eq 'APPLYOP')){ | ||
| 607 | # shift(@more); | ||
| 608 | # if(@more && (($self->getGrammaticalRole($more[0])||'') eq 'OPEN')){ | ||
| 609 | # ($open,$close,$seps,$arg,@more)=$self->parse_kludge_fence(@more); | ||
| 610 | # $item = Apply(Annotate($item,argopen=>$open, argclose=>$close, separators=>$seps),@$arg); } | ||
| 611 | # else { | ||
| 612 | # ($arg,@more)=$self->parse_kludge_rec(@more); | ||
| 613 | # $item = Apply($item,$arg); }} | ||
| 614 | # ($item,@more); } | ||
| 615 | |||
| 616 | # sub parse_kludge_reca { | ||
| 617 | # my($self,$next,@more)=@_; | ||
| 618 | # my $role = $self->getGrammaticalRole($next); | ||
| 619 | # if($role =~ /^FLOAT(SUB|SUPER)SCRIPT$/){ | ||
| 620 | # my($base,@rest) = $self->parse_kludge_rec(@more); | ||
| 621 | # (NewScript($base,$next),@rest); } | ||
| 622 | # elsif($role eq 'OPEN'){ | ||
| 623 | # my($open,$close,$seps,$list,@more)=$self->parse_kludge_fence($next,@more); | ||
| 624 | # (Apply(Annotate(New(undef,undef,role=>'FENCED'), | ||
| 625 | # argopen=>$open, argclose=>$close, separators=>$seps),@$list), @more); } | ||
| 626 | # else { | ||
| 627 | # ($next,@more); }} | ||
| 628 | |||
| 629 | # sub parse_kludge_fence { | ||
| 630 | # my($self,$next,@more)=@_; | ||
| 631 | # my($open,$close,$punct,$r,$item,@list)=($next,undef,'',undef); | ||
| 632 | # while(@more){ | ||
| 633 | # my @i=(); | ||
| 634 | # while(@more && (($r=($self->getGrammaticalRole($more[0])||'')) !~ /^(CLOSE|PUNCT)$/)){ | ||
| 635 | # ($item,@more)=$self->parse_kludge_rec(@more); | ||
| 636 | # push(@i,$item); } | ||
| 637 | # push(@list,(scalar(@i > 1) ? ['ltx:XMWrap',{},@i] : $i[0])); | ||
| 638 | # if($r eq 'CLOSE'){ | ||
| 639 | # $close=shift(@more); last; } | ||
| 640 | # else { | ||
| 641 | # $punct .= ($punct ? ' ':''). p_getValue(shift(@more)); }} # Delimited by SINGLE SPACE! | ||
| 642 | # ($open,$close,$punct,[@list],@more); } | ||
| 643 | |||
| 644 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 645 | # Low-level Parser: parse a single expression | ||
| 646 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 647 | # Convert to textual form for processing by MathGrammar | ||
| 648 | sub parse_single { | ||
| 649 | my ($self, $mathnode, $document, $rule) = @_; | ||
| 650 | my @nodes = $self->filter_hints($document, $mathnode->childNodes); | ||
| 651 | |||
| 652 | my ($punct, $result, $unparsed); | ||
| 653 | my @punct = (); | ||
| 654 | # Extract trailing punctuation, if rule allows it. | ||
| 655 | if ($rule =~ s/,$//) { | ||
| 656 | # Collect ALL trailing PUNCT|PERIOD's... | ||
| 657 | my ($x, $r); | ||
| 658 | while (($x = $nodes[-1]) && ($x = realizeXMNode($x)) | ||
| 659 | && ($r = $x->getAttribute('role')) && (($r eq 'PUNCT') || ($r eq 'PERIOD'))) { | ||
| 660 | my $p = pop(@nodes); | ||
| 661 | # Special case hackery, in case this thing is XMRef'd!!! | ||
| 662 | # We could just stick it on the end of the presentation, | ||
| 663 | # but it doesn't belong in the content at all!?!? | ||
| 664 | if (my $id = $p->getAttribute('xml:id')) { | ||
| 665 | $$LaTeXML::MathParser::PUNCTUATION{$id} = $p; } | ||
| 666 | unshift(@punct, $p); } | ||
| 667 | } | ||
| 668 | |||
| 669 | if ($LaTeXML::DEBUG{recdescent}) { | ||
| 670 | $::RD_TRACE = 1; # Turn on MathGrammar tracing | ||
| 671 | # my $box = $document->getNodeBox($LaTeXML::MathParser::XNODE); | ||
| 672 | my $box = $document->getNodeBox($mathnode); | ||
| 673 | Debug(('=' x 60) . | ||
| 674 | "\nParsing formula \"" . ToString($box) . "\"" | ||
| 675 | . "\n from " . ToString($box->getLocator) | ||
| 676 | . "\n == \"" . join(' ', map { node_string($_, $document) } @nodes) . "\"" | ||
| 677 | . "\n == " . ToString($mathnode)); } | ||
| 678 | |||
| 679 | if (scalar(@nodes) < 2) { # Too few nodes? What's to parse? | ||
| 680 | $result = $nodes[0] || Absent(); } | ||
| 681 | else { | ||
| 682 | # Now do the actual parse. | ||
| 683 | ($result, $unparsed) = $self->parse_internal($rule, @nodes); | ||
| 684 | } | ||
| 685 | # Failure? No result or uparsed lexemes remain. | ||
| 686 | # NOTE: Should do script hack?? | ||
| 687 | if ((!defined $result) || $unparsed) { | ||
| 688 | $LaTeXML::MathParser::UNPARSED = 1; | ||
| 689 | $self->failureReport($document, $mathnode, $rule, $unparsed, @nodes); | ||
| 690 | return; } | ||
| 691 | # Success! | ||
| 692 | else { | ||
| 693 | if (@punct) { # create a trivial XMDual to treat the punctuation as presentation | ||
| 694 | $result = ['ltx:XMDual', {}, | ||
| 695 | LaTeXML::Package::createXMRefs($document, $result), | ||
| 696 | ['ltx:XMWrap', {}, $result, @punct]]; } # or perhaps: Apply, punctuated??? | ||
| 697 | if ($LaTeXML::DEBUG{recdescent}) { | ||
| 698 | Debug("=>" . printNode($result) . "\n" . ('=' x 60)); } | ||
| 699 | return $result; } } | ||
| 700 | |||
| 701 | sub node_to_lexeme { | ||
| 702 | my ($self, $node) = @_; | ||
| 703 | my $qname = getQName($node); | ||
| 704 | my $is_inked = 1; | ||
| 705 | my $lexeme = ($qname eq 'ltx:XMTok' or $qname eq 'ltx:text') && $node->textContent; | ||
| 706 | if (!defined($lexeme) || length($lexeme) == 0) { | ||
| 707 | $is_inked = 0; | ||
| 708 | $lexeme = getTokenMeaning($node); } | ||
| 709 | my $document = $LaTeXML::MathParser::DOCUMENT; | ||
| 710 | return unless defined $lexeme; | ||
| 711 | if ($is_inked) { | ||
| 712 | if (my $font = $node->getAttribute('_font')) { | ||
| 713 | my $font_spec = $document->decodeFont($font); | ||
| 714 | if (my %declarations = $font_spec && $font_spec->relativeTo(LaTeXML::Common::Font->textDefault)) { | ||
| 715 | my %to_add = (); | ||
| 716 | my $font_pending = $declarations{font} || {}; | ||
| 717 | my $properties_pending = $$font_pending{properties} || {}; | ||
| 718 | foreach my $attr (qw(family series shape)) { | ||
| 719 | if (my $value = $$properties_pending{$attr}) { | ||
| 720 | $to_add{$value} = 1; } } | ||
| 721 | if (%to_add) { | ||
| 722 | my $prefix = join("_", sort(keys(%to_add))) . "_"; | ||
| 723 | $lexeme = join(" ", map { $prefix . $_ } split(' ', $lexeme)); } | ||
| 724 | } } | ||
| 725 | if ($lexeme =~ /^\p{L}+$/) { | ||
| 726 | $lexeme = join(" ", map { 'roman_' . $_ } split(' ', $lexeme)); } } | ||
| 727 | $lexeme =~ s/\s+$//; | ||
| 728 | return $lexeme; } | ||
| 729 | |||
| 730 | sub node_to_lexeme_full { | ||
| 731 | my ($self, $unrealized_node) = @_; | ||
| 732 | my $node = realizeXMNode($unrealized_node); | ||
| 733 | my $tag = getQName($node); | ||
| 734 | if ($tag eq 'ltx:XMHint') { return ""; } # just skip XMHints, they don't contain lexemes | ||
| 735 | my $role = p_getAttribute($node, 'role'); | ||
| 736 | if ($tag eq 'ltx:XMTok' || ($role && ($tag !~ 'ltx:XM(Dual|App|Arg|Array|Wrap|ath)'))) { | ||
| 737 | # Elements that directly represent a lexeme, or intended operation with a syntactic role (such as a postscript), | ||
| 738 | # can proceed to building the lexeme from the leaf node. | ||
| 739 | my $lexeme = $self->node_to_lexeme($node); | ||
| 740 | if ($role && $role =~ /^(UNDER|OVER)ACCENT$/) { | ||
| 741 | # over¯ and under¯ are the lexeme names of choice for \overline and \underline | ||
| 742 | $lexeme = lc($1) . $lexeme; } | ||
| 743 | return $lexeme; } | ||
| 744 | # Elements that do not have a role and are intermediate "may" need an argument wrapper, so that arguments | ||
| 745 | # remain unambiguous. For instance a `\frac{a}{b}` has clear argument structure to be preserved. | ||
| 746 | my ($mark_start, $mark_end) = ('', ''); | ||
| 747 | if ($tag ne 'ltx:XMath') { | ||
| 748 | if ($role) { | ||
| 749 | $mark_start = "start_$role "; | ||
| 750 | $mark_end = " end_$role"; | ||
| 751 | } elsif ($tag =~ '^ltx:XM(Arg|Row|Cell)') { | ||
| 752 | my $tag_role = uc($1); | ||
| 753 | $mark_start = "start_$tag_role "; | ||
| 754 | $mark_end = " end_$tag_role"; | ||
| 755 | } | ||
| 756 | } | ||
| 757 | my $lexemes = $mark_start; | ||
| 758 | if ($tag eq 'ltx:XMDual') { | ||
| 759 | $lexemes .= $self->node_to_lexeme_full($LaTeXML::MathParser::DOCUMENT->getSecondChildElement($node)); } | ||
| 760 | elsif ($tag eq 'ltx:XMText') { | ||
| 761 | # can be either a single node XMText, we're looking at a leaf text node | ||
| 762 | # or \text{}-like construct, with multiple math formulas and interleaved text | ||
| 763 | foreach my $child ($node->childNodes) { | ||
| 764 | if (ref($child) eq 'XML::LibXML::Text') { | ||
| 765 | $lexemes .= $child->textContent() . ' '; } | ||
| 766 | else { | ||
| 767 | my $child_lexeme = $self->node_to_lexeme_full($child); | ||
| 768 | if (defined $child_lexeme && length($child_lexeme) > 0) { | ||
| 769 | $lexemes .= $child_lexeme . ' '; } } } } | ||
| 770 | elsif ($tag eq 'ltx:text') { # could recurse in from ltx:XMText | ||
| 771 | return $self->node_to_lexeme($node); } | ||
| 772 | else { | ||
| 773 | my @child_elements = element_nodes($node); | ||
| 774 | # skip through single child wrappers (don't serialize) | ||
| 775 | while (scalar(@child_elements) == 1 && getQName($child_elements[0]) =~ /^ltx:XM(Arg|Wrap)$/) { | ||
| 776 | @child_elements = element_nodes($child_elements[0]); | ||
| 777 | } | ||
| 778 | foreach my $child (@child_elements) { | ||
| 779 | my $child_lexeme = $self->node_to_lexeme_full($child); | ||
| 780 | if (defined $child_lexeme && length($child_lexeme) > 0) { | ||
| 781 | $lexemes .= $child_lexeme . ' '; | ||
| 782 | } } } | ||
| 783 | $lexemes .= $mark_end; | ||
| 784 | return $lexemes; } | ||
| 785 | |||
| 786 | sub parse_internal { | ||
| 787 | my ($self, $rule, @nodes) = @_; | ||
| 788 | #------------ | ||
| 789 | # Generate a textual token for each node; The parser operates on this encoded string. | ||
| 790 | local $LaTeXML::MathParser::LEXEMES = {}; | ||
| 791 | my $i = 0; | ||
| 792 | my $lexemes = ''; | ||
| 793 | |||
| 794 | foreach my $node (@nodes) { | ||
| 795 | # This is a parser-specific lexeme, but it is not (yet) identical to the serialized lexeme by | ||
| 796 | # ->node_to_lexeme, which is currently experimental | ||
| 797 | my $role = $self->getGrammaticalRole($node); | ||
| 798 | my $text = getTokenMeaning($node); | ||
| 799 | $text = 'Unknown' unless defined $text; | ||
| 800 | my $lexeme = $role . ":" . $text . ":" . ++$i; | ||
| 801 | $lexeme =~ s/\s//g; | ||
| 802 | |||
| 803 | $$LaTeXML::MathParser::LEXEMES{$lexeme} = $node; | ||
| 804 | $lexemes .= ' ' . $lexeme; } | ||
| 805 | |||
| 806 | #------------ | ||
| 807 | # apply the parser to the textified sequence. | ||
| 808 | local $LaTeXML::MathParser::PARSER = $self; | ||
| 809 | local %LaTeXML::MathParser::SEEN_NOTATIONS = (); | ||
| 810 | local %LaTeXML::MathParser::DISALLOWED_NOTATIONS = (); | ||
| 811 | local $LaTeXML::MathParser::MAX_ABS_DEPTH = 1; | ||
| 812 | # "Speculative" mode is disabled by default (set MATHPARSER_SPECULATE). | ||
| 813 | # It causes the parser to explore possible undeclared notatations, | ||
| 814 | # like possible functions (the only one, so far) | ||
| 815 | # These extra grammar rules can be costly in corner cases. | ||
| 816 | # This should be disabled when speed (but feedback) is essential | ||
| 817 | if ($STATE->lookupValue('MATHPARSER_SPECULATE')) { | ||
| 818 | $LaTeXML::MathParser::DISALLOWED_NOTATIONS{MaybeFunctions} = 0; } | ||
| 819 | else { | ||
| 820 | $LaTeXML::MathParser::DISALLOWED_NOTATIONS{MaybeFunctions} = 1; } | ||
| 821 | my $unparsed = $lexemes; | ||
| 822 | my $result = $$self{internalparser}->$rule(\$unparsed); | ||
| 823 | if (((!defined $result) || $unparsed) # If parsing Failed | ||
| 824 | && $LaTeXML::MathParser::SEEN_NOTATIONS{QM}) { # & Saw some QM stuff. | ||
| 825 | $LaTeXML::MathParser::DISALLOWED_NOTATIONS{QM} = 1; # Retry w/o QM notations | ||
| 826 | $unparsed = $lexemes; | ||
| 827 | $result = $$self{internalparser}->$rule(\$unparsed); } | ||
| 828 | while (((!defined $result) || $unparsed) # If parsing Failed | ||
| 829 | && ($LaTeXML::MathParser::SEEN_NOTATIONS{AbsFail}) # & Attempted deeper abs nesting? | ||
| 830 | && ($LaTeXML::MathParser::MAX_ABS_DEPTH < 3)) { # & Not ridiculously deep | ||
| 831 | delete $LaTeXML::MathParser::SEEN_NOTATIONS{AbsFail}; | ||
| 832 | ++$LaTeXML::MathParser::MAX_ABS_DEPTH; # Try deeper. | ||
| 833 | $unparsed = $lexemes; | ||
| 834 | $result = $$self{internalparser}->$rule(\$unparsed); } | ||
| 835 | |||
| 836 | # If still failed, try other strategies? | ||
| 837 | |||
| 838 | return ($result, $unparsed); } | ||
| 839 | |||
| 840 | sub getGrammaticalRole { | ||
| 841 | my ($self, $node) = @_; | ||
| 842 | $node = realizeXMNode($node); | ||
| 843 | my $role = p_getAttribute($node, 'role'); | ||
| 844 | if (!defined $role) { | ||
| 845 | my $tag = getQName($node); | ||
| 846 | if ($tag eq 'ltx:XMTok') { | ||
| 847 | $role = 'UNKNOWN'; } | ||
| 848 | elsif ($tag eq 'ltx:XMDual') { | ||
| 849 | my ($content, $presentation) = element_nodes($node); | ||
| 850 | $role = p_getAttribute($content, 'role') || p_getAttribute($presentation, 'role'); } | ||
| 851 | $role = 'ATOM' unless defined $role; } | ||
| 852 | $self->note_unknown($node) if ($role eq 'UNKNOWN') && $LaTeXML::MathParser::STRICT; | ||
| 853 | return $role; } | ||
| 854 | |||
| 855 | # How many tokens before & after the failure point to report in the Warning message. | ||
| 856 | my $FAILURE_PRETOKENS = 3; # [CONSTANT] | ||
| 857 | my $FAILURE_POSTTOKENS = 1; # [CONSTANT] | ||
| 858 | |||
| 859 | sub failureReport { | ||
| 860 | my ($self, $document, $mathnode, $rule, $unparsed, @nodes) = @_; | ||
| 861 | if ($LaTeXML::MathParser::STRICT || (($LaTeXML::Common::Error::VERBOSITY || 0) > 1)) { | ||
| 862 | my $loc = ""; | ||
| 863 | # If we haven't already done it for this formula, show the original TeX. | ||
| 864 | if (!$LaTeXML::MathParser::WARNED) { | ||
| 865 | $LaTeXML::MathParser::WARNED = 1; | ||
| 866 | my $box = $document->getNodeBox($LaTeXML::MathParser::XNODE); | ||
| 867 | $loc = "In \"" . UnTeX($box) . "\""; } | ||
| 868 | $unparsed =~ s/^\s*//; | ||
| 869 | my @rest = split(/ /, $unparsed); | ||
| 870 | my $pos = scalar(@nodes) - scalar(@rest); | ||
| 871 | # Break up the input at the point where the parse failed. | ||
| 872 | my $max = 50; | ||
| 873 | my $parsed = join(' ', ($pos > $max ? ('...') : ()), | ||
| 874 | (map { node_string($_, $document) } @nodes[max(0, $pos - 50) .. $pos - 1])); | ||
| 875 | my $toparse = join(' ', | ||
| 876 | (map { node_string($_, $document) } @nodes[$pos .. min($pos + 50, $#nodes)]), | ||
| 877 | ($#nodes > $pos + 50 ? ('...') : ())); | ||
| 878 | my $parsefail | ||
| 879 | = join('.', map { $self->getGrammaticalRole($_) } | ||
| 880 | @nodes[($pos - $FAILURE_PRETOKENS >= 0 | ||
| 881 | ? $pos - $FAILURE_PRETOKENS : 0) .. $pos - 1]) | ||
| 882 | . ">" | ||
| 883 | . join('.', map { $self->getGrammaticalRole($_) } | ||
| 884 | @nodes[$pos .. ($pos + $FAILURE_POSTTOKENS - 1 < $#nodes | ||
| 885 | ? $pos + $FAILURE_POSTTOKENS - 1 : $#nodes)]); | ||
| 886 | my $lexeme = node_location($nodes[$pos] || $nodes[$pos - 1] || $mathnode); | ||
| 887 | my $indent = length($parsed) - 2; $indent = 8 if $indent > 8; | ||
| 888 | Warn('not_parsed', $parsefail, $mathnode, | ||
| 889 | "MathParser failed to match rule '$rule'", | ||
| 890 | ($loc ? ($loc) : ()), | ||
| 891 | ($parsed | ||
| 892 | ? ($parsed, (' ' x $indent) . "> " . $toparse) | ||
| 893 | : ("> " . $toparse))); | ||
| 894 | } | ||
| 895 | return; } | ||
| 896 | |||
| 897 | # used for debugging & failure reporting. | ||
| 898 | sub node_string { | ||
| 899 | my ($node, $document) = @_; | ||
| 900 | my $role = $node->getAttribute('role') || 'UNKNOWN'; | ||
| 901 | my $box = $document->getNodeBox($node); | ||
| 902 | return ($box ? ToString($box) : text_form($node)) . "[[$role]]"; } | ||
| 903 | |||
| 904 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 905 | # Conversion to a less ambiguous, mostly-prefix form. | ||
| 906 | # Mostly for debugging information? | ||
| 907 | # Note that the nodes are true libXML nodes, already absorbed into the document | ||
| 908 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 909 | sub text_form { | ||
| 910 | my ($node) = @_; | ||
| 911 | return textrec($node, undef); } | ||
| 912 | |||
| 913 | my %PREFIX_ALIAS = ( # [CONSTANT] | ||
| 914 | SUPERSCRIPTOP => '^', SUBSCRIPTOP => '_', times => '*', | ||
| 915 | 'equals' => '=', 'less-than' => '<', 'greater-than' => '>', | ||
| 916 | 'less-than-or-equals' => '<=', 'greater-than-or-equals' => '>=', | ||
| 917 | 'much-less-than' => '<<', 'much-greater-than' => '>>', | ||
| 918 | 'plus' => '+', 'minus' => '-', 'divide' => '/'); | ||
| 919 | # Put infix, along with `binding power' | ||
| 920 | my %IS_INFIX = (METARELOP => 1, # [CONSTANT] | ||
| 921 | RELOP => 2, ARROW => 2, | ||
| 922 | ADDOP => 10, MULOP => 100, FRACOP => 100, | ||
| 923 | SUPERSCRIPTOP => 1000, SUBSCRIPTOP => 1000); | ||
| 924 | |||
| 925 | sub textrec { | ||
| 926 | no warnings 'recursion'; | ||
| 927 | my ($node, $outer_bp, $outer_name) = @_; | ||
| 928 | return '[missing]' unless defined $node; | ||
| 929 | $node = realizeXMNode($node); | ||
| 930 | my $tag = getQName($node); | ||
| 931 | $outer_bp = 0 unless defined $outer_bp; | ||
| 932 | $outer_name = '' unless defined $outer_name; | ||
| 933 | # If node has meaning, that's the text form. | ||
| 934 | if (my $meaning = p_getAttribute($node, 'meaning') || p_getAttribute($node, 'name')) { | ||
| 935 | return $PREFIX_ALIAS{$meaning} || $meaning; } | ||
| 936 | elsif ($tag eq 'ltx:XMApp') { | ||
| 937 | my $app_role = $node->getAttribute('role'); | ||
| 938 | my ($op, @args) = element_nodes($node); | ||
| 939 | $op = realizeXMNode($op); | ||
| 940 | if ($app_role && $app_role =~ /^FLOAT(SUB|SUPER)SCRIPT$/) { | ||
| 941 | return ($1 eq 'SUPER' ? '^' : '_') . textrec($op); } | ||
| 942 | else { | ||
| 943 | my $name = ((getQName($op) eq 'ltx:XMTok') && getTokenMeaning($op)) || 'unknown'; | ||
| 944 | my ($bp, $string) = textrec_apply($name, $op, @args); | ||
| 945 | return (($bp < $outer_bp) || (($bp == $outer_bp) && ($name ne $outer_name)) | ||
| 946 | ? '(' . $string . ')' : $string); } } | ||
| 947 | elsif ($tag eq 'ltx:XMDual') { | ||
| 948 | my ($content, $presentation) = element_nodes($node); | ||
| 949 | my $text = textrec($content, $outer_bp, $outer_name); # Just send out the semantic form. | ||
| 950 | # Fall back to presentation, if content has poor semantics (eg. from replacement patterns) | ||
| 951 | return ($text =~ /^\(*Unknown/ ? textrec($presentation, $outer_bp, $outer_name) : $text); } | ||
| 952 | elsif ($tag eq 'ltx:XMTok') { | ||
| 953 | my $name = getTokenMeaning($node); | ||
| 954 | $name = 'Unknown' unless defined $name; | ||
| 955 | return $PREFIX_ALIAS{$name} || $name; } | ||
| 956 | elsif (($tag eq 'ltx:XMWrap') || ($tag eq 'ltx:XMCell')) { | ||
| 957 | # ?? | ||
| 958 | return join('@', map { textrec($_) } element_nodes($node)); } | ||
| 959 | elsif ($tag eq 'ltx:XMArray') { | ||
| 960 | return textrec_array($node); } | ||
| 961 | else { | ||
| 962 | return '[' . (p_getValue($node) || '') . ']'; } } | ||
| 963 | |||
| 964 | sub textrec_apply { | ||
| 965 | no warnings 'recursion'; | ||
| 966 | my ($name, $op, @args) = @_; | ||
| 967 | my $role = ((ref $op ne 'ARRAY') && $op->getAttribute('role')) || 'Unknown'; | ||
| 968 | if (($role =~ /^(SUB|SUPER)SCRIPTOP$/) && (($op->getAttribute('scriptpos') || '') =~ /^pre\d+$/)) { | ||
| 969 | # Note that this will likely get parenthesized due to high bp | ||
| 970 | return (5000, textrec($op) . " " . textrec($args[1]) . " " . textrec($args[0])); } | ||
| 971 | elsif (my $bp = $IS_INFIX{$role}) { | ||
| 972 | # A sub/superscript with a meaning probably should be prefix | ||
| 973 | if (($role =~ /^(SUB|SUPER)SCRIPTOP$/) && $op->getAttribute('meaning')) { | ||
| 974 | return (500, textrec($op, 10000, $name) . '@(' . join(', ', map { textrec($_) } @args) . ')') } | ||
| 975 | else { # Format as infix. | ||
| 976 | return ($bp, (scalar(@args) == 1 # unless a single arg; then prefix. | ||
| 977 | ? textrec($op) . ' ' . textrec($args[0], $bp, $name) | ||
| 978 | : join(' ' . textrec($op) . ' ', map { textrec($_, $bp, $name) } @args))); } } | ||
| 979 | elsif ($role eq 'POSTFIX') { | ||
| 980 | return (10000, textrec($args[0], 10000, $name) . textrec($op)); } | ||
| 981 | elsif ($name eq 'multirelation') { | ||
| 982 | return (2, join(' ', map { textrec($_, 2, $name) } @args)); } | ||
| 983 | else { | ||
| 984 | return (500, textrec($op, 10000, $name) . '@(' . join(', ', map { textrec($_) } @args) . ')'); } } | ||
| 985 | |||
| 986 | sub textrec_array { | ||
| 987 | my ($node) = @_; | ||
| 988 | my $name = $node->getAttribute('meaning') || $node->getAttribute('name') || 'Array'; | ||
| 989 | my @rows = (); | ||
| 990 | foreach my $row (element_nodes($node)) { | ||
| 991 | push(@rows, '[' . join(', ', map { ($_->firstChild ? textrec($_->firstChild) : '') } element_nodes($row)) . ']'); } | ||
| 992 | return $name . '[' . join(', ', @rows) . ']'; } | ||
| 993 | |||
| 994 | sub is_genuinely_unparsed { | ||
| 995 | my ($node, $document) = @_; | ||
| 996 | # any unparsed fragment should be considered legitimate with one exception | ||
| 997 | # author-provided ungrammatical snippets in the presentation branches of XMDual | ||
| 998 | # are allowed to fail the parse process. | ||
| 999 | # | ||
| 1000 | # For now a reliable way of if we are in that case is to descend the formula through | ||
| 1001 | # the content branch of XMDual and check if any node has an "unparsed" mark. | ||
| 1002 | # Then only genuine parse failures will be detected. | ||
| 1003 | my $tag = $document->getNodeQName($node); | ||
| 1004 | if (($tag eq 'ltx:XMWrap') || ($tag eq 'ltx:XMArg') || $node->hasAttribute('_unparsed')) { | ||
| 1005 | return 1; } | ||
| 1006 | elsif (($tag eq 'ltx:XMTok') || ($tag eq 'ltx:XMText') || ($tag eq 'ltx:XMHint')) { | ||
| 1007 | return 0; } | ||
| 1008 | elsif ($tag eq 'ltx:XMRef') { | ||
| 1009 | # avoid infinite loops on malformed XMRefs that don't point anywhere | ||
| 1010 | if (!$node->getAttribute('idref')) { | ||
| 1011 | return 1; } | ||
| 1012 | else { | ||
| 1013 | my $real_node = realizeXMNode($node); | ||
| 1014 | # Note that some parses fail with an ARRAY ref [ltx:Error,...] | ||
| 1015 | # so realizeXMNode may fail! | ||
| 1016 | return 1 if (ref $real_node) !~ /^XML::LibXML/; | ||
| 1017 | return is_genuinely_unparsed($real_node, $document); } } | ||
| 1018 | elsif ($tag eq 'ltx:XMDual') { | ||
| 1019 | my ($content, $presentation) = element_nodes($node); | ||
| 1020 | return is_genuinely_unparsed($content, $document); } | ||
| 1021 | else { | ||
| 1022 | foreach my $child (element_nodes($node)) { | ||
| 1023 | return 1 if is_genuinely_unparsed($child, $document); } | ||
| 1024 | return 0; } } | ||
| 1025 | |||
| 1026 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 1027 | # Cute! Were it NOT for Sub/Superscripts, the whole parsing process only | ||
| 1028 | # builds a new superstructure around the sequence of token nodes in the input. | ||
| 1029 | # Thus, any internal structure is unchanged. | ||
| 1030 | # They get re-parented, but if the parse fails, we've only got to put them | ||
| 1031 | # BACK into the original node, to recover the original arrangment!!! | ||
| 1032 | # Thus, we don't have to clone, and deal with namespace duplication. | ||
| 1033 | # ... | ||
| 1034 | # EXCEPT, as I said, for sub/superscripts!!!! | ||
| 1035 | # | ||
| 1036 | |||
| 1037 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 1038 | # Constructors used in grammar | ||
| 1039 | # All the tree construction in the grammar should come through these operations. | ||
| 1040 | # We avoid mucking with the actual XML nodes (both to avoid modifying the original | ||
| 1041 | # tree until we have a successful parse, and to avoid XML::LibXML cloning nightmares) | ||
| 1042 | # We are converting XML nodes to array representation: [$tag, {%attr},@children] | ||
| 1043 | # This means any inspection of nodes has to recognize that | ||
| 1044 | # * node may be in XML vs ARRAY representation | ||
| 1045 | # * node may be an XMRef to another node whose properties are the ones we should use. | ||
| 1046 | # | ||
| 1047 | # Also, when we are examining a node's properties (roles, fences, script positioning, etc) | ||
| 1048 | # we should be careful to check for XMRef indirection and examine the properties | ||
| 1049 | # of the node that was referred to. | ||
| 1050 | # HOWEVER, we should construct our parse tree using (a clone of) the XMRef node, | ||
| 1051 | # rather than (a clone of) the referred to node, so as to preserve identity. | ||
| 1052 | #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% | ||
| 1053 | # We're currently keeping the id's on the nodes as they get cloned, | ||
| 1054 | # since they'll (maybe) replace the unparsed nodes. | ||
| 1055 | # However, if we consider multiple parses or preserving both parsed & unparsed, | ||
| 1056 | # we may have to do some adaptation and id shifting. | ||
| 1057 | # ================================================================================ | ||
| 1058 | |||
| 1059 | # ================================================================================ | ||
| 1060 | # Low-level accessors | ||
| 1061 | sub Lookup { | ||
| 1062 | my ($lexeme) = @_; | ||
| 1063 | return $$LaTeXML::MathParser::LEXEMES{$lexeme}; } | ||
| 1064 | |||
| 1065 | # The following accessors work on both the LibXML and ARRAY representations | ||
| 1066 | # but they do NOT automatically dereference XMRef! | ||
| 1067 | sub p_getValue { | ||
| 1068 | my ($node) = @_; | ||
| 1069 | if (!defined $node) { | ||
| 1070 | return; } | ||
| 1071 | elsif (ref $node eq 'XML::LibXML::Element') { | ||
| 1072 | my $x; | ||
| 1073 | return (($x = $node->textContent) ne '' ? $x # get content, or fall back to name | ||
| 1074 | : (defined($x = $node->getAttribute('name')) ? $x | ||
| 1075 | : undef)); } | ||
| 1076 | elsif (ref $node eq 'ARRAY') { | ||
| 1077 | my ($op, $attr, @args) = @$node; | ||
| 1078 | if (@args) { | ||
| 1079 | return join('', grep { defined $_ } map { p_getValue($_) } @args); } | ||
| 1080 | else { | ||
| 1081 | return $$node[1]{name}; } } | ||
| 1082 | elsif (ref $node eq 'XML::LibXML::Text') { | ||
| 1083 | return $node->textContent; } | ||
| 1084 | else { # ???? | ||
| 1085 | return $node; } } | ||
| 1086 | |||
| 1087 | sub p_getTokenMeaning { | ||
| 1088 | my ($item) = @_; | ||
| 1089 | if (!defined $item) { | ||
| 1090 | return; } | ||
| 1091 | elsif (ref $item eq 'ARRAY') { | ||
| 1092 | my ($op, $attr, @args) = @$item; | ||
| 1093 | return $$attr{meaning} || $$attr{name} || $args[0] || $$attr{role}; } | ||
| 1094 | elsif (ref $item eq 'XML::LibXML::Element') { | ||
| 1095 | return getTokenMeaning($item); } } | ||
| 1096 | |||
| 1097 | sub p_getAttribute { | ||
| 1098 | my ($item, $key) = @_; | ||
| 1099 | if (!defined $item) { | ||
| 1100 | return; } | ||
| 1101 | elsif (ref $item eq 'ARRAY') { | ||
| 1102 | return $$item[1]{$key}; } | ||
| 1103 | elsif (ref $item eq 'XML::LibXML::Element') { | ||
| 1104 | return $item->getAttribute($key); } } | ||
| 1105 | |||
| 1106 | sub p_setAttribute { | ||
| 1107 | my ($item, $key, $value) = @_; | ||
| 1108 | if (!defined $item) { | ||
| 1109 | return; } | ||
| 1110 | elsif (ref $item eq 'ARRAY') { | ||
| 1111 | return $$item[1]{$key} = $value; } | ||
| 1112 | elsif (ref $item eq 'XML::LibXML::Element') { | ||
| 1113 | return $item->setAttribute($key, $value); } } | ||
| 1114 | |||
| 1115 | sub p_element_nodes { | ||
| 1116 | my ($item) = @_; | ||
| 1117 | if (!defined $item) { | ||
| 1118 | return (); } | ||
| 1119 | elsif (ref $item eq 'ARRAY') { | ||
| 1120 | my ($op, $attr, @args) = @$item; | ||
| 1121 | return grep { ref $_ } @args; } | ||
| 1122 | elsif (ref $item eq 'XML::LibXML::Element') { | ||
| 1123 | return element_nodes($item); } } | ||
| 1124 | |||
| 1125 | sub p_getQName { | ||
| 1126 | my ($item) = @_; | ||
| 1127 | if (!defined $item) { | ||
| 1128 | return; } | ||
| 1129 | elsif (ref $item eq 'ARRAY') { | ||
| 1130 | return $$item[0]; } | ||
| 1131 | elsif (ref $item eq 'XML::LibXML::Element') { | ||
| 1132 | return getQName($item); } } | ||
| 1133 | |||
| 1134 | # Make a new Token node with given name, content, and attributes. | ||
| 1135 | # $content is an array of nodes (which may need to be cloned if still attached) | ||
| 1136 | sub New { | ||
| 1137 | my ($meaning, $content, %attributes) = @_; | ||
| 1138 | my %attr = (); | ||
| 1139 | $attr{meaning} = $meaning if $meaning; | ||
| 1140 | foreach my $key (sort keys %attributes) { | ||
| 1141 | my $value = p_getValue($attributes{$key}); | ||
| 1142 | $attr{$key} = $value if defined $value; } | ||
| 1143 | if (!$attr{font}) { | ||
| 1144 | $attr{font} = ($content && $content =~ /\S/ | ||
| 1145 | ? LaTeXML::Common::Font->textDefault->specialize($content) | ||
| 1146 | : LaTeXML::Common::Font->new()); } | ||
| 1147 | return ['ltx:XMTok', {%attr}, ($content ? ($content) : ())]; } | ||
| 1148 | |||
| 1149 | # Like New(), but concatenating several symbols into one new one. | ||
| 1150 | # This accounts for the symbols "disappearing", being replaced by one, | ||
| 1151 | # and thus any references to the old one have to be adjusted ... or removed | ||
| 1152 | # I'm a little concerned that we should be defering this till a successful parse? | ||
| 1153 | sub CatSymbols { | ||
| 1154 | my ($symbol1, $symbol2, $meaning, $content, %attributes) = @_; | ||
| 1155 | my $new = New($meaning, $content, %attributes); | ||
| 1156 | my $repl; | ||
| 1157 | my $symbols = [$symbol1, $symbol2]; | ||
| 1158 | my $doc = $LaTeXML::MathParser::DOCUMENT; | ||
| 1159 | foreach my $symbol (@$symbols) { | ||
| 1160 | if (my $id = p_getAttribute(realizeXMNode($symbol), 'xml:id')) { | ||
| 1161 | if (!$repl) { | ||
| 1162 | my $newid = $doc->modifyID($id); | ||
| 1163 | $$new[1]{'xml:id'} = $newid; | ||
| 1164 | ReplacedBy($symbol, $new); | ||
| 1165 | $repl = 1; } | ||
| 1166 | else { | ||
| 1167 | foreach my $r ($doc->findnodes("descendant-or-self::ltx:XMRef[\@idref='$id']")) { | ||
| 1168 | $doc->removeNode($r); } } # ? Hopefully this is safe. | ||
| 1169 | } } | ||
| 1170 | return $new; } | ||
| 1171 | |||
| 1172 | # Some handy shorthands. | ||
| 1173 | sub Absent { | ||
| 1174 | return New('absent'); } | ||
| 1175 | |||
| 1176 | sub InvisibleTimes { | ||
| 1177 | return New('times', "\x{2062}", role => 'MULOP'); } | ||
| 1178 | |||
| 1179 | sub InvisibleComma { | ||
| 1180 | return New(undef, "\x{2063}", role => 'PUNCT'); } | ||
| 1181 | |||
| 1182 | # Get n-th arg of an XMApp. | ||
| 1183 | # However, this is really only used to get the script out of a sub/super script | ||
| 1184 | sub Arg { | ||
| 1185 | my ($node, $n) = @_; | ||
| 1186 | my $onode = $node; | ||
| 1187 | $node = realizeXMNode($node); | ||
| 1188 | if (ref $node eq 'ARRAY') { | ||
| 1189 | return $$node[$n + 2]; } | ||
| 1190 | else { | ||
| 1191 | my @args = element_nodes($node); | ||
| 1192 | my $nth = $args[$n]; | ||
| 1193 | # Tricky case: if $node is an XMRef, we'll want to reference the SUB node too | ||
| 1194 | # and not just use it directly; else that node will be duplicated in both branches of XMDual | ||
| 1195 | if ($nth && !$node->isSameNode($onode)) { | ||
| 1196 | return LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $nth); } | ||
| 1197 | else { | ||
| 1198 | return $args[$n]; } } } # will get cloned if/when needed. | ||
| 1199 | |||
| 1200 | # Add more attributes to a node. | ||
| 1201 | # Values can be strings or nodes whose text content is used. | ||
| 1202 | # Note that it avoids changing the underlying XML, so it may return a new object! | ||
| 1203 | sub Annotate { | ||
| 1204 | my ($node, %attributes) = @_; | ||
| 1205 | my %attrib = (); | ||
| 1206 | # first scan & convert any attributes, to make sure there really are any values. | ||
| 1207 | foreach my $attr (keys %attributes) { | ||
| 1208 | my $value = p_getValue($attributes{$attr}); | ||
| 1209 | $attrib{$attr} = $value if defined $value; } | ||
| 1210 | # Hmm.... maybe we need to merge some things, open close? | ||
| 1211 | if (keys %attrib) { # Any attributes to assign? | ||
| 1212 | # If we've gotten a real XML node, convert to array representation | ||
| 1213 | # We do NOT want to modify any of the original XML!!!! | ||
| 1214 | if (ref $node ne 'ARRAY') { | ||
| 1215 | $node = [getQName($node), | ||
| 1216 | { map { (getQName($_) => $_->getValue) } | ||
| 1217 | grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes }, | ||
| 1218 | $node->childNodes]; } | ||
| 1219 | my $qname = $$node[0]; | ||
| 1220 | # Remove any attributes that aren't allowed!!! | ||
| 1221 | foreach my $k (keys %attrib) { | ||
| 1222 | delete $attrib{$k} unless $k =~ /^_/ || $LaTeXML::MathParser::DOCUMENT->canHaveAttribute($qname, $k); } | ||
| 1223 | # Make sure font is "Appropriate", if we're creating a new token | ||
| 1224 | if ($attrib{_font} && ($qname eq 'ltx:XMTok')) { | ||
| 1225 | my $content = join('', @$node[2 .. $#$node]); | ||
| 1226 | if ((!defined $content) || ($content eq '')) { | ||
| 1227 | delete $attrib{_font}; } # No font needed | ||
| 1228 | elsif (my $font = $LaTeXML::MathParser::DOCUMENT->decodeFont($attrib{_font})) { | ||
| 1229 | delete $attrib{_font}; | ||
| 1230 | $attrib{font} = $font->specialize($content); } } | ||
| 1231 | map { $$node[1]{$_} = $attrib{$_} } keys %attrib; } # Now add them. | ||
| 1232 | return $node; } | ||
| 1233 | |||
| 1234 | sub UTF { | ||
| 1235 | my ($code) = @_; | ||
| 1236 | return pack('U', $code); } | ||
| 1237 | |||
| 1238 | # Convert a token with role=VERTBAR to a different role, | ||
| 1239 | # possibly changing the content to the appropriate Unicode. | ||
| 1240 | my %vertbar_category = ( | ||
| 1241 | OPEN => 'delimiter', CLOSE => 'delimiter', MIDDLE => 'delimiter', | ||
| 1242 | PUNCT => 'operator'); | ||
| 1243 | my %vertbar_glyph = ( | ||
| 1244 | delimiter => { '|' => UTF(0x7C), '||' => "\x{2016}", '|||' => "\x{2980}", | ||
| 1245 | "\x{2225}" => "\x{2016}" }, | ||
| 1246 | operator => { '|' => "\x{2223}", '||' => "\x{2225}", '|||' => "\x{2AF4}", | ||
| 1247 | "\x{2016}" => "\x{2225}" }, | ||
| 1248 | ); | ||
| 1249 | |||
| 1250 | sub MorphVertbar { | ||
| 1251 | my ($node, $role) = @_; | ||
| 1252 | #return $node; | ||
| 1253 | my $rnode = realizeXMNode($node); | ||
| 1254 | my $oldrole = p_getAttribute($rnode, 'role'); | ||
| 1255 | my $oldchar = $rnode->textContent; | ||
| 1256 | my $category = $vertbar_category{$role}; | ||
| 1257 | my $char = ($oldchar && $category && $vertbar_glyph{$category}{$oldchar}) || $oldchar; | ||
| 1258 | my $newnode = $node; | ||
| 1259 | # Need to modify the token. | ||
| 1260 | if ($category && (($char ne $oldchar) || ($role ne $oldrole))) { | ||
| 1261 | # Collect up the attributes | ||
| 1262 | my %attrib = (); | ||
| 1263 | if (ref $node eq 'ARRAY') { | ||
| 1264 | %attrib = %{ $$node[2] }; } | ||
| 1265 | else { | ||
| 1266 | %attrib = map { (getQName($_) => $_->getValue) } | ||
| 1267 | grep { $_->nodeType == XML_ATTRIBUTE_NODE } $node->attributes; } | ||
| 1268 | $attrib{role} = $role; | ||
| 1269 | $newnode = [getQName($node), {%attrib}, $char]; } | ||
| 1270 | return $newnode; } | ||
| 1271 | |||
| 1272 | # ================================================================================ | ||
| 1273 | # Mid-level constructors | ||
| 1274 | |||
| 1275 | # Apply $op to the list of arguments | ||
| 1276 | # args may be array-rep or lexemes (or nodes?) | ||
| 1277 | sub Apply { | ||
| 1278 | my ($op, @args) = @_; | ||
| 1279 | my $font = p_getAttribute($op, '_font'); | ||
| 1280 | return ['ltx:XMApp', { ($font ? (_font => $font) : ()) }, $op, @args]; } | ||
| 1281 | |||
| 1282 | # Apply $op to a `delimited' list of arguments of the form | ||
| 1283 | # open, expr (punct expr)* close | ||
| 1284 | # after extracting the opening and closing delimiters, and the separating punctuation | ||
| 1285 | # Generate an XMDual, so that any styling of delimiters & punctuation is preserved. | ||
| 1286 | sub ApplyDelimited { | ||
| 1287 | my ($op, @stuff) = @_; | ||
| 1288 | my $open = $stuff[0]; | ||
| 1289 | my $close = $stuff[-1]; | ||
| 1290 | my ($seps, @args) = extract_separators(@stuff[1 .. $#stuff - 1]); | ||
| 1291 | return ['ltx:XMDual', {}, | ||
| 1292 | Apply(LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $op), | ||
| 1293 | LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @args)), | ||
| 1294 | Apply($op, ['ltx:XMWrap', {}, @stuff])]; } | ||
| 1295 | |||
| 1296 | # This is similar, but "interprets" a delimited list as being the | ||
| 1297 | # application of some operator to the items in the list. | ||
| 1298 | sub InterpretDelimited { | ||
| 1299 | my ($op, @stuff) = @_; | ||
| 1300 | my $open = $stuff[0]; | ||
| 1301 | my $close = $stuff[-1]; | ||
| 1302 | my ($seps, @args) = extract_separators(@stuff[1 .. $#stuff - 1]); | ||
| 1303 | return ['ltx:XMDual', {}, | ||
| 1304 | Apply($op, LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @args)), | ||
| 1305 | ['ltx:XMWrap', {}, @stuff]]; } | ||
| 1306 | |||
| 1307 | # Given a sequence of operators, form the nested application op(op(...(arg))) | ||
| 1308 | sub recApply { | ||
| 1309 | my (@ops) = @_; | ||
| 1310 | return (scalar(@ops) > 1 ? Apply(shift(@ops), recApply(@ops)) : $ops[0]); } | ||
| 1311 | |||
| 1312 | # Given alternating expressions & separators (punctuation,...) | ||
| 1313 | # extract the separators as a concatenated string, | ||
| 1314 | # returning (separators, args...) | ||
| 1315 | # But note that the separators are never used for anything!? | ||
| 1316 | sub extract_separators { | ||
| 1317 | my (@stuff) = @_; | ||
| 1318 | my ($punct, @args); | ||
| 1319 | if (@stuff) { | ||
| 1320 | push(@args, shift(@stuff)); # Grab 1st expression | ||
| 1321 | while (@stuff) { # Expecting pairs of punct, expression | ||
| 1322 | my $p = realizeXMNode(shift(@stuff)); | ||
| 1323 | $punct .= | ||
| 1324 | ($punct ? ' ' : '') # Delimited by SINGLE SPACE! | ||
| 1325 | . spacingToString(getXMHintSpacing(p_getAttribute($p, 'lpadding'))) | ||
| 1326 | . (p_getValue($p) // ' ') | ||
| 1327 | . spacingToString(getXMHintSpacing(p_getAttribute($p, 'rpadding'))); | ||
| 1328 | push(@args, shift(@stuff)); } } # Collect the next expression. | ||
| 1329 | @args = grep { $_ } @args; # drop all undef args, trailing punct could have added an undef | ||
| 1330 | return ($punct, @args); } | ||
| 1331 | |||
| 1332 | # ================================================================================ | ||
| 1333 | # Some special cases | ||
| 1334 | |||
| 1335 | # This specifies the "meaning" of things within a pair | ||
| 1336 | # of open/close delimiters, depending on the number of things. | ||
| 1337 | # Really should be customizable? | ||
| 1338 | # Note that these are all Best Guesses, but really can have | ||
| 1339 | # alternate interpretations depending on context, field, etc. | ||
| 1340 | # Question: Is there enough context to guess better? | ||
| 1341 | # For example, whether (a,b) is an interval or list? | ||
| 1342 | # (both could reasonably be preceded by \in ) | ||
| 1343 | my %balanced = ( # [CONSTANT] | ||
| 1344 | '(' => ')', '[' => ']', '{' => '}', | ||
| 1345 | '|' => '|', '||' => '||', "\x{2016}" => "\x{2016}", "\x{2980}" => "\x{2980}", | ||
| 1346 | "\x{230A}" => "\x{230B}", # lfloor, rfloor | ||
| 1347 | "\x{2308}" => "\x{2309}", # lceil, rceil | ||
| 1348 | "\x{2329}" => "\x{232A}", # angle brackets; NOT mathematical, but balance in case they show up. | ||
| 1349 | "\x{27E8}" => "\x{27E9}", # angle brackets (preferred) | ||
| 1350 | "\x{2225}" => "\x{2225}", # lVert, rVert | ||
| 1351 | ); | ||
| 1352 | # For enclosing a single object | ||
| 1353 | # Note that the default here is just to put open/closed attributes on the single object | ||
| 1354 | my %enclose1 = ( # [CONSTANT] | ||
| 1355 | '{@}' => 'set', # alternatively, just variant parentheses | ||
| 1356 | '|@|' => 'absolute-value', | ||
| 1357 | '||@||' => 'norm', "\x{2016}@\x{2016}" => 'norm', | ||
| 1358 | "\x{230A}@\x{230B}" => 'floor', | ||
| 1359 | "\x{2308}@\x{2309}" => 'ceiling', | ||
| 1360 | '<@>' => 'expectation', # or just average? | ||
| 1361 | '<@|' => 'bra', '|@>' => 'ket'); | ||
| 1362 | # For enclosing more than 2 objects; the punctuation is significant too | ||
| 1363 | my %enclose2 = ( # [CONSTANT] | ||
| 1364 | '(@,@)' => 'open-interval', # alternatively, just a list | ||
| 1365 | '[@,@]' => 'closed-interval', | ||
| 1366 | '(@,@]' => 'open-closed-interval', '[@,@)' => 'closed-open-interval', | ||
| 1367 | '{@,@}' => 'set', # alternatively, just a list ? | ||
| 1368 | ); | ||
| 1369 | # For enclosing more than 2 objects. | ||
| 1370 | # assume 1st punct? or should we check all are same? | ||
| 1371 | my %encloseN = ( # [CONSTANT] | ||
| 1372 | '(@,@)' => 'vector', '{@,@}' => 'set',); | ||
| 1373 | |||
| 1374 | sub isMatchingClose { | ||
| 1375 | my ($open, $close) = @_; | ||
| 1376 | my $oname = p_getValue(realizeXMNode($open)); | ||
| 1377 | my $cname = p_getValue(realizeXMNode($close)); | ||
| 1378 | my $expect = $oname && $balanced{$oname}; | ||
| 1379 | return (defined $expect) && (defined $cname) && ($expect eq $cname); } | ||
| 1380 | |||
| 1381 | # Given a delimited sequence: open expr (punct expr)* close | ||
| 1382 | # (OR, an empty sequence open close) | ||
| 1383 | # Convert it into the appropriate thing, depending on the specific open & close used. | ||
| 1384 | # Generate an XMDual to preserve any styling of delimiters and punctuation. | ||
| 1385 | sub Fence { | ||
| 1386 | my (@stuff) = @_; | ||
| 1387 | # Peak at delimiters to guess what kind of construct this is. | ||
| 1388 | my $nargs = scalar(@stuff); | ||
| 1389 | Error("expected", "arguments", undef, | ||
| 1390 | "Even number of arguments to Fence(); should be of form open,expr,(punct,expr)*,close", | ||
| 1391 | "got " . join(' ', map { ToString($_) } @stuff)) | ||
| 1392 | if ($nargs != 2) && (($nargs % 2) == 0); # either empty or odd number | ||
| 1393 | my ($open, $close) = (realizeXMNode($stuff[0]), realizeXMNode($stuff[-1])); | ||
| 1394 | my $o = p_getValue($open); | ||
| 1395 | my $c = p_getValue($close); | ||
| 1396 | my $n = int(($nargs - 2 + 1) / 2); | ||
| 1397 | my @p = map { p_getValue(realizeXMNode(@stuff[2 * $_])) } 1 .. $n - 1; | ||
| 1398 | my $op = ($n == 0 | ||
| 1399 | ? 'list' # ? | ||
| 1400 | : ($n == 1 | ||
| 1401 | ? $enclose1{ $o . '@' . $c } | ||
| 1402 | : ($n == 2 | ||
| 1403 | ? ($enclose2{ $o . '@' . $p[0] . '@' . $c } || 'list') | ||
| 1404 | : ($encloseN{ $o . '@' . $p[0] . '@' . $c } || 'list')))); | ||
| 1405 | $op = 'delimited-' . $o . $c unless defined $op; | ||
| 1406 | my $decl_id = p_getAttribute($open, 'decl_id'); | ||
| 1407 | if (($n == 1) && ($op eq 'delimited-()')) { # Hopefully, can just ignore the parens? | ||
| 1408 | return ['ltx:XMDual', {}, | ||
| 1409 | LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, $stuff[1]), | ||
| 1410 | ['ltx:XMWrap', {}, @stuff]]; } | ||
| 1411 | else { | ||
| 1412 | return InterpretDelimited(New($op, undef, ($decl_id ? (decl_id => $decl_id) : ())), @stuff); } } | ||
| 1413 | |||
| 1414 | # Compose a complex relational operator from two tokens, such as >=, >> | ||
| 1415 | # (similar to CatSymbols, but specialized to relops) | ||
| 1416 | sub TwoPartRelop { | ||
| 1417 | my ($op1, $op2) = @_; | ||
| 1418 | $op1 = Lookup($op1); | ||
| 1419 | $op2 = Lookup($op2); | ||
| 1420 | my $m1 = p_getTokenMeaning($op1); | ||
| 1421 | my $m2 = p_getTokenMeaning($op2); | ||
| 1422 | my $font = p_getAttribute($op1, '_font'); | ||
| 1423 | my $meaning; | ||
| 1424 | if ($m1 eq $m2) { | ||
| 1425 | $meaning = "much-$m1"; } | ||
| 1426 | else { | ||
| 1427 | $meaning = "$m1-or-$m2"; } | ||
| 1428 | my $content = $op1->textContent . $op2->textContent; | ||
| 1429 | return ['ltx:XMTok', { role => "RELOP", meaning => $meaning, ($font ? (_font => $font) : ()) }, | ||
| 1430 | $content]; } | ||
| 1431 | |||
| 1432 | # NOTE: It might be best to separate the multiple Formulae into separate XMath's??? | ||
| 1433 | # but only at the top level! | ||
| 1434 | sub NewFormulae { | ||
| 1435 | my (@stuff) = @_; | ||
| 1436 | if (scalar(@stuff) == 1) { | ||
| 1437 | return $stuff[0]; } | ||
| 1438 | else { | ||
| 1439 | my ($seps, @formula) = extract_separators(@stuff); | ||
| 1440 | return ['ltx:XMDual', {}, | ||
| 1441 | Apply(New('formulae'), | ||
| 1442 | LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @formula)), | ||
| 1443 | ['ltx:XMWrap', {}, @stuff]]; } } | ||
| 1444 | |||
| 1445 | # A Formula is an alternation of expr (relationalop expr)* | ||
| 1446 | # It presumably would be equivalent to (expr1 relop1 expr2) AND (expr2 relop2 expr3) ... | ||
| 1447 | # But, I haven't figured out the ideal prefix form that can easily be converted to presentation. | ||
| 1448 | sub NewFormula { | ||
| 1449 | my (@args) = @_; | ||
| 1450 | my $n = scalar(@args); | ||
| 1451 | if ($n == 1) { | ||
| 1452 | return $args[0]; } | ||
| 1453 | elsif ($n == 3) { | ||
| 1454 | return Apply($args[1], $args[0], $args[2]); } | ||
| 1455 | else { | ||
| 1456 | return Apply(New('multirelation'), @args); } } | ||
| 1457 | |||
| 1458 | sub NewList { | ||
| 1459 | my (@stuff) = @_; | ||
| 1460 | # drop placeholder token for missing trailing punct, if any | ||
| 1461 | if (scalar(@stuff) > 1 && ((p_getTokenMeaning($stuff[-1]) || '') eq 'absent')) { | ||
| 1462 | pop(@stuff); } | ||
| 1463 | if (@stuff == 1) { | ||
| 1464 | return $stuff[0]; } | ||
| 1465 | else { | ||
| 1466 | my ($seps, @items) = extract_separators(@stuff); | ||
| 1467 | return ['ltx:XMDual', {}, | ||
| 1468 | Apply(New('list'), | ||
| 1469 | LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, @items)), | ||
| 1470 | ['ltx:XMWrap', {}, @stuff]]; } } | ||
| 1471 | |||
| 1472 | # Given alternation of expr (addop expr)*, compose the tree (left recursive), | ||
| 1473 | # flattenning portions that have the same operator | ||
| 1474 | # ie. a + b + c - d => (- (+ a b c) d) | ||
| 1475 | sub LeftRec { | ||
| 1476 | no warnings 'recursion'; | ||
| 1477 | my ($arg1, @more) = @_; | ||
| 1478 | if (@more) { | ||
| 1479 | my $op = shift(@more); | ||
| 1480 | my $opname = p_getTokenMeaning(realizeXMNode($op)); | ||
| 1481 | my @args = ($arg1, shift(@more)); | ||
| 1482 | while (@more && isSameExpr($op, $more[0])) { | ||
| 1483 | ReplacedBy($more[0], $op, 1); | ||
| 1484 | shift(@more); | ||
| 1485 | push(@args, shift(@more)); } | ||
| 1486 | return LeftRec(Apply($op, @args), @more); } | ||
| 1487 | else { | ||
| 1488 | return $arg1; } } | ||
| 1489 | |||
| 1490 | # Like apply($op,$arg1,$arg2), but if $op is 'same' as the operator in $arg1, | ||
| 1491 | # then combine as an nary apply of $op to $arg1's arguments and $arg2. | ||
| 1492 | sub ApplyNary { | ||
| 1493 | my ($op, $arg1, $arg2) = @_; | ||
| 1494 | my $rop = realizeXMNode($op); | ||
| 1495 | my $opname = p_getTokenMeaning($rop) || '__undef_meaning__'; | ||
| 1496 | my $opcontent = p_getValue($rop) || '__undef_content__'; | ||
| 1497 | my @args = (); | ||
| 1498 | if (p_getQName($arg1) eq 'ltx:XMApp') { | ||
| 1499 | my ($op1, @args1) = p_element_nodes($arg1); | ||
| 1500 | my $rop1 = realizeXMNode($op1); | ||
| 1501 | if (isSameExpr($rop, $rop1) | ||
| 1502 | # Check that arg1 isn't wrapped, fenced or enclosed in some restrictive way | ||
| 1503 | # Especially an ID! (but really only important if the id is referenced somewhere?) | ||
| 1504 | && !(grep { p_getAttribute(realizeXMNode($arg1), $_) } qw(enclose xml:id))) { | ||
| 1505 | # Note that $op1 GOES AWAY!!! | ||
| 1506 | ReplacedBy($op1, $rop, 1); | ||
| 1507 | push(@args, @args1); } | ||
| 1508 | else { | ||
| 1509 | push(@args, $arg1); } } | ||
| 1510 | else { | ||
| 1511 | push(@args, $arg1); } | ||
| 1512 | return Apply($op, @args, $arg2); } | ||
| 1513 | |||
| 1514 | # Usually we just expect to compare a token + to another. | ||
| 1515 | # but want (to some extent) to deal with embellished operators (eg. sub, sup...) | ||
| 1516 | # Rather involved if we want to try to do it "Right". | ||
| 1517 | sub isSameExpr { | ||
| 1518 | my ($op1, $op2) = @_; | ||
| 1519 | $op1 = realizeXMNode($op1); | ||
| 1520 | $op2 = realizeXMNode($op2); | ||
| 1521 | my $tag1 = getQName($op1); | ||
| 1522 | my $tag2 = getQName($op2); | ||
| 1523 | # Either both are tokens, | ||
| 1524 | # OR both have structure, but then we need to compare the children!!!! | ||
| 1525 | # First check, same top-level and critical attributes | ||
| 1526 | return unless | ||
| 1527 | ($tag1 eq $tag2) | ||
| 1528 | && ((p_getTokenMeaning($op1) || '__undef_meaning__') | ||
| 1529 | eq (p_getTokenMeaning($op2) || '__undef_meaning__')) | ||
| 1530 | && ((p_getValue($op1) || '__undef_content__') | ||
| 1531 | eq (p_getValue($op2) || '__undef_content__')) | ||
| 1532 | # Check that ops are used in same way. | ||
| 1533 | && ((p_getAttribute($op1, 'mathstyle') || '<none>') | ||
| 1534 | eq (p_getAttribute($op2, 'mathstyle') || '<none>')); | ||
| 1535 | if ($tag1 eq 'ltx:XMTok') { return 1; } # If tokens, they match | ||
| 1536 | else { | ||
| 1537 | my @ch1 = p_element_nodes($op1); | ||
| 1538 | my @ch2 = p_element_nodes($op2); | ||
| 1539 | my $n = scalar(@ch1); | ||
| 1540 | return unless $n == scalar(@ch2); | ||
| 1541 | foreach my $i (0 .. $n - 1) { | ||
| 1542 | return unless isSameExpr($ch1[$i], $ch1[$i]); } | ||
| 1543 | return 1; } } | ||
| 1544 | |||
| 1545 | # There are several cases where parsing a formula will rearrange nodes | ||
| 1546 | # such that some nodes will no-longer be used. For example, when | ||
| 1547 | # converting a nested set of infix + into a single n-ary sum. | ||
| 1548 | # In effect, all those excess +'s are subsumed by the single first one. | ||
| 1549 | # It may be, however, that those lost nodes are referenced (XMRef) from the | ||
| 1550 | # other branch of an XMDual, and those references should be updated to refer | ||
| 1551 | # to the single node replacing the lost ones. | ||
| 1552 | # This function records that replacement, and the top-level parser fixes up the tree. | ||
| 1553 | # NOTE: There may be cases (in the Grammar, eg) where punctuation & ApplyOp's | ||
| 1554 | # get lost completely? Watch out for this! | ||
| 1555 | # If $isdup, assume the two trees are equivalent structure, | ||
| 1556 | # and so mark corresponding internal nodes as "replaced" as well. | ||
| 1557 | sub ReplacedBy { | ||
| 1558 | my ($lostnode, $keepnode, $isdup) = @_; | ||
| 1559 | return unless ref $lostnode && ref $keepnode; | ||
| 1560 | if (my $lostid = p_getAttribute($lostnode, 'xml:id')) { | ||
| 1561 | # Could be we want to generate an id for $keepnode, here? | ||
| 1562 | if (my $keepid = p_getAttribute($keepnode, 'xml:id')) { | ||
| 1563 | $$LaTeXML::MathParser::LOSTNODES{$lostid} = $keepid; } | ||
| 1564 | else { | ||
| 1565 | Warn('expected', 'id', undef, "LOST $lostid but no replacement!\n"); } } | ||
| 1566 | |||
| 1567 | # The following recurses into the two trees, | ||
| 1568 | # This is on the assumption that they are "equivalent" trees | ||
| 1569 | if ($isdup) { | ||
| 1570 | my @lostkids = p_element_nodes($lostnode); | ||
| 1571 | my @keepkids = p_element_nodes($keepnode); | ||
| 1572 | my $n = scalar(@lostkids); | ||
| 1573 | my $m = scalar(@keepkids); | ||
| 1574 | if ($n != $m) { | ||
| 1575 | Error("unexpected", "nodes", undef, "Nodes aren't the same structure ($n vs $m)", | ||
| 1576 | "Old: " . printNode($lostnode), | ||
| 1577 | "New: " . printNode($keepnode) | ||
| 1578 | ); | ||
| 1579 | $n = $m if $m < $n; } | ||
| 1580 | foreach my $i (0 .. $n - 1) { | ||
| 1581 | ReplacedBy($lostkids[$i], $keepkids[$i], $isdup); } } | ||
| 1582 | return; } | ||
| 1583 | |||
| 1584 | # ================================================================================ | ||
| 1585 | # Construct an appropriate application of sub/superscripts | ||
| 1586 | # This accounts for script positioning: | ||
| 1587 | # Whether it precedes (float), is over/under (if base requests), | ||
| 1588 | # or follows (normal case), along with whether sub/super. | ||
| 1589 | # the alignment of multiple sub/superscripts derived from the binding level when created. | ||
| 1590 | # scriptpos = (pre|mod|post) number; where number is the binding-level. | ||
| 1591 | # If $pos is given (pre|mid|post), it overrides the position implied by the script | ||
| 1592 | sub NewScript { | ||
| 1593 | my ($base, $script, $pos) = @_; | ||
| 1594 | my $role; | ||
| 1595 | my $rbase = realizeXMNode($base); | ||
| 1596 | my $rscript = realizeXMNode($script); | ||
| 1597 | my $ibase = $rbase; | ||
| 1598 | # Get "inner" (content) base, if the base is a dual it may be more relevant | ||
| 1599 | if (p_getQName($rbase) eq 'ltx:XMDual') { | ||
| 1600 | ($ibase) = p_element_nodes($rbase); } | ||
| 1601 | elsif (p_getQName($rbase) eq 'ltx:XMApp') { | ||
| 1602 | my ($op, $a1, @rest) = p_element_nodes($rbase); | ||
| 1603 | if ((p_getQName($op) eq 'ltx:XMTok') | ||
| 1604 | && ((p_getAttribute($op, 'role') || '') =~ /^(FLOAT|POST)?(SUB|SUPER)SCRIPT(OP)?$/)) { | ||
| 1605 | $ibase = $op; } } | ||
| 1606 | my @bsp = grep { $_ } (p_getAttribute($rbase, 'scriptpos'), | ||
| 1607 | p_getAttribute($rbase, 'scriptpos'), | ||
| 1608 | p_getAttribute($ibase, 'scriptpos'), | ||
| 1609 | 'post'); | ||
| 1610 | my ($bx) = grep { $_ } map { /^(pre|mid|post)?(\d+)?$/ && $1; } @bsp; | ||
| 1611 | my ($bl) = grep { $_ } map { /^(pre|mid|post)?(\d+)?$/ && $2; } @bsp; | ||
| 1612 | my ($sx, $sl) = (p_getAttribute($rscript, 'scriptpos') || 'post') =~ /^(pre|mid|post)?(\d+)?$/; | ||
| 1613 | $bl = ($sl || 1) unless $bl; | ||
| 1614 | $sl = ($bl || 1) unless $sl; | ||
| 1615 | $sx = ($bl == $sl ? $bx : 'post') unless $sx; | ||
| 1616 | my ($mode, $y) = p_getAttribute($rscript, 'role') =~ /^(FLOAT|POST)?(SUB|SUPER)SCRIPT$/; | ||
| 1617 | my $x = ($pos ? $pos : ($mode eq 'FLOAT' ? 'pre' | ||
| 1618 | : ($bl == $sl ? $bx : $sx) || 'post')); | ||
| 1619 | my $lpad = ($x eq 'pre') && p_getAttribute($rscript, 'lpadding'); | ||
| 1620 | my $rpad = ($x ne 'pre') && p_getAttribute($rscript, 'rpadding'); | ||
| 1621 | my $t; | ||
| 1622 | my $l = $sl || $bl || | ||
| 1623 | (($t = $LaTeXML::MathParser::DOCUMENT->getNodeBox($script)) | ||
| 1624 | && ($t->getProperty('scriptlevel'))) || 0; | ||
| 1625 | # If the INNER script was a floating script (ie. {}^{x}) | ||
| 1626 | # we'll NOT want this one to stack over it so bump the level. | ||
| 1627 | my $bumped; | ||
| 1628 | if (p_getAttribute($rbase, '_wasfloat')) { | ||
| 1629 | $l++; $bumped = 1 } | ||
| 1630 | elsif (my $innerl = p_getAttribute($rbase, '_bumplevel')) { | ||
| 1631 | $l = $innerl; } | ||
| 1632 | my $app = Apply(New(undef, undef, role => $y . 'SCRIPTOP', scriptpos => "$x$l"), | ||
| 1633 | $base, Arg($script, 0)); | ||
| 1634 | # Record whether this script was a floating one | ||
| 1635 | $$app[1]{_wasfloat} = 1 if $mode eq 'FLOAT'; | ||
| 1636 | $$app[1]{_bumplevel} = $l if $bumped; | ||
| 1637 | $$app[1]{lpadding} = $lpad if $lpad && !$$app[1]{lpadding}; # better to add? | ||
| 1638 | $$app[1]{rpadding} = $rpad if $rpad && !$$app[1]{rpadding}; # better to add? | ||
| 1639 | return $app; } | ||
| 1640 | |||
| 1641 | # Basically, like NewScript, but decorates an operator with sub/superscripts | ||
| 1642 | # (with vague unknown implications for meaning?) | ||
| 1643 | # but which will preserve the role (& meaning?) | ||
| 1644 | sub DecorateOperator { | ||
| 1645 | my ($op, $script) = @_; | ||
| 1646 | my $decop = NewScript($op, $script); | ||
| 1647 | my $rop = realizeXMNode($op); | ||
| 1648 | my $role = p_getAttribute($rop, 'role'); | ||
| 1649 | return Annotate($decop, role => $role); } | ||
| 1650 | |||
| 1651 | sub NewEvalAt { | ||
| 1652 | my ($base, $vertbar, $lower, $upper) = @_; | ||
| 1653 | my $pres = ['ltx:XMWrap', {}, $base, Annotate($vertbar, role => 'CLOSE')]; | ||
| 1654 | $pres = NewScript($pres, $lower) if $lower; | ||
| 1655 | $pres = NewScript($pres, $upper) if $upper; | ||
| 1656 | return ['ltx:XMDual', {}, | ||
| 1657 | ['ltx:XMApp', {}, | ||
| 1658 | ['ltx:XMTok', { meaning => 'evaluated-at' }], | ||
| 1659 | LaTeXML::Package::createXMRefs($LaTeXML::MathParser::DOCUMENT, | ||
| 1660 | $base, | ||
| 1661 | ($lower ? (Arg($lower, 0)) : ()), | ||
| 1662 | ($upper ? (Arg($upper, 0)) : ()))], | ||
| 1663 | $pres]; } | ||
| 1664 | |||
| 1665 | # ================================================================================ | ||
| 1666 | # A "notation" is a language construct or set thereof. | ||
| 1667 | |||
| 1668 | # Called from the grammar to record the fact that a notation was seen. | ||
| 1669 | sub SawNotation { | ||
| 1670 | my ($notation, $node) = @_; | ||
| 1671 | $LaTeXML::MathParser::SEEN_NOTATIONS{$notation} = 1; | ||
| 1672 | return 1; } | ||
| 1673 | |||
| 1674 | # Called by the grammar to determine whether we should try productions | ||
| 1675 | # which involve the given notation. | ||
| 1676 | sub IsNotationAllowed { | ||
| 1677 | my ($notation) = @_; | ||
| 1678 | return ($LaTeXML::MathParser::DISALLOWED_NOTATIONS{$notation} ? undef : 1); } | ||
| 1679 | |||
| 1680 | # ================================================================================ | ||
| 1681 | |||
| 1682 | # Note that an UNKNOWN token may have been used as a function. | ||
| 1683 | # For simplicity in the grammar, we accept a token that has sub|super scripts applied. | ||
| 1684 | sub MaybeFunction { | ||
| 1685 | my ($token) = @_; | ||
| 1686 | $token = realizeXMNode($token); | ||
| 1687 | my $self = $LaTeXML::MathParser::PARSER; | ||
| 1688 | while (p_getQName($token) eq 'ltx:XMApp') { | ||
| 1689 | $token = Arg($token, 1); } | ||
| 1690 | my $name = token_prettyname($token); | ||
| 1691 | # DANGER!! | ||
| 1692 | # We want to be using Annotate here, but we're screwed up by the | ||
| 1693 | # potential "embellishing" of the function token. | ||
| 1694 | # (ie. the descent above past all XMApp's) | ||
| 1695 | $token->setAttribute('possibleFunction', 'yes'); | ||
| 1696 | $$self{maybe_functions}{$name}++ | ||
| 1697 | if $LaTeXML::MathParser::STRICT && !$$self{suspicious_tokens}{$token}; | ||
| 1698 | $$self{suspicious_tokens}{$token} = 1; | ||
| 1699 | return; } | ||
| 1700 | |||
| 1701 | # ================================================================================ | ||
| 1702 | 1; | ||
| 1703 | |||
| 1704 | __END__ |