| 1 | # /=====================================================================\ # | ||
| 2 | # | LaTeXML | # | ||
| 3 | # | Overall LaTeXML Converter | # | ||
| 4 | # |=====================================================================| # | ||
| 5 | # | Part of LaTeXML: | # | ||
| 6 | # | Public domain software, produced as part of work done by the | # | ||
| 7 | # | United States Government & not subject to copyright in the US. | # | ||
| 8 | # |---------------------------------------------------------------------| # | ||
| 9 | # | Deyan Ginev <d.ginev@jacobs-university.de> #_# | # | ||
| 10 | # | http://dlmf.nist.gov/LaTeXML/ (o o) | # | ||
| 11 | # \=========================================================ooo==U==ooo=/ # | ||
| 12 | package LaTeXML; | ||
| 13 | 5 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 14 | 5 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 15 | 5 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 16 | 5 | ▶ | use Encode; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 17 | 5 | ▶ | use Data::Dumper; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 18 | 5 | ▶ | use File::Temp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 19 | File::Temp->safe_level(File::Temp::MEDIUM); | ||
| 20 | 5 | ▶ | use File::Path qw(rmtree); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 21 | 5 | ▶ | use File::Spec; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 22 | 5 | ▶ | use List::Util qw(max); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 23 | 5 | ▶ | use LaTeXML::Common::Config; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 24 | 5 | ▶ | use LaTeXML::Common::Error; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | 0 | ▶ | use LaTeXML::Core; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 26 | use LaTeXML::Util::Pack; | ||
| 27 | use LaTeXML::Util::Pathname; | ||
| 28 | use LaTeXML::Util::WWW; | ||
| 29 | use LaTeXML::Util::ObjectDB; | ||
| 30 | use LaTeXML::Post::Scan; | ||
| 31 | use vars qw($VERSION); | ||
| 32 | # This is the main version of LaTeXML being claimed. | ||
| 33 | use version; our $VERSION = version->declare("0.8.8"); | ||
| 34 | use LaTeXML::Version; | ||
| 35 | # Derived, more informative version numbers | ||
| 36 | our $FULLVERSION = "LaTeXML version $LaTeXML::VERSION" | ||
| 37 | . ($LaTeXML::Version::REVISION ? "; revision $LaTeXML::Version::REVISION" : ''); | ||
| 38 | # Handy identifier string for any executable. | ||
| 39 | our $IDENTITY = "$FindBin::Script ($LaTeXML::FULLVERSION)"; | ||
| 40 | |||
| 41 | our $LOG_STACK = 0; | ||
| 42 | |||
| 43 | #********************************************************************** | ||
| 44 | #our @IGNORABLE = qw(timeout profile port preamble postamble port destination log removed_math_formats whatsin whatsout math_formats input_limit input_counter dographics mathimagemag ); | ||
| 45 | |||
| 46 | # Switching to white-listing options that are important for new_latexml: | ||
| 47 | our @COMPARABLE = qw(preload paths verbosity strict comments inputencoding includestyles documentid mathparse); | ||
| 48 | |||
| 49 | sub new { | ||
| 50 | my ($class, $config) = @_; | ||
| 51 | $config = LaTeXML::Common::Config->new() unless (defined $config); | ||
| 52 | # The daemon should be setting the identity: | ||
| 53 | my $self = bless { opts => $config->options, ready => 0, log => q{}, runtime => {}, | ||
| 54 | latexml => undef }, $class; | ||
| 55 | # Special check if the debug directive is on, just to neutralize the bind_log | ||
| 56 | my $debug_directives = $$self{opts}->{debug}; | ||
| 57 | $LaTeXML::DEBUG{latexml} = 1 if (ref $debug_directives eq 'ARRAY') && (grep { /latexml/i } @$debug_directives); | ||
| 58 | eval { $config->check; }; | ||
| 59 | if ($@) { | ||
| 60 | $self->bind_log; | ||
| 61 | NoteLog($@); | ||
| 62 | $$self{log} .= $self->flush_log; } | ||
| 63 | |||
| 64 | return $self; } | ||
| 65 | |||
| 66 | sub prepare_session { | ||
| 67 | my ($self, $config) = @_; | ||
| 68 | # 1. Ensure option "sanity" | ||
| 69 | eval { $config->check; }; | ||
| 70 | if ($@) { | ||
| 71 | $self->bind_log; | ||
| 72 | NoteLog($@); | ||
| 73 | $$self{log} .= $self->flush_log; } | ||
| 74 | |||
| 75 | my $opts = $config->options; | ||
| 76 | my $opts_comparable = { map { $_ => $$opts{$_} } @COMPARABLE }; | ||
| 77 | my $self_opts_comparable = { map { $_ => $$self{opts}{$_} } @COMPARABLE }; | ||
| 78 | #TODO: Some options like paths and includes are additive, we need special treatment there | ||
| 79 | #2.2. Compare old and new $opts hash | ||
| 80 | my $something_to_do; | ||
| 81 | $something_to_do = LaTeXML::Util::ObjectDB::compare($opts_comparable, $self_opts_comparable) ? 0 : 1; | ||
| 82 | #2.3. Set new options in converter: | ||
| 83 | $$self{opts} = $opts; | ||
| 84 | |||
| 85 | #3. If there is something to do, initialize a session: | ||
| 86 | $self->initialize_session if ($something_to_do || (!$$self{ready})); | ||
| 87 | |||
| 88 | return; | ||
| 89 | } | ||
| 90 | |||
| 91 | sub initialize_session { | ||
| 92 | my ($self) = @_; | ||
| 93 | $$self{runtime} = {} unless $$self{runtime}; | ||
| 94 | # TTL = time-to-live, starting with the allotted timeout (or 0 for unlimited) | ||
| 95 | my $runtime = $$self{runtime}; | ||
| 96 | $$runtime{TTL} = $$self{opts}{timeout} unless defined $$runtime{TTL}; | ||
| 97 | $self->bind_log; | ||
| 98 | # Empty the package namespace | ||
| 99 | foreach my $subname (keys %LaTeXML::Package::Pool::) { | ||
| 100 | delete $LaTeXML::Package::Pool::{$subname}; | ||
| 101 | } | ||
| 102 | |||
| 103 | my $latexml; | ||
| 104 | eval { | ||
| 105 | # Prepare LaTeXML object | ||
| 106 | local $SIG{'ALRM'} = sub { die "Fatal:conversion:init Failed to initialize LaTeXML state\n" }; | ||
| 107 | alarm($$runtime{TTL}); | ||
| 108 | |||
| 109 | $latexml = new_latexml($$self{opts}); | ||
| 110 | |||
| 111 | $$runtime{TTL} = alarm(0); | ||
| 112 | 1; | ||
| 113 | }; | ||
| 114 | ## NOTE: This will give double errors, if latexml has already handled it! | ||
| 115 | $$latexml{state}->noteStatus('fatal') if $latexml && $@; # Fatal Error? | ||
| 116 | if ($@) { #Fatal occured! | ||
| 117 | Debug($@); | ||
| 118 | Note("Initialization complete: " . $latexml->getStatusMessage . ". Aborting.") if defined $latexml; | ||
| 119 | # Close and restore STDERR to original condition. | ||
| 120 | $$self{log} .= $self->flush_log; | ||
| 121 | $$self{ready} = 0; | ||
| 122 | return; | ||
| 123 | } else { | ||
| 124 | # Demand errorless initialization | ||
| 125 | my $init_status = $latexml->getStatusMessage; | ||
| 126 | if ($init_status =~ /error/i) { | ||
| 127 | Note("Initialization complete: " . $init_status . ". Aborting."); | ||
| 128 | $$self{log} .= $self->flush_log; | ||
| 129 | $$self{ready} = 0; | ||
| 130 | return; | ||
| 131 | } | ||
| 132 | } | ||
| 133 | |||
| 134 | # Save latexml in object: | ||
| 135 | $$self{log} .= $self->flush_log; | ||
| 136 | $$self{latexml} = $latexml; | ||
| 137 | $$self{ready} = 1; | ||
| 138 | return; | ||
| 139 | } | ||
| 140 | |||
| 141 | sub convert { | ||
| 142 | my ($self, $source) = @_; | ||
| 143 | # 1 Prepare for conversion | ||
| 144 | # 1.1 Initialize session if needed: | ||
| 145 | # TTL = time-to-live, starting with the allotted timeout | ||
| 146 | # on each call to convert (or 0 for unlimited) | ||
| 147 | $$self{runtime} = { TTL => $$self{opts}{timeout} }; | ||
| 148 | my $runtime = $$self{runtime}; | ||
| 149 | $self->initialize_session unless $$self{ready}; | ||
| 150 | if (!$$self{ready}) { # We can't initialize, return error: | ||
| 151 | return { result => undef, log => $self->flush_log, status => "Initialization failed.", status_code => 3 }; } | ||
| 152 | |||
| 153 | $self->bind_log; | ||
| 154 | # 1.2 Inform of identity, increase conversion counter | ||
| 155 | my $opts = $$self{opts}; | ||
| 156 | ($$runtime{status}, $$runtime{status_code}) = (undef, undef); | ||
| 157 | Note("$LaTeXML::IDENTITY"); | ||
| 158 | NoteLog("invoked as [$0 " . join(' ', @ARGV) . "]"); | ||
| 159 | Note(($$opts{recursive} ? "recursive " : "") . "processing started " . localtime()); | ||
| 160 | |||
| 161 | # 1.3 Prepare for What's IN: | ||
| 162 | # We use a new temporary variable to avoid confusion with daemon caching | ||
| 163 | my ($current_preamble, $current_postamble); | ||
| 164 | # 1.3.1 Math needs to magically trigger math mode if needed | ||
| 165 | if ($$opts{whatsin} eq "math") { | ||
| 166 | $current_preamble = 'literal:\begin{document}\ensuremathfollows'; | ||
| 167 | $current_postamble = 'literal:\ensuremathpreceeds\end{document}'; } | ||
| 168 | # 1.3.2 Fragments need to have a default pre- and postamble, if none provided | ||
| 169 | elsif ($$opts{whatsin} eq 'fragment') { | ||
| 170 | $current_preamble = $$opts{preamble} || 'standard_preamble.tex'; | ||
| 171 | $current_postamble = $$opts{postamble} || 'standard_postamble.tex'; } | ||
| 172 | # 1.3.3 Archives need to get unpacked in a sandbox (with sufficient bookkeeping) | ||
| 173 | elsif ($$opts{whatsin} =~ /^archive/) { | ||
| 174 | # Sandbox the input | ||
| 175 | $$opts{archive_sourcedirectory} = $$opts{sourcedirectory}; | ||
| 176 | my $sandbox_directory = File::Temp->newdir(TMPDIR => 1); | ||
| 177 | $$opts{sourcedirectory} = $sandbox_directory; | ||
| 178 | # Extract the archive in the sandbox | ||
| 179 | $source = unpack_source($source, $sandbox_directory); | ||
| 180 | if (!defined $source) { # Unpacking failed to find a source | ||
| 181 | $$opts{sourcedirectory} = $$opts{archive_sourcedirectory}; | ||
| 182 | my $log = $self->flush_log; | ||
| 183 | $log .= "\nFatal:invalid:Archive Can't detect a source TeX file!\nStatus:conversion:3\n"; | ||
| 184 | return { result => undef, log => $log, status => "Fatal:invalid:Archive Can't detect a source TeX file!", status_code => 3 }; } | ||
| 185 | # Destination magic: If we expect an archive on output, we need to invent the appropriate destination ourselves when not given. | ||
| 186 | # Since the LaTeXML API never writes the final archive file to disk, we just use a pretend sourcename.zip: | ||
| 187 | if (($$opts{whatsout} =~ /^archive/) && (!$$opts{destination})) { | ||
| 188 | $$opts{placeholder_destination} = 1; | ||
| 189 | $$opts{destination} = pathname_name($source) . ".zip"; } } | ||
| 190 | |||
| 191 | # 1.4 Prepare for What's OUT (if we need a sandbox) | ||
| 192 | if ($$opts{whatsout} =~ /^archive/) { | ||
| 193 | $$opts{archive_sitedirectory} = $$opts{sitedirectory}; | ||
| 194 | $$opts{archive_destination} = $$opts{destination}; | ||
| 195 | my $destination_name = $$opts{destination} ? pathname_name($$opts{destination}) : 'document'; | ||
| 196 | my $sandbox_directory = File::Temp->newdir(TMPDIR => 1); | ||
| 197 | my $extension = $$opts{format}; | ||
| 198 | $extension =~ s/\d+$//; | ||
| 199 | $extension =~ s/^epub|mobi$/xhtml/; | ||
| 200 | my $sandbox_destination = "$destination_name.$extension"; | ||
| 201 | $$opts{sitedirectory} = $sandbox_directory; | ||
| 202 | |||
| 203 | if ($$opts{format} eq 'epub') { | ||
| 204 | $$opts{resource_directory} = File::Spec->catdir($sandbox_directory, 'OPS'); | ||
| 205 | $$opts{destination} = pathname_concat(File::Spec->catdir($sandbox_directory, 'OPS'), $sandbox_destination); } | ||
| 206 | else { | ||
| 207 | $$opts{destination} = pathname_concat($sandbox_directory, $sandbox_destination); } } | ||
| 208 | # 1.4.1 Since we can allow "virtual" destinations for archives / webservice APIs, | ||
| 209 | # we postpone the auxiliary resource sanity check (logically of LaTeXML::Config) | ||
| 210 | # to this time, where we can be certain if a user has run a local job without --dest | ||
| 211 | if ((!$$opts{destination}) | ||
| 212 | && ($$opts{dographics} || $$opts{picimages} || grep { $_ eq 'images' or $_ eq 'svg' } @{ $$opts{math_formats} })) { | ||
| 213 | Warn("expected", "options", undef, | ||
| 214 | "must supply --destination to support auxilliary files", | ||
| 215 | " disabling: --nomathimages --nographicimages --nopictureimages"); | ||
| 216 | # default resources is sorta ok: we might not copy, but we'll still have the links/script/etc | ||
| 217 | $$opts{dographics} = 0; | ||
| 218 | $$opts{picimages} = 0; | ||
| 219 | removeMathFormat($opts, 'images'); | ||
| 220 | removeMathFormat($opts, 'svg'); | ||
| 221 | maybeAddMathFormat($opts, 'pmml'); } | ||
| 222 | |||
| 223 | # 1.5 Prepare a daemon frame | ||
| 224 | my $latexml = $$self{latexml}; | ||
| 225 | $latexml->withState(sub { | ||
| 226 | my ($state) = @_; # Sandbox state | ||
| 227 | $$state{status} = {}; | ||
| 228 | my $stomach = $$state{stomach}; | ||
| 229 | delete $$stomach{rescued_boxes} if $$stomach{rescued_boxes}; | ||
| 230 | $state->pushDaemonFrame; | ||
| 231 | $state->assignValue('_authlist', $$opts{authlist}, 'global'); | ||
| 232 | $state->assignValue('REMOTE_REQUEST', (!$$opts{local}), 'global'); | ||
| 233 | }); | ||
| 234 | |||
| 235 | # 2 Beginning Core conversion - digest the source: | ||
| 236 | my ($digested, $dom, $serialized) = (undef, undef, undef); | ||
| 237 | eval { | ||
| 238 | alarm($$runtime{TTL}); | ||
| 239 | my $mode = ($$opts{type} eq 'auto') ? 'TeX' : $$opts{type}; | ||
| 240 | $digested = $latexml->digestFile($source, preamble => $current_preamble, | ||
| 241 | postamble => $current_postamble, | ||
| 242 | mode => $mode, | ||
| 243 | noinitialize => 1); | ||
| 244 | $$runtime{TTL} = alarm(0); }; | ||
| 245 | my $eval_report = $@; | ||
| 246 | if (!$digested && $eval_report) { | ||
| 247 | # We can retry finishing digestion if hit a Fatal, | ||
| 248 | # sometimes there are leftover boxes we can accept. | ||
| 249 | eval { | ||
| 250 | alarm($$runtime{TTL}); | ||
| 251 | $digested = $latexml->withState(sub { | ||
| 252 | return $latexml->finishDigestion; }); | ||
| 253 | $$runtime{TTL} = alarm(0); }; | ||
| 254 | $eval_report .= $@ if $@; } | ||
| 255 | # 2.1 Now, convert to DOM and output, if desired. | ||
| 256 | my $core_target = $$opts{format}; | ||
| 257 | # Default Core target is XML | ||
| 258 | if ($core_target ne 'tex' and $core_target ne 'box') { | ||
| 259 | $core_target = 'xml'; } | ||
| 260 | if ($digested) { | ||
| 261 | eval { | ||
| 262 | alarm($$runtime{TTL}); | ||
| 263 | $latexml->withState(sub { | ||
| 264 | if ($core_target eq 'tex') { | ||
| 265 | $serialized = LaTeXML::Core::Token::UnTeX($digested); } | ||
| 266 | elsif ($core_target eq 'box') { | ||
| 267 | $serialized = ($$opts{verbosity} > 0 ? $digested->stringify : $digested->toString); } | ||
| 268 | elsif ($core_target eq 'xml') { | ||
| 269 | $dom = $latexml->convertDocument($digested); } }); | ||
| 270 | $$runtime{TTL} = alarm(0); }; | ||
| 271 | $eval_report .= $@ if $@; | ||
| 272 | # Try to rescue the document if e.g. math parsing hit a Fatal error | ||
| 273 | if (!$dom && $@ && $core_target eq 'xml') { | ||
| 274 | $dom = $latexml->withState(sub { | ||
| 275 | my ($state) = @_; | ||
| 276 | my $rescued = $$state{rescued_document}; | ||
| 277 | $rescued->finalize() if $rescued; | ||
| 278 | return $rescued; }); } } | ||
| 279 | $$runtime{status} = $latexml->getStatusMessage; | ||
| 280 | $$runtime{status_code} = $latexml->getStatusCode; | ||
| 281 | |||
| 282 | $latexml->showProfile(); # Show profile (if any) | ||
| 283 | |||
| 284 | # 2.2 Bookkeeping in case in-eval perl die() deaths occurred | ||
| 285 | if ($eval_report) { | ||
| 286 | $$runtime{status} .= "\n" . $eval_report . "\n"; | ||
| 287 | $$runtime{status_code} = 3; } | ||
| 288 | |||
| 289 | # End daemon run, by popping frame: | ||
| 290 | $latexml->withState(sub { | ||
| 291 | my ($state) = @_; # Remove current state frame | ||
| 292 | ## TODO: This section of option preparations can be factored out as a subroutine if it grows further | ||
| 293 | ## the general idea is that right before the "pop" of the daemon frame, we have access to all meaningful | ||
| 294 | ## global state values, and we can preserve the relevant ones for the post-processing stage | ||
| 295 | ## BEGIN POST-PROCESSING-PREP | ||
| 296 | $$opts{searchpaths} = $state->lookupValue('SEARCHPATHS'); # save the searchpaths for post-processing | ||
| 297 | if ($state->lookupValue('LEXEMATIZE_MATH')) { # save potential request for serializing math lexemes | ||
| 298 | $$opts{math_formats} ||= []; | ||
| 299 | push @{ $$opts{math_formats} }, 'lexemes'; | ||
| 300 | # recheck need for parallel | ||
| 301 | $$opts{parallelmath} = 1 if (@{ $$opts{math_formats} } > 1); } | ||
| 302 | ## END POST-PROCESSING-PREP | ||
| 303 | $state->popDaemonFrame; | ||
| 304 | }); | ||
| 305 | if ($LaTeXML::UNSAFE_FATAL) { | ||
| 306 | # If the conversion hit an unsafe fatal, we need to reinitialize | ||
| 307 | $LaTeXML::UNSAFE_FATAL = 0; | ||
| 308 | $$self{ready} = 0; | ||
| 309 | } | ||
| 310 | Note(($$opts{recursive} ? "recursive " : "") . "Conversion complete: " . $$runtime{status}); | ||
| 311 | |||
| 312 | # 2.3 Clean up and exit if we only wanted the serialization of the core conversion | ||
| 313 | if ($serialized) { | ||
| 314 | # If serialized has been set, we are done with the job | ||
| 315 | # If we just processed an archive, clean up sandbox directory. | ||
| 316 | if ($$opts{whatsin} =~ /^archive/) { | ||
| 317 | rmtree($$opts{sourcedirectory}); | ||
| 318 | $$opts{sourcedirectory} = $$opts{archive_sourcedirectory}; } | ||
| 319 | return { result => $serialized, log => $self->flush_log, status => $$runtime{status}, status_code => $$runtime{status_code} }; | ||
| 320 | } | ||
| 321 | |||
| 322 | # 3 If desired, post-process | ||
| 323 | my $result = $dom; | ||
| 324 | if ($$opts{post} && $dom) { | ||
| 325 | if (!$dom->documentElement) { | ||
| 326 | # Make a completely empty document have at least one element for post-processing | ||
| 327 | # important for utility features such as packing .zip archives for output | ||
| 328 | $$dom{document}->setDocumentElement($$dom{document}->createElement("document")); | ||
| 329 | } | ||
| 330 | $result = $self->convert_post($dom); | ||
| 331 | } | ||
| 332 | # 4 Clean-up: undo everything we sandboxed | ||
| 333 | if ($$opts{whatsin} =~ /^archive/) { | ||
| 334 | rmtree($$opts{sourcedirectory}); | ||
| 335 | $$opts{sourcedirectory} = $$opts{archive_sourcedirectory}; } | ||
| 336 | if ($$opts{whatsout} =~ /^archive/) { | ||
| 337 | rmtree($$opts{sitedirectory}); | ||
| 338 | $$opts{sitedirectory} = $$opts{archive_sitedirectory}; | ||
| 339 | $$opts{destination} = $$opts{archive_destination}; | ||
| 340 | if (delete $$opts{placeholder_destination}) { | ||
| 341 | delete $$opts{destination}; } } | ||
| 342 | |||
| 343 | # 5 Output -- if not using Post::Writer, which never considers this serialization logic | ||
| 344 | # | ||
| 345 | # 5.1 Serialize the XML/HTML result (or just return the Perl object, if requested) | ||
| 346 | # GOAL: $serialized must contain a utf8-encoded string at the return. | ||
| 347 | # | ||
| 348 | # This logic can be fragile, especially due to external components changing behavior between versions. | ||
| 349 | # but also due to having a variety of serialization routes based on Core vs Post, whatsout, destination... | ||
| 350 | # | ||
| 351 | undef $serialized; | ||
| 352 | my $ref_result = ref($result) || ''; | ||
| 353 | if ($$opts{format} eq 'dom') { # No serialize needed in DOM output case | ||
| 354 | $serialized = $result; } | ||
| 355 | elsif ($ref_result =~ /^(:?LaTe)?XML/) { | ||
| 356 | if ($$opts{format} =~ /^jats|x(ht)?ml$/) { | ||
| 357 | if ($ref_result =~ /Document$/) { | ||
| 358 | $serialized = $result->toString(1); | ||
| 359 | if ($serialized and $ref_result eq 'LaTeXML::Core::Document') { | ||
| 360 | $serialized = Encode::encode('UTF-8', $serialized); } | ||
| 361 | } else { # fragment case | ||
| 362 | $serialized = $result->toString(1, 1); | ||
| 363 | } } | ||
| 364 | elsif ($$opts{format} =~ /^html/) { | ||
| 365 | if (ref($result) =~ /^LaTeXML::(Post::)?Document$/) { | ||
| 366 | $serialized = $result->getDocument->toStringHTML; } | ||
| 367 | else { # fragment case | ||
| 368 | local $XML::LibXML::setTagCompression = 1; | ||
| 369 | $serialized = $result->toString(1, 1); } } } | ||
| 370 | # Compressed/archive/other case, just pass on | ||
| 371 | else { $serialized = $result; } | ||
| 372 | |||
| 373 | # 5.2 Finalize logging and return a response containing the document result, log and status | ||
| 374 | Note("Status:conversion:" . ($$runtime{status_code} || '0')); | ||
| 375 | return { result => $serialized, log => $self->flush_log, status => $$runtime{status}, 'status_code' => $$runtime{status_code} }; | ||
| 376 | } | ||
| 377 | |||
| 378 | ########################################### | ||
| 379 | #### Converter Management ##### | ||
| 380 | ########################################### | ||
| 381 | |||
| 382 | our %DAEMON_CACHE = (); | ||
| 383 | our %CONFIG_CACHE = (); | ||
| 384 | |||
| 385 | sub get_converter { | ||
| 386 | my ($self, $config, $key) = @_; | ||
| 387 | # Default key, unless made explicit | ||
| 388 | if (!$key && $config) { | ||
| 389 | $key = $config->get('cache_key') || $config->get('profile'); } | ||
| 390 | |||
| 391 | my $converter = $key && $DAEMON_CACHE{$key}; | ||
| 392 | |||
| 393 | if (!defined $converter) { | ||
| 394 | # Trading flexibility for performance: | ||
| 395 | # once a cache_key is used, it can not be redefined in the same session | ||
| 396 | # (and we never needed it redefine-able) | ||
| 397 | # | ||
| 398 | # Instead, once a cache_key is specified, (re)booting converter objects | ||
| 399 | # for that config set can be done without spending time examining options | ||
| 400 | # via the $CONFIG_CACHE | ||
| 401 | $config ||= $CONFIG_CACHE{$key}; | ||
| 402 | |||
| 403 | $converter = LaTeXML->new($config->clone); | ||
| 404 | if ($key) { | ||
| 405 | # cache both converter and config, for self-contained reloading | ||
| 406 | $DAEMON_CACHE{$key} = $converter; | ||
| 407 | $CONFIG_CACHE{$key} = $config; } } | ||
| 408 | return $converter; } | ||
| 409 | |||
| 410 | ########################################### | ||
| 411 | #### Helper routines ##### | ||
| 412 | ########################################### | ||
| 413 | sub convert_post { | ||
| 414 | my ($self, $dom) = @_; | ||
| 415 | my $opts = $$self{opts}; | ||
| 416 | my $runtime = $$self{runtime}; | ||
| 417 | my ($xslt, $parallel, $math_formats, $format, $verbosity, $defaultresources, $embed) = | ||
| 418 | map { $$opts{$_} } qw(stylesheet parallelmath math_formats format verbosity defaultresources embed); | ||
| 419 | ## $verbosity = $verbosity || 0; | ||
| 420 | SetVerbosity($verbosity) if defined $verbosity; | ||
| 421 | my %PostOPS = ( ####verbosity => $verbosity, | ||
| 422 | validate => $$opts{validate}, | ||
| 423 | sourceDirectory => $$opts{sourcedirectory}, | ||
| 424 | siteDirectory => $$opts{sitedirectory}, | ||
| 425 | resource_directory => $$opts{resource_directory}, | ||
| 426 | searchpaths => $$opts{searchpaths}, | ||
| 427 | nocache => 1, | ||
| 428 | destination => $$opts{destination}, | ||
| 429 | is_html => $$opts{is_html}); | ||
| 430 | # Compute destinationDirectory here, | ||
| 431 | # so that we don't depend on post-processing returning a usable Post::Document (Fatal-resilience) | ||
| 432 | if ($PostOPS{destination}) { | ||
| 433 | my ($dir, $name, $ext) = pathname_split($PostOPS{destination}); | ||
| 434 | $PostOPS{destinationDirectory} = $dir || '.'; } | ||
| 435 | |||
| 436 | #Postprocess | ||
| 437 | $parallel = $parallel || 0; | ||
| 438 | |||
| 439 | my $DOCUMENT = LaTeXML::Post::Document->new($dom, %PostOPS); | ||
| 440 | my @procs = (); | ||
| 441 | #TODO: Add support for the following: | ||
| 442 | my $dbfile = $$opts{dbfile}; | ||
| 443 | if (defined $dbfile && !-f $dbfile) { | ||
| 444 | if (my $dbdir = pathname_directory($dbfile)) { | ||
| 445 | pathname_mkdir($dbdir); } } | ||
| 446 | my $DB = LaTeXML::Util::ObjectDB->new(dbfile => $dbfile, %PostOPS); | ||
| 447 | if ($format eq 'epub') { # epub requires a TOC | ||
| 448 | $self->check_TOC($DOCUMENT); } | ||
| 449 | ### Advanced Processors: | ||
| 450 | if ($$opts{split}) { | ||
| 451 | require LaTeXML::Post::Split; | ||
| 452 | push(@procs, LaTeXML::Post::Split->new(split_xpath => $$opts{splitpath}, splitnaming => $$opts{splitnaming}, | ||
| 453 | db => $DB, %PostOPS)); } | ||
| 454 | my $scanner = ($$opts{scan} || $DB) && (LaTeXML::Post::Scan->new( | ||
| 455 | db => $DB, | ||
| 456 | labelids => ($$opts{splitnaming} && ($$opts{splitnaming} =~ /^label/) ? 1 : 0), | ||
| 457 | %PostOPS)); | ||
| 458 | push(@procs, $scanner) if $$opts{scan}; | ||
| 459 | if (!($$opts{prescan})) { | ||
| 460 | if ($$opts{index}) { | ||
| 461 | require LaTeXML::Post::MakeIndex; | ||
| 462 | push(@procs, LaTeXML::Post::MakeIndex->new(db => $DB, permuted => $$opts{permutedindex}, | ||
| 463 | split => $$opts{splitindex}, scanner => $scanner, | ||
| 464 | %PostOPS)); } | ||
| 465 | require LaTeXML::Post::MakeBibliography; | ||
| 466 | push(@procs, LaTeXML::Post::MakeBibliography->new(db => $DB, bibliographies => $$opts{bibliographies}, | ||
| 467 | split => $$opts{splitbibliography}, scanner => $scanner, | ||
| 468 | %PostOPS)); | ||
| 469 | if ($$opts{crossref}) { | ||
| 470 | require LaTeXML::Post::CrossRef; | ||
| 471 | push(@procs, LaTeXML::Post::CrossRef->new( | ||
| 472 | db => $DB, urlstyle => $$opts{urlstyle}, | ||
| 473 | extension => $$opts{extension}, | ||
| 474 | ($$opts{numbersections} ? (number_sections => 1) : ()), | ||
| 475 | ($$opts{navtoc} ? (navigation_toc => $$opts{navtoc}) : ()), | ||
| 476 | %PostOPS)); } | ||
| 477 | if ($$opts{picimages}) { | ||
| 478 | require LaTeXML::Post::PictureImages; | ||
| 479 | push(@procs, LaTeXML::Post::PictureImages->new(%PostOPS)); | ||
| 480 | } | ||
| 481 | else { | ||
| 482 | require LaTeXML::Post::PictureImages; | ||
| 483 | push(@procs, LaTeXML::Post::PictureImages->new(empty_only => 1, %PostOPS)); } | ||
| 484 | |||
| 485 | if ($$opts{dographics}) { | ||
| 486 | # TODO: Rethink full-fledged graphics support | ||
| 487 | require LaTeXML::Post::Graphics; | ||
| 488 | my @g_options = (); | ||
| 489 | if ($$opts{graphicsmaps} && scalar(@{ $$opts{graphicsmaps} })) { | ||
| 490 | my @maps = map { [split(/\./, $_)] } @{ $$opts{graphicsmaps} }; | ||
| 491 | push(@g_options, (graphics_types => [map { $$_[0] } @maps], | ||
| 492 | type_properties => { map { ($$_[0] => { destination_type => ($$_[1] || $$_[0]) }) } @maps })); } | ||
| 493 | push(@procs, LaTeXML::Post::Graphics->new(@g_options, %PostOPS)); | ||
| 494 | } | ||
| 495 | if ($$opts{svg}) { | ||
| 496 | require LaTeXML::Post::SVG; | ||
| 497 | push(@procs, LaTeXML::Post::SVG->new(%PostOPS)); } | ||
| 498 | if (@$math_formats) { | ||
| 499 | my @mprocs = (); | ||
| 500 | ### # If XMath is not first, it must be at END! Or... ??? | ||
| 501 | foreach my $fmt (@$math_formats) { | ||
| 502 | if ($fmt eq 'xmath') { | ||
| 503 | require LaTeXML::Post::XMath; | ||
| 504 | push(@mprocs, LaTeXML::Post::XMath->new(%PostOPS)); } | ||
| 505 | elsif ($fmt eq 'pmml') { | ||
| 506 | require LaTeXML::Post::MathML::Presentation; | ||
| 507 | push(@mprocs, LaTeXML::Post::MathML::Presentation->new( | ||
| 508 | linelength => $$opts{linelength}, | ||
| 509 | (defined $$opts{plane1} ? (plane1 => $$opts{plane1}) : (plane1 => 1)), | ||
| 510 | ($$opts{hackplane1} ? (hackplane1 => 1) : ()), | ||
| 511 | (defined $$opts{invisibletimes} ? (invisibletimes => $$opts{invisibletimes}) : | ||
| 512 | (invisibletimes => 1)), | ||
| 513 | %PostOPS)); } | ||
| 514 | elsif ($fmt eq 'cmml') { | ||
| 515 | require LaTeXML::Post::MathML::Content; | ||
| 516 | push(@mprocs, LaTeXML::Post::MathML::Content->new( | ||
| 517 | (defined $$opts{plane1} ? (plane1 => $$opts{plane1}) : (plane1 => 1)), | ||
| 518 | ($$opts{hackplane1} ? (hackplane1 => 1) : ()), | ||
| 519 | %PostOPS)); } | ||
| 520 | elsif ($fmt eq 'om') { | ||
| 521 | require LaTeXML::Post::OpenMath; | ||
| 522 | push(@mprocs, LaTeXML::Post::OpenMath->new( | ||
| 523 | (defined $$opts{plane1} ? (plane1 => $$opts{plane1}) : (plane1 => 1)), | ||
| 524 | ($$opts{hackplane1} ? (hackplane1 => 1) : ()), | ||
| 525 | %PostOPS)); } | ||
| 526 | elsif ($fmt eq 'images') { | ||
| 527 | require LaTeXML::Post::MathImages; | ||
| 528 | push(@mprocs, LaTeXML::Post::MathImages->new(magnification => $$opts{mathimagemag}, | ||
| 529 | %PostOPS)); } | ||
| 530 | elsif ($fmt eq 'svg') { | ||
| 531 | require LaTeXML::Post::MathImages; | ||
| 532 | push(@mprocs, LaTeXML::Post::MathImages->new(magnification => $$opts{mathimagemag}, | ||
| 533 | imagetype => 'svg', | ||
| 534 | %PostOPS)); } | ||
| 535 | elsif ($fmt eq 'mathtex') { | ||
| 536 | require LaTeXML::Post::TeXMath; | ||
| 537 | push(@mprocs, LaTeXML::Post::TeXMath->new(%PostOPS)); } | ||
| 538 | elsif ($fmt eq 'unicodemath') { | ||
| 539 | require LaTeXML::Post::UnicodeMath; | ||
| 540 | push(@mprocs, LaTeXML::Post::UnicodeMath->new(%PostOPS)); } | ||
| 541 | elsif ($fmt eq 'lexemes') { | ||
| 542 | require LaTeXML::Post::LexMath; | ||
| 543 | push(@mprocs, LaTeXML::Post::LexMath->new(%PostOPS)); } | ||
| 544 | } | ||
| 545 | ### $keepXMath = 0 unless defined $keepXMath; | ||
| 546 | ### OR is $parallelmath ALWAYS on whenever there's more than one math processor? | ||
| 547 | if ($parallel) { | ||
| 548 | my $main = shift(@mprocs); | ||
| 549 | $main->setParallel(@mprocs); | ||
| 550 | push(@procs, $main); } | ||
| 551 | else { | ||
| 552 | push(@procs, @mprocs); } | ||
| 553 | } | ||
| 554 | if ($xslt) { | ||
| 555 | require LaTeXML::Post::XSLT; | ||
| 556 | my $parameters = { LATEXML_VERSION => "'$LaTeXML::VERSION'" }; | ||
| 557 | my @searchpaths = ('.', $DOCUMENT->getSearchPaths); | ||
| 558 | # store these for the XSLT; XSLT Processor will copy resources where needed. | ||
| 559 | foreach my $css (@{ $$opts{css} }) { | ||
| 560 | push(@{ $$parameters{CSS} }, $css); } | ||
| 561 | foreach my $js (@{ $$opts{javascript} }) { | ||
| 562 | push(@{ $$parameters{JAVASCRIPT} }, $js); } | ||
| 563 | if ($$opts{icon}) { | ||
| 564 | $$parameters{ICON} = $$opts{icon}; } | ||
| 565 | if (!defined $$opts{timestamp}) { $$opts{timestamp} = defined $ENV{SOURCE_DATE_EPOCH} ? gmtime($ENV{SOURCE_DATE_EPOCH}) : localtime(); } | ||
| 566 | if ($$opts{timestamp}) { $$parameters{TIMESTAMP} = "'" . $$opts{timestamp} . "'"; } | ||
| 567 | # Now add in the explicitly given XSLT parameters | ||
| 568 | foreach my $parm (@{ $$opts{xsltparameters} }) { | ||
| 569 | if ($parm =~ /^\s*(\w+)\s*:\s*(.*)$/) { | ||
| 570 | $$parameters{$1} = "'" . $2 . "'"; } | ||
| 571 | else { | ||
| 572 | warn "xsltparameter not in recognized format: 'name:value' got: '$parm'\n"; } | ||
| 573 | } | ||
| 574 | |||
| 575 | push(@procs, LaTeXML::Post::XSLT->new(stylesheet => $xslt, | ||
| 576 | parameters => $parameters, | ||
| 577 | searchpaths => [@searchpaths], | ||
| 578 | noresources => (defined $$opts{defaultresources}) && !$$opts{defaultresources}, | ||
| 579 | %PostOPS)); | ||
| 580 | } | ||
| 581 | } | ||
| 582 | |||
| 583 | # If we are doing a local conversion OR | ||
| 584 | # we are going to package into an archive | ||
| 585 | # write all the files to disk during post-processing | ||
| 586 | if ($$opts{destination} && | ||
| 587 | (($$opts{local} && ($$opts{whatsout} eq 'document')) | ||
| 588 | || ($$opts{whatsout} =~ /^archive/))) { | ||
| 589 | require LaTeXML::Post::Writer; | ||
| 590 | push(@procs, LaTeXML::Post::Writer->new( | ||
| 591 | format => $format, omit_doctype => $$opts{omit_doctype}, | ||
| 592 | %PostOPS)); | ||
| 593 | } | ||
| 594 | |||
| 595 | # Do the actual post-processing: | ||
| 596 | my @postdocs; | ||
| 597 | ## my $latexmlpost = LaTeXML::Post->new(verbosity => $verbosity || 0); | ||
| 598 | my $latexmlpost = LaTeXML::Post->new(); | ||
| 599 | eval { | ||
| 600 | alarm($$runtime{TTL}); | ||
| 601 | @postdocs = $latexmlpost->ProcessChain($DOCUMENT, @procs); | ||
| 602 | $$runtime{TTL} = alarm(0); | ||
| 603 | 1; }; | ||
| 604 | # 3.1 Bookkeeping if a post-processing Fatal error occurred | ||
| 605 | if ($@) { #Fatal occured! | ||
| 606 | $$runtime{status_code} = 3; | ||
| 607 | local $@ = 'Fatal:conversion:unknown ' . $@ unless $@ =~ /^\n?\S*Fatal:/s; | ||
| 608 | Debug($@); } | ||
| 609 | |||
| 610 | # Finalize by arranging any manifests and packaging the output. | ||
| 611 | # If our format requires a manifest, create one | ||
| 612 | if (($$opts{whatsout} =~ /^archive/) && ($format !~ /^x?html|xml/)) { | ||
| 613 | require LaTeXML::Post::Manifest; | ||
| 614 | my $manifest_maker = LaTeXML::Post::Manifest->new(db => $DB, format => $format, log => $$opts{log}, %PostOPS); | ||
| 615 | $manifest_maker->process(@postdocs); } | ||
| 616 | # Archives: when a relative --log is requested, write to sandbox prior packing | ||
| 617 | # TODO: This can be enhanced, as any path can be relativized. | ||
| 618 | # TODO: ALSO, we now always have a log file, so maybe always add it as default? | ||
| 619 | if ($$opts{log} && ($$opts{whatsout} =~ /^archive/) && (!pathname_is_absolute($$opts{log}))) { | ||
| 620 | ### We can't rely on the ->getDestinationDirectory method, as Fatal post-processing jobs have UNDEF @postdocs !!! | ||
| 621 | ### my $destination_directory = $postdocs[0]->getDestinationDirectory(); | ||
| 622 | my $destination_directory = $PostOPS{destinationDirectory}; | ||
| 623 | my $log_file = pathname_absolute($$opts{log}, $destination_directory); | ||
| 624 | if (pathname_is_contained($log_file, $destination_directory)) { | ||
| 625 | NoteLog(($$opts{recursive} ? "recursive " : "") . "Post-processing complete: " . $latexmlpost->getStatusMessage); | ||
| 626 | NoteLog(($$opts{recursive} ? "recursive " : "") . "processing finished " . localtime()); | ||
| 627 | my $archive_log_status_code = max($$runtime{status_code}, $latexmlpost->getStatusCode); | ||
| 628 | Note("Status:conversion:" . $archive_log_status_code); | ||
| 629 | open my $log_fh, '>', $log_file; | ||
| 630 | print $log_fh $self->flush_log; | ||
| 631 | close $log_fh; | ||
| 632 | $self->bind_log; } | ||
| 633 | # TODO: This needs a bit of rethinking, likely a fallback.log file should be created and returned with the archive | ||
| 634 | else { Error("I/O", "log", "The target log file isn't contained in the destination directory!"); } } | ||
| 635 | # Handle the output packaging | ||
| 636 | |||
| 637 | my ($postdoc) = pack_collection(collection => [@postdocs], whatsout => $$opts{whatsout}, format => $format, %PostOPS); | ||
| 638 | |||
| 639 | $DB->finish; | ||
| 640 | |||
| 641 | # Merge postprocessing and main processing reports | ||
| 642 | ### HACKY until we use a single Core::State object, we'll just "wing it" for status messages: | ||
| 643 | my $post_status = $latexmlpost->getStatusMessage; | ||
| 644 | if ($post_status ne $$runtime{status}) { # Just so that we avoid double "No problem" reporting | ||
| 645 | $$runtime{status} .= "\n" . $post_status; | ||
| 646 | } | ||
| 647 | $$runtime{status_code} = max($$runtime{status_code}, $latexmlpost->getStatusCode); | ||
| 648 | ### HACKY END | ||
| 649 | Note(($$opts{recursive} ? "recursive " : "") . "Post-processing complete: " . $latexmlpost->getStatusMessage); | ||
| 650 | NoteLog(($$opts{recursive} ? "recursive " : "") . "processing finished " . localtime()); | ||
| 651 | |||
| 652 | # Avoid writing the main file twice (non-archive documents): | ||
| 653 | if ($$opts{destination} && $$opts{local} && ($$opts{whatsout} eq 'document')) { | ||
| 654 | undef $postdoc; } | ||
| 655 | return $postdoc; } | ||
| 656 | |||
| 657 | # This is *very* unmodular, but introducing a pre-Epub postprocessor | ||
| 658 | # seems rather heavy at this stage? | ||
| 659 | # Add a full TOC, to be used for navigation, but *not* displayed. | ||
| 660 | sub check_TOC { | ||
| 661 | my ($self, $document) = @_; | ||
| 662 | if (!$document->findnode('//ltx:TOC[@lists="toc"]')) { | ||
| 663 | my @s = (qw(ltx:part ltx:chapter ltx:section ltx:subsection ltx:subsubsection | ||
| 664 | ltx:paragraph ltx:subparagraph ltx:appendix ltx:index ltx:bibliography)); | ||
| 665 | $document->prependNodes($document->getDocumentElement, | ||
| 666 | ['ltx:TOC', { lists => 'toc', scope => 'global', | ||
| 667 | select => join(' | ', @s), class => 'ltx_nodisplay' }]); } | ||
| 668 | return; } | ||
| 669 | |||
| 670 | sub new_latexml { | ||
| 671 | my ($opts) = @_; | ||
| 672 | |||
| 673 | # TODO: Do this in a GOOD way to support filepath/URL/string snippets | ||
| 674 | # If we are given string preloads, load them and remove them from the preload list: | ||
| 675 | my $preloads = $$opts{preload}; | ||
| 676 | my (@pre, @str_pre); | ||
| 677 | foreach my $pre (@$preloads) { | ||
| 678 | if (pathname_is_literaldata($pre)) { | ||
| 679 | push @str_pre, $pre; | ||
| 680 | } else { | ||
| 681 | push @pre, $pre; | ||
| 682 | } | ||
| 683 | } | ||
| 684 | my $includepathpis = !(exists $$opts{xsltparameters} && | ||
| 685 | (grep { $_ eq 'LATEXML_VERSION:TEST' } @{ $$opts{xsltparameters} })); | ||
| 686 | require LaTeXML; | ||
| 687 | SetVerbosity($$opts{verbosity}) if defined $$opts{verbosity}; | ||
| 688 | my $latexml = LaTeXML::Core->new(preload => [@pre], searchpaths => [@{ $$opts{paths} }], | ||
| 689 | graphicspaths => ['.'], | ||
| 690 | ### verbosity => $$opts{verbosity}, strict => $$opts{strict}, | ||
| 691 | includecomments => $$opts{comments}, | ||
| 692 | includepathpis => $includepathpis, | ||
| 693 | inputencoding => $$opts{inputencoding}, | ||
| 694 | includestyles => $$opts{includestyles}, | ||
| 695 | documentid => $$opts{documentid}, | ||
| 696 | nomathparse => $$opts{nomathparse}, # Backwards compatibility | ||
| 697 | mathparse => $$opts{mathparse}); | ||
| 698 | |||
| 699 | if (my @baddirs = grep { !-d $_ } @{ $$opts{paths} }) { | ||
| 700 | warn "\n$LaTeXML::IDENTITY : these path directories do not exist: " . join(', ', @baddirs) . "\n"; } | ||
| 701 | |||
| 702 | $latexml->withState(sub { | ||
| 703 | my ($state) = @_; | ||
| 704 | $latexml->initializeState('TeX.pool', @{ $$latexml{preload} || [] }); | ||
| 705 | }); | ||
| 706 | |||
| 707 | # TODO: Do again, need to do this in a GOOD way as well: | ||
| 708 | $latexml->digestFile($_, noinitialize => 1) foreach (@str_pre); | ||
| 709 | |||
| 710 | return $latexml; | ||
| 711 | } | ||
| 712 | |||
| 713 | sub bind_log { | ||
| 714 | my ($self) = @_; | ||
| 715 | my $opts = $$self{opts}; | ||
| 716 | # For now --whatsout=archive just gets deferred back to the old code | ||
| 717 | # But this needs a streamlined refactor in the future. | ||
| 718 | # Something to consider: | ||
| 719 | # 1. Multiple latexml_workers are started in the same CWD by their harness | ||
| 720 | # 2. Before an archive job starts, they would be opening the same default | ||
| 721 | # "latexml.log" or "cortex.log" etc. | ||
| 722 | # 3. The final log destination is available only after the temp dir is setup | ||
| 723 | # ... So we may want to completely remodel the whatsout handling | ||
| 724 | # but what is cleanest and least confusing? | ||
| 725 | if (my $logfile = $$opts{log}) { | ||
| 726 | my $whatsout = $$opts{whatsout}; | ||
| 727 | if ($whatsout && ($whatsout !~ /^archive/)) { | ||
| 728 | UseLog($logfile, 1); } | ||
| 729 | else { | ||
| 730 | UseLog(\$$self{log}, 1); | ||
| 731 | } | ||
| 732 | } else { | ||
| 733 | UseLog(\$$self{log}, 1); } | ||
| 734 | return; } | ||
| 735 | |||
| 736 | sub flush_log { | ||
| 737 | my ($self) = @_; | ||
| 738 | UseLog(undef); | ||
| 739 | my $log = $$self{log}; | ||
| 740 | $$self{log} = q{}; | ||
| 741 | return $log; } | ||
| 742 | |||
| 743 | 1; | ||
| 744 | |||
| 745 | __END__ | ||