lib/LaTeXML/Util/Pack.pm untested

filter: Statementstmt 0.0% 0 / 293 Branchbran 0.0% 0 / 142 Conditioncond 0.0% 0 / 87 MC/DCmcdc 0.0% 0 / 53 Subroutinesub 0.0% 0 / 14 Podpod 0.0% 0 / 6 totaltotal 0.0% 0 / 595 CC 72 SCAR 69.5
CC
72
Cov
0%
unpack_source
1980.0
get_math
90.0
pack_collection
56.0
CRAP 1047.8
1 # /=====================================================================\ #
2 # | LaTeXML::Util::Pack | #
3 # | Packs the requested output (document, fragment, math, archive) | #
4 # |=====================================================================| #
5 # | Part of LaTeXML: | #
6 # | Public domain software, produced as part of work done by the | #
7 # | United States Government & not subject to copyright in the US. | #
8 # |---------------------------------------------------------------------| #
9 # | Bruce Miller <bruce.miller@nist.gov> #_# | #
10 # | http://dlmf.nist.gov/LaTeXML/ (o o) | #
11 # \=========================================================ooo==U==ooo=/ #
12 package LaTeXML::Util::Pack;
13 use strict;
14 use warnings;
15  
16 use LaTeXML::Util::Pathname;
17 use File::Spec::Functions qw(catfile);
18 use File::Path qw(rmtree);
19 use IO::String;
20 use Archive::Zip qw(:CONSTANTS :ERROR_CODES);
21  
22 use base qw(Exporter);
23 our @EXPORT = qw(&unpack_source &pack_collection);
24 our $archive_file_exclusion_regex = qr/(?:^\.)|(?:\.(?:zip|gz|epub|tex|bib|mobi|cache)$)|(?:~$)/;
25  
26 sub unpack_source {
27 my ($source, $sandbox_directory) = @_;
28 my $main_source;
29 my $zip_handle = Archive::Zip->new();
30 if (pathname_is_literaldata($source)) {
31 # If literal, just use the data
32 $source =~ s/^literal\://;
33 my $content_handle = IO::String->new($source);
34 unless ($zip_handle->readFromFileHandle($content_handle) == AZ_OK) {
35 print STDERR "Fatal:I/O:Archive Can't read in literal archive:\n $source\n"; } }
36 else { # Otherwise, read in from file
37 unless ($zip_handle->read($source) == AZ_OK) {
38 print STDERR "Fatal:I/O:Archive Can't read in source archive: $source\n"; } }
39 # Extract the Perl zip datastructure to the temporary directory
40 foreach my $member ($zip_handle->memberNames()) {
41 $zip_handle->extractMember($member, catfile($sandbox_directory, $member)); }
42  
43 # I. Detect and return the main TeX file in that directory (or .txt, for old arXiv bundles)
44  
45 # I.1. arXiv has a special metadata file identifying the primary source, and ignoring assets
46 if (my $readme_member = $zip_handle->memberNamed('00README.XXX')) {
47 my $readme_file = catfile($sandbox_directory, $readme_member->fileName());
48 open(my $README_FH, '<', $readme_file) or
49 (print STDERR "failed to open '$readme_file' for use as ZIP readme: $!. Continuing.\n");
50 local $/ = "\n";
51 my $toplevelfile;
52 while (<$README_FH>) {
53 chomp($_);
54 my ($name, $directive) = split(/\s+/, $_);
55 if ($directive) {
56 if ($directive eq 'toplevelfile') {
57 # shortcut guessing the top file, the user has provided it explicitly.
58 $toplevelfile = catfile($sandbox_directory, $name);
59 } elsif ($directive eq 'ignore') {
60 my $ignored_filepath = catfile($sandbox_directory, $name);
61 unlink($ignored_filepath) if -e $ignored_filepath; } } }
62 return $toplevelfile if $toplevelfile; }
63  
64 # I.2. Without an explicit directive,
65 # heuristically determine the input (borrowed from arXiv::FileGuess)
66 my @TeX_file_members = map { $_->fileName() } $zip_handle->membersMatching('\.(?:[tT](:?[eE][xX]|[xX][tT])|ltx|LTX)$');
67 if (!@TeX_file_members) { # No .tex file? Try files with no, or unusually long, extensions
68 @TeX_file_members = grep { !/\./ || /\.[^.]{4,}$/ } map { $_->fileName() } $zip_handle->members();
69 }
70  
71 my (%Main_TeX_likelihood, %Main_TeX_level);
72 my @vetoed = ();
73 foreach my $tex_file (@TeX_file_members) {
74 # Read in the content
75 $tex_file = catfile($sandbox_directory, $tex_file);
76 # Open file and read first few bytes to do magic sequence identification
77 # note that file will be auto-closed when $FILE_TO_GUESS goes out of scope
78 next unless -e $tex_file; # skip deleted "ignored" files.
79 open(my $FILE_TO_GUESS, '<', $tex_file) or
80 (print STDERR "failed to open '$tex_file' to guess its format: $!. Continuing.\n");
81 local $/ = "\n";
82 my ($maybe_tex, $maybe_tex_priority, $maybe_tex_priority2);
83 TEX_FILE_TRAVERSAL:
84 while (<$FILE_TO_GUESS>) {
85 if ((/\%auto-ignore/ && $. <= 10) || # Ignore
86 ($. <= 10 && /\\input texinfo/) || # TeXInfo
87 ($. <= 10 && /\%auto-include/)) # Auto-include
88 { $Main_TeX_likelihood{$tex_file} = 0; last TEX_FILE_TRAVERSAL; } # Not primary
89 if ($. <= 12 && /^\r?%\&([^\s\n]+)/) {
90 if ($1 eq 'latex209' || $1 eq 'biglatex' || $1 eq 'latex' || $1 eq 'LaTeX') {
91 $Main_TeX_likelihood{$tex_file} = 3; last TEX_FILE_TRAVERSAL; } # LaTeX
92 else {
93 $Main_TeX_likelihood{$tex_file} = 1; last TEX_FILE_TRAVERSAL; } } # Mac TeX
94 # All subsequent checks have lines with '%' in them chopped.
95 # if we need to look for a % then do it earlier!
96 s/\%[^\r]*//;
97 if (/(?:^|\r)\s*\\document(?:style|class)/) {
98 $Main_TeX_likelihood{$tex_file} = 3; last TEX_FILE_TRAVERSAL; } # LaTeX
99 if (/(?:^|\r)\s*(?:\\font|\\magnification|\\input|\\def|\\special|\\baselineskip|\\begin)/) {
100 $maybe_tex = 1; }
101 if (/\\(?:input|include)(?:\s+|\{)([^ \}]+)/) {
102 $maybe_tex = 1;
103 # the argument of \input can't be the main file
104 # (it could in very elaborate multi-target setups, but we DON'T support those)
105 # so veto it.
106 my $vetoed_file = $1;
107 if ($vetoed_file eq 'amstex') { # TeX Priority
108 $Main_TeX_likelihood{$tex_file} = 2; last TEX_FILE_TRAVERSAL; }
109 if ($vetoed_file !~ /\./) {
110 $vetoed_file =~ s/\s+$//; # drop trailing spaces if any;
111 $vetoed_file .= '.tex';
112 }
113 my $base = $tex_file;
114 $base =~ s/\/[^\/]+$//;
115 $vetoed_file = "$base/$vetoed_file";
116 push @vetoed, $vetoed_file; }
117 if (/(?:^|\r)\s*\\(?:end|bye)(?:\s|$)/) {
118 $maybe_tex_priority = 1; }
119 if (/\\(?:end|bye)(?:\s|$)/) {
120 $maybe_tex_priority2 = 1; }
121 if (/\\input *(?:harv|lanl)mac/ || /\\input\s+phyzzx/) {
122 $Main_TeX_likelihood{$tex_file} = 1; last TEX_FILE_TRAVERSAL; } # Mac TeX
123 if (/beginchar\(/) {
124 $Main_TeX_likelihood{$tex_file} = 0; last TEX_FILE_TRAVERSAL; } # MetaFont
125 if (/(?:^|\r)\@(?:book|article|inbook|unpublished)\{/i) {
126 $Main_TeX_likelihood{$tex_file} = 0; last TEX_FILE_TRAVERSAL; } # BibTeX
127 if (/^begin \d{1,4}\s+[^\s]+\r?$/) {
128 if ($maybe_tex_priority) {
129 $Main_TeX_likelihood{$tex_file} = 2; last TEX_FILE_TRAVERSAL; } # TeX Priority
130 if ($maybe_tex) {
131 $Main_TeX_likelihood{$tex_file} = 1; last TEX_FILE_TRAVERSAL; } # TeX
132 $Main_TeX_likelihood{$tex_file} = 0; last TEX_FILE_TRAVERSAL; } # UUEncoded or PC
133 if (m/paper deliberately replaced by what little/) {
134 $Main_TeX_likelihood{$tex_file} = 0; last TEX_FILE_TRAVERSAL; }
135 }
136 close $FILE_TO_GUESS or (print STDERR "couldn't close file: $!");
137 if (!defined $Main_TeX_likelihood{$tex_file}) {
138 if ($maybe_tex_priority) {
139 $Main_TeX_likelihood{$tex_file} = 2; }
140 elsif ($maybe_tex_priority2) {
141 $Main_TeX_likelihood{$tex_file} = 1.5; }
142 elsif ($maybe_tex) {
143 $Main_TeX_likelihood{$tex_file} = 1; }
144 else {
145 $Main_TeX_likelihood{$tex_file} = 0; }
146 }
147 }
148 # Veto files that were e.g. arguments of \input macros
149 for my $filename (@vetoed) {
150 delete $Main_TeX_likelihood{$filename}; }
151 # Examine only the max-scoring candidates (if any)
152 my @files_by_likelihood = sort { $Main_TeX_likelihood{$b} <=> $Main_TeX_likelihood{$a} }
153 grep { $Main_TeX_likelihood{$_} > 0 } keys %Main_TeX_likelihood;
154 if (@files_by_likelihood) {
155 my $max_likelihood = $Main_TeX_likelihood{ $files_by_likelihood[0] };
156 @files_by_likelihood = grep { $Main_TeX_likelihood{$_} == $max_likelihood } @files_by_likelihood;
157 # Special heuristic 1: If we have multiple "best" candidates, prefer those closer to the archive root '/'
158 if (scalar(@files_by_likelihood) > 1) {
159 my $min_count = 100;
160 foreach my $file (@files_by_likelihood) {
161 my $count = $file =~ tr/\///;
162 $Main_TeX_level{$file} = $count;
163 $min_count = $count if $min_count > $count; }
164 @files_by_likelihood = grep { $Main_TeX_level{$_} == $min_count } @files_by_likelihood; }
165 # Special heuristic 2: prefer candidates with PDF-like \includegraphics
166 if (scalar(@files_by_likelihood) > 1) {
167 my @pdf_includes = heuristic_check_for_pdftex(@files_by_likelihood);
168 @files_by_likelihood = @pdf_includes if @pdf_includes; }
169 # Special heuristic 3: prefer "best" candidates with a .bbl file
170 if (scalar(@files_by_likelihood) > 1) {
171 my @with_bbl = grep { my $base = $_; $base =~ s/\.tex$//; -e "$base.bbl"; } @files_by_likelihood;
172 @files_by_likelihood = @with_bbl if @with_bbl; }
173 # Special heuristic 4 ?!: Sometimes in arXiv the decision is made in an unclear manner
174 # (example: see 2112.08935 v1, which has equally good main.tex and bare_adv.tex)
175 # so, for now, err on the side of preferring one of the extremely common names, when they are available at highest score.
176 if (scalar(@files_by_likelihood) > 1) {
177 my @common_name = grep { /(^|\W)(?:main|ms|paper)\.tex$/ } @files_by_likelihood;
178 @files_by_likelihood = @common_name if @common_name; }
179 # Final heuristic: tie-breaker is lexicographical order
180 if (scalar(@files_by_likelihood) > 1) {
181 @files_by_likelihood = sort { $a cmp $b } @files_by_likelihood; }
182 # set the winner for main source
183 $main_source = shift @files_by_likelihood; }
184  
185 # If failed, clean up sandbox directory.
186 rmtree($sandbox_directory) unless $main_source;
187 # Return the main source from the unpacked files in the sandbox directory (or undef if failed)
188 return $main_source; }
189  
190 sub heuristic_check_for_pdftex {
191 my @filenames = @_;
192 my @pdf_includes = ();
193 for my $tex_file (@filenames) {
194 my $is_open = open(my $TEX_FH, '<', $tex_file);
195 if (!$is_open) {
196 print STDERR "failed to open '$tex_file' to guess its format: $!. Continuing.\n";
197 next; }
198 local $/ = "\n";
199 my $pdfoutput_checks = 5;
200 while (<$TEX_FH>) {
201 # Check from arXiv::FileGuess, line 317
202 # https://metacpan.org/release/SIMEON/arXiv-1.01/source/lib/arXiv/FileGuess.pm#L317
203 if (/^[^%]*\\includegraphics[^%]*\.(?:pdf|png|gif|jpg)\s?\}/i ||
204 ($pdfoutput_checks >= 0 && /^[^%]*\\pdfoutput(?:\s+)?=(?:\s+)?1/)) {
205 push(@pdf_includes, $tex_file);
206 last; }
207 $pdfoutput_checks-- if $pdfoutput_checks; }
208 close $TEX_FH or (print STDERR "couldn't close file: $!"); }
209 return @pdf_includes; }
210  
211 # Options:
212 # whatsout: determine what shape and size we want to pack into
213 # admissible: document (default), fragment, math, archive
214 # siteDirectory: the directory to compress into a ZIP archive
215 # collection: the collection of documents to be packed
216 sub pack_collection {
217 my (%options) = @_;
218 my @packed_docs;
219 my $whatsout = $options{whatsout};
220 my @docs = @{ $options{collection} };
221 # Archive once if requested
222 if ($whatsout =~ /^archive/) {
223 my $archive = get_archive($options{siteDirectory}, $whatsout);
224 # TODO: Error API should be integrated once generalized to Util::
225 #Fatal("I/O", $self, $docs[0], "Writing archive to IO::String handle failed") unless defined $archive;
226 print STDERR "Fatal:I/O:Archive Writing archive to IO::String handle failed" unless defined $archive;
227 return ($archive); }
228 # Otherwise pack each document passed
229 foreach my $doc (@docs) {
230 next unless defined $doc;
231 if ((!$whatsout) || ($whatsout eq 'document')) {
232 push @packed_docs, $doc; } # Document is no-op
233 elsif ($whatsout eq 'fragment') {
234 # If we want an embedable snippet, unwrap to body's "main" div
235 push @packed_docs, get_embeddable($doc); }
236 elsif ($whatsout eq 'math') {
237 # Math output - least common ancestor of all math in the document
238 push @packed_docs, get_math($doc);
239 unlink('LaTeXML.cache'); }
240 else { push @packed_docs, $doc; } }
241 return @packed_docs; }
242  
243 ## Helpers for pack_collection:
244 sub get_archive {
245 my ($directory, $whatsout) = @_;
246 # Zip and send back
247 my $archive = Archive::Zip->new();
248 opendir(my $dirhandle, $directory)
249 # TODO: Switch to Error API
250 # or Fatal('expected', 'directory', undef,
251 # "Expected a directory to archive '$directory':", $@);
252 or (print STDERR 'Fatal:expected:directory Failed to compress directory \'$directory\': $@');
253 my @entries = grep { /^[^.]/ } readdir($dirhandle);
254 closedir $dirhandle;
255 my @files = grep { !/$archive_file_exclusion_regex/ && -f pathname_concat($directory, $_) } @entries;
256 my @subdirs = grep { -d File::Spec->catdir($directory, $_) } @entries;
257 # We want to first add the files instead of simply invoking ->addTree on the top level
258 # without ANY file attributes at all,
259 # since EPUB is VERY picky about the first entry in the archive starting at byte 38 (file 'mimetype')
260 @files = sort @files;
261 my @nomime_files = grep { !/^mimetype$/ } @files;
262 if (scalar(@nomime_files) != scalar(@files)) {
263 @files = ('mimetype', @nomime_files); }
264 foreach my $file (@files) {
265 local $/ = undef;
266 my $FH;
267 my $pathname = pathname_concat($directory, $file);
268 open $FH, "<", $pathname
269 # TODO: Switch to Error API
270 #or Fatal('I/O', $pathname, undef, "File $pathname is not readable.");
271 or (print STDERR "Fatal:I/O:$pathname File $pathname is not readable.");
272 my $file_contents = <$FH>;
273 close($FH);
274 # Compress all files except mimetype
275 my $compression = ($file eq 'mimetype' ? COMPRESSION_STORED : COMPRESSION_DEFLATED);
276 $archive->addString($file_contents, $file,)->desiredCompressionMethod($compression); }
277  
278 foreach my $subdir (sort @subdirs) {
279 my $current_dir = File::Spec->catdir($directory, $subdir);
280 $archive->addTree($current_dir, $subdir, sub { !/$archive_file_exclusion_regex/ }, COMPRESSION_DEFLATED); }
281  
282 if (defined $ENV{SOURCE_DATE_EPOCH}) {
283 for my $member ($archive->members()) {
284 $member->setLastModFileDateTimeFromUnix($ENV{SOURCE_DATE_EPOCH}); } }
285  
286 my $payload;
287 if ($whatsout =~ /^archive(::zip)?$/) {
288 my $content_handle = IO::String->new($payload);
289 undef $payload unless ($archive->writeToFileHandle($content_handle) == AZ_OK); }
290 elsif ($whatsout eq 'archive::zip::perl') {
291 $payload = $archive; }
292 return $payload; }
293  
294 sub get_math {
295 my ($doc) = @_;
296 my $math_xpath = '//*[local-name()="math" or local-name()="Math"]';
297 return unless defined $doc;
298 my @mnodes = $doc->findnodes($math_xpath);
299 my $math_count = scalar(@mnodes);
300 if (!$math_count) { # If no real math nodes, look for math image nodes
301 my $math_img_xpath = '//*[local-name()="img" and contains(@class,"ltx_Math")]';
302 @mnodes = $doc->findnodes($math_img_xpath);
303 $math_count = scalar(@mnodes);
304 }
305 if (!$math_count) {
306 return get_embeddable($doc); }
307 my $math = $mnodes[0];
308 if ($math_count > 1) {
309 my $math_found = 0;
310 while ($math_found != $math_count) {
311 $math_found = $math->findnodes('.' . $math_xpath)->size;
312 $math_found++ if ($math->localname =~ /^math$/i);
313 $math = $math->parentNode if ($math_found != $math_count);
314 }
315 $math = $math->parentNode while ($math->nodeName =~ '^t[rd]$'); }
316 if ($math) {
317 my $imagesrc = $math->getAttribute('imagesrc') || $math->getAttribute('src');
318 if ($imagesrc && $imagesrc =~ /[.]svg$/) {
319 # Return the SVG directly
320 $math = LaTeXML::Common::XML::Parser->new()->parseFile($imagesrc);
321 $math = $math && $math->getDocumentElement;
322 }
323 # Copy over document namespace declarations:
324 # NOTE: This copies ALL the namespaces, not just the needed ones!
325 foreach ($doc->getDocumentElement->getNamespaces) {
326 $math->setNamespace($_->getData, $_->getLocalName, 0); } }
327 return $math; }
328  
329 sub get_embeddable {
330 my ($doc) = @_;
331 return unless defined $doc;
332 my ($embeddable) = $doc->findnodes('//*[contains(@class,"ltx_document")]');
333 if ($embeddable) {
334 # Only one child? Then get it, must be a inline-compatible one!
335 while (($embeddable->nodeName eq 'div') && (scalar(@{ $embeddable->childNodes }) == 1) &&
336 ($embeddable->getAttribute('class') =~ /^ltx_(page_(main|content)|document|para|header)$/) &&
337 (!defined $embeddable->getAttribute('style'))) {
338 if (defined $embeddable->firstChild) {
339 $embeddable = $embeddable->firstChild; }
340 else {
341 last; }
342 }
343 # Is the root a <p>? Make it a span then, if it has only math/text/spans - it should be inline
344 # For MathJax-like inline conversion mode
345 # TODO: Make sure we are schema-complete wrt nestable inline elements, and maybe find a smarter way to do this?
346 if (($embeddable->nodeName eq 'p') && ((@{ $embeddable->childNodes }) == (grep { $_->nodeName =~ /math|text|span/ } $embeddable->childNodes))) {
347 $embeddable->setNodeName('span');
348 $embeddable->setAttribute('class', 'text');
349 }
350  
351 # Copy over document namespace declarations:
352 foreach ($doc->getDocumentElement->getNamespaces) {
353 $embeddable->setNamespace($_->getData, $_->getLocalName, 0);
354 }
355 # Also, copy RDFa attributes:
356 foreach my $rdfa_attr (qw(prefix property content resource about typeof rel rev datatype)) {
357 if (my $rdfa_value = $doc->getDocumentElement->getAttribute($rdfa_attr)) {
358 $embeddable->setAttribute($rdfa_attr, $rdfa_value); } }
359 }
360 return $embeddable || $doc; }
361  
362 1;
363  
364 __END__