lib/Data/Edit/Xml/Xref.pm

filter: Statementstmt 88.2% 15 / 17 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 6 / 6 Podpod n/a totaltotal 91.3% 21 / 23 CC 1 SCAR 0.5
CC
1
Cov
88%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
↑ Summary
1 #!/usr/bin/perl -I/home/phil/perl/cpan/DataEditXml/lib/ -I/home/phil/perl/cpan/DataTableText/lib/ -I/home/phil/perl/cpan/DitaGBStandard/lib/
2 #-------------------------------------------------------------------------------
3 # Cross reference Dita XML, match topics and ameliorate missing references.
4 # Philip R Brenan at gmail dot com, Appa Apps Ltd Inc, 2016-2019
5 # Improvements and maxzoomin
6 #-------------------------------------------------------------------------------
7 # Check for image formats that will not display in a browser
8 # Do not consider companion files!
9 # Images that are referenced by topics which are not referenced by bookmaps showup as referenced
10 # It should be possible to remove reportImages by using generic references instead
11 # Conref processing in reportReferencesFromBookmaps
12 # Fix xref external/scope and eliminate error count if fixbadrefs in operation.
13 # Add labels to ditaRefs processing so that references to labels are also fixed
14 # Add xref expansion from id in file as it is a pain to code up the full details by hand
15 # Find topics that have no text in them per: PS2-617
16 # Need test for changeBadXrefToPh
17 # Unique target needs tests
18 # Create list of images found in input folder
19 # Conrefs report should use targets/ to update the conref file so conrefs fixed by fixDitaRefs are considered
20  
21 package Data::Edit::Xml::Xref;
22 our $VERSION = 20200424;
23 1 ▶use v5.26;
Subroutine BEGIN: called CC=1 SCAR=0.0
24 1 ▶use warnings FATAL => qw(all);
Subroutine BEGIN: called CC=1 SCAR=0.0
25 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
26 1 ▶use Carp qw(confess cluck);
Subroutine BEGIN: called CC=1 SCAR=0.0
27 1 ▶use Data::Dump qw(dump);
Subroutine BEGIN: called CC=1 SCAR=0.0
28 0 ▶use Data::Edit::Xml;
Subroutine BEGIN: called CC=1 SCAR=2.6
29 use Data::Table::Text qw(:all);
30 use Dita::GB::Standard;
31 use Storable qw(store retrieve);
32 use Time::HiRes qw(time);
33 use utf8;
34  
35 #sub improvementLength {80} #P Maximum length of the test of an improvement suggestion
36 sub classificationMapSuffix{q(_classification.ditamap)} #P Suffix to add to map files to create corresponding classification map file
37  
38 #D1 Cross reference # Check the cross references in a set of Dita files and report the results.
39  
40 sub newXref(%) #P Create a new cross referencer
41 {my (%attributes) = @_; # Attributes
42  
43 my $xref = genHash(__PACKAGE__, # Attributes used by the Xref cross referencer.
44 addNavTitles => undef, #I If true, add navtitle to outgoing bookmap references to show the title of the target topic.
45 allowUniquePartialMatches => undef, # Allow unique partial matches - i.e ignore the stuff to the right of the # in a reference if doing so produces a unique result. This feature has been explicitly disabled for conrefs (PS2-561) and might need to be disabled for other types of reference as well.
46 attributeCount => {}, # {file}{attribute name} == count of the different xml attributes found in the xml files.
47 attributeNamesAndValuesCount => {}, # {file}{attribute name}{value} = count
48 author => {}, # {file} = author of this file.
49 badGuidHrefs => {}, # Bad conrefs - all.
50 badNavTitles => {}, # Details of nav titles that were not resolved
51 badReferencesCount => 0, # The number of bad references at the start of the run - however depending on what options were chosen Xref might ameliorate these bad references and thereby reduce this count.
52 badTables => [], # Array of tables that need fixing.
53 badXml1 => {}, # [Files] with a bad xml encoding header on the first line.
54 badXml2 => {}, # [Files] with a bad xml doc type on the second line.
55 baseFiles => {}, # {base of file name}{full file name}++ Current location of the file via uniqueness guaranteed by the GB standard
56 baseTag => {}, # Base Tag for each file
57 bookMapRefs => {}, # {bookmap full file name}{href}{navTitle}++ References from bookmaps to topics via appendix, chapter, bookmapref.
58 changeBadXrefToPh => undef, #I Change xrefs being placed in B<M3> by L<fixBadRefs> to B<ph>.
59 classificationMaps => undef, #I Create classification maps if true
60 conRefs => {}, # {file}{href}{tag}++ : conref source detail
61 createReports1 => [], # Reports requested before references fixed
62 createReports2 => [], # Reports requested after references fixed
63 currentFolder => currentDirectory, # The current working folder used to make absolute file names from relative ones
64 deleteUnusedIds => 0, #I Delete ids (except on topics) that are not referenced in any reference in the corpus regardless of the file component of any such reference.
65 deguidize => undef, #I Set true to replace guids in dita references with file name. Given reference B<g1#g2/id> convert B<g1> to a file name by locating the topic with topicId B<g2>. This requires the guids to be genuinely unique. SDL guids are thought to be unique by language code but the same topic, translated to a different language might well have the same guid as the original topic with a different language code: =(de|en|es|fr). If the source is in just one language then the guid uniqueness is a reasonable assumption. If the conversion can be done in phases by language then the uniqueness of guids is again reasonably assured. L<Data::Edit::Xml::Lint> provides an alternative solution to deguidizing by using labels to record the dita reference in the input corpus for each id encountered, these references can then be resolved in the usual manner by L<Data::Edit::Xml::Lint::relint>.
66 docType => {}, # {file} == docType: the docType for each xml file.
67 duplicateIds => {}, # [file, id] Duplicate id definitions within each file.
68 duplicateTopicIds => {}, # Duplicate topic ids
69 duplicateTopicIds => {}, # [topicId, [files]] Files with duplicate topic ids - the id on the outermost tag.
70 emptyTopics => {}, # {file} : topics where the *body is empty.
71 errors => 0, # Number of significant errors as reported in L<statusLine> or 0 if no such errors found
72 exteriorMaps => {}, # {exterior map} : maps that are not referenced by another map
73 fileExtensions => [qw(.dita .ditamap .xml .fodt)], # Default file extensions to load
74 fixBadRefs => undef, #I Fix any remaining bad references after any all allowed attempts have been made to fix failing references by moving the failing reference to the B<xtrf> attribute i.e. placing it in B<M3> possibly renaming the tag to B<ph> if L<changeBadXrefToPh> is in effect as well.
75 fixDitaRefs => undef, #I Fix references in a corpus of L<Dita> documents that have been converted to the L<GBStandard> and whose target structure has been written to the named folder.
76 fixedFolder => undef, #I Fixed files are placed in this folder.
77 fixedFolderTemp => undef, #I Fixed files are placed in this folder if we are on aws but nit the session leader - this folder is then copied back to L<fixedFolder> on the session leader.
78 fixedRefsBad => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> which were moved to the "xtrf" attribute as requested by the L<fixBadHrefs|/fixBadHrefs> attribute because the reference was invalid and could not be improved by L<deguidization|/deguidize>.
79 fixedRefsGB => [], # [] files fixed to the Gearhart-Brenan file naming standard
80 fixedRefsGood => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> which were invalid but have been fixed by L<deguidizing|/deguidize> them to a valid file name.
81 fixedRefsNoAction => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> for which no action was taken.
82 fixRefs => {}, # {file}{ref} where the href or conref target is not valid.
83 fixRelocatedRefs => undef, #I Fix references to topics that have been moved around in the out folder structure assuming that all file names are unique which they will be if they have been renamed to the GB Standard.
84 fixXrefsByTitle => undef, #I Try to fix invalid xrefs by the Gearhart Title Method enhanced by the Monroe map method if true
85 flattenFiles => {}, # {old full file name} = file renamed to Gearhart-Brenan file naming standard
86 flattenFolder => undef, #I Files are renamed to the Gearhart standard and placed in this folder if set. References to the unflattened files are updated to references to the flattened files. This option will eventually be deprecated as the Dita::GB::Standard is now fully available allowing files to be easily flattened before being processed by Xref.
87 getFileUrl => qq(/cgi-bin/uiSelfServiceXref/client.pl?getFile=), #I A url to retrieve a specified file from the server running xref used in generating html reports. The complete url is obtained by appending the fully qualified file name to this value.
88 goodImageFiles => {}, # {file}++ : number of references to each good image
89 goodNavTitles => {}, # Details of nav titles that were resolved.
90 guidHrefs => {}, # {file}{href} = location where href starts with GUID- and is thus probably a guid.
91 guidToFile => {}, # {topic id which is a guid} = file defining topic id.
92 hrefUrlEncoding => {}, # Hrefs that need url encoding because they contain white space.
93 html => undef, #I Generate html version of reports in this folder if supplied
94 idNotReferenced => {}, # {file}{id}++ - id in a file that is not referenced
95 idReferencedCount => {}, # {file}{id}++ - the number of times this id in this file is referenced from the rest of the corpus
96 ids => {}, # {file}{id} - id definitions across all files.
97 idsRemoved => {}, # {id}++ : Ids removed from all files
98 idTags => {}, # {file}{id}[tag] The tags associated with each id in a file - there might be more than one if the id is duplicated
99 images => {}, # {file}{href} Count of image references in each file.
100 imagesReferencedFromBookMaps => {}, # {bookmap full file name}{full name of image referenced from topic referenced from bookmap}++
101 imagesReferencedFromTopics => {}, # {topic full file name}{full name of image referenced from topic}++
102 imagesToRefferingBookMaps => {}, # {image full file name}{bookmap full file name}++ : images to referring bookmaps
103 indexWords => undef, #I Index words to topics and topics to words if true.
104 indexWordsFolder => undef, #I Folder into which to save words to topic and topics to word indexes if L<indexWords> is true.
105 indexedWords => {}, # {word}{full file name of topic the words occurs in}.
106 inputFiles => [], # Input files from L<inputFolder|/inputFolder>.
107 inputFileToTargetTopics => {}, # {input file}{target file}++ : Tells us the topics an input file was split into
108 inputFolderImages => {}, # {full image file name} for all files in input folder thus including any images resent
109 inputFolder => undef, #I A folder containing the dita and ditamap files to be cross referenced.
110 ltgt => {}, # {text between &lt; and &gt}{filename} = count giving the count of text items found between &lt; and &gt;
111 matchTopics => undef, #I Match topics by title and by vocabulary to the specified confidence level between 0 and 1. This operation might take some time to complete on a large corpus.
112 maximumNumberOfProcesses => numberOfCpus(8), #I Maximum number of processes to run in parallel at any one time with a sensible default.
113 maxZoomIn => undef, #I Optional hash of names to regular expressions to look for in each file
114 maxZoomOut => {}, # Results from L<maxZoomIn|/maxZoomIn> where {file name}{regular expression key name in L<maxZoomIn|/maxZoomIn>}++
115 md5Sum => {}, # MD5 sum for each input file.
116 md5SumDuplicates => {}, # {md5sum}{file}++ : md5 sums with more than one file
117 missingImageFiles => {}, # [file, href] == Missing images in each file.
118 missingTopicIds => {}, # Missing topic ids.
119 noHref => {}, # Tags that should have an href but do not have one.
120 notReferenced => {}, # {file name} Files in input area that are not referenced by a conref, image, bookmapref or xref tag and are not a bookmap.
121 olBody => {}, # The number of ol under body by file
122 originalSourceFileAndIdToNewFile => {}, # {original file}{id} = new file: Record mapping from original source file and id to the new file containing the id
123 otherMeta => {}, # {original file}{othermeta name}{othermeta content}++ : the contents of the other meta tags
124 otherMetaDuplicatesSeparately => [], # Duplicate othermeta in bookmaps and topics considered separately
125 otherMetaDuplicatesCombined => [], # Duplicate othermeta in bookmaps with called topics othermeta included
126 otherMetaRemainWithTopic => [], # Othermeta that must stay in the topic
127 otherMetaPushToBookMap => [], # Othermeta that can be pushed to the calling book map
128 otherMetaBookMapsBeforeTopicIncludes=> [], # Bookmap othermeta before topic othermeta has been included
129 otherMetaBookMapsAfterTopicIncludes => [], # Bookmap othermeta after topic othermeta has been included
130 otherMetaConsolidated => {}, # {Name}{Content}++ : consolidated other meta data across entire corpus
131 oxygenProjects => undef, #I Create oxygen project files for each map - the project file will have an extension of .xpr and the same name and path as the map file or the name return by your implementation of: Data::Edit::Xml::Xref::xprName($map) if present.
132 parseFailed => {}, # {file} files that failed to parse.
133 publicId => {}, # {file} = Public id on Doctype
134 references => {}, # {file}{reference}++ - the various references encountered
135 relocatedReferencesFailed => [], # Failing references that were not fixed by relocation
136 relocatedReferencesFixed => [], # Relocated references fixed
137 requestAttributeNameAndValueCounts => undef, #I Report attribute name and value counts
138 requiredCleanUp => undef, # {full file name}{cleanup} = number of required-cleanups
139 reports => undef, #I Reports folder: Xref will write text versions of the generated reports to files in this folder.
140 results => [], # Summary of results table.
141 # sourceFile => undef, # The source file from whic#h this structure was generated.
142 sourceTopicToTargetBookMap => {}, # {input topic cut into multiple pieces} = output bookmap representing pieces
143 statusLine => undef, # Status line summarizing the cross reference.
144 statusTable => undef, # Status table summarizing the cross reference.
145 subjectSchemeMap => undef, #I Create a subject scheme map in the named file
146 suppressReferenceChecks => undef, #I Suppress reference checking - which normally happens by default - but which takes time and might be irrelevant if an earlier xref has already checked all the references.
147 tableDimensions => {}, # {file}{columns}{rows} == count
148 tagCount => {}, # {file}{tags} == count of the different tag names found in the xml files.
149 tagsTextsRatio => undef, # Ratio of tags to text encountered
150 tags => undef, # Number of tags encountered
151 targetFolderContent => {}, # {file} = bookmap file name : the target folder content which shows us where an input file went
152 targetTopicToInputFiles => {}, # {current file} = the source file from which the current file was obtained
153 texts => undef, # Number of texts encountered
154 timeEnded => undef, # Time the run ended
155 timeStart => undef, # Time the run started
156 title => {}, # {full file name} = title of file.
157 titleToFile => {}, # {title}{file}++ if L<fixXrefsByTitle> is in effect
158 topicFlatteningFactor => {}, # Topic flattening factor - higher is better
159 topicFlattening => {}, # {topic}{sources}++ : the source files for each topic that was flattened
160 topicIds => {}, # {file} = topic id - the id on the outermost tag.
161 topicsFlattened => undef, # Number of topics flattened
162 topicsNotReferencedFromBookMaps => {}, # {topic file not referenced from any bookmap} = 1
163 topicsReferencedFromBookMaps => {}, # {bookmap full file name}{topic full file name}++ : bookmaps to topics
164 topicsToReferringBookMaps => {}, # {topic full file name}{bookmap full file name}++ : topics to referring bookmaps
165 urls => {}, # {topic full file name}{url}++ : urls found in each file
166 urlsBad => {}, # {url}{topic full file name}++ : failing urls found in each file
167 urlsGood => {}, # {url}{topic full file name}++ : passing urls found in each file
168 validateUrls => undef, #I Validate urls if true by fetching their headers with L<curl>
169 validationErrors => {}, # True means that Lint detected errors in the xml contained in the file.
170 vocabulary => {}, # The text of each topic shorn of attributes for vocabulary comparison.
171 xrefBadFormat => {}, # External xrefs with no format=html.
172 xrefBadScope => {}, # External xrefs with no scope=external.
173 xRefs => {}, # {file}{href}++ Xrefs references.
174 xrefsFixedByTitle => [], # Xrefs fixed by locating a matching topic title from their text content.
175 );
176  
177 loadHash($xref, @_); # Load attributes complaining about any invalid ones
178 } # newXref
179  
180 sub xref2(%) #P Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>
181 {my (%attributes) = @_; # Attributes of cross referencer
182 my ($xref) = newXref(@_); # Cross referencer
183 $xref->timeStart = time; # Start time
184  
185 $xref->inputFolder or confess "Please supply a value for: inputFolder";
186 $xref->inputFolder =~ s(\/+\Z) (\/)gs; # Cleanup path names
187 $xref->inputFolder = # Make input folder absolute
188 absFromAbsPlusRel($xref->currentFolder, $xref->inputFolder)
189 if $xref->inputFolder !~ m(\A/);
190  
191 $xref->reports or confess "Please supply a value for: reports";
192  
193 if (1) # Write title and some of the parameters
194 {my $r = $xref->reports;
195 owf(fpe($r, qw(xref_parameter_settings txt)), dump($xref)) if $r; # Print all parameters
196  
197 my $i = $xref->inputFolder;
198 lll "Xref started on folder: $i, reports: $r" # Show that we are starting unless in development
199 unless $i =~ m(/tmp/);
200 }
201  
202 if (my $d = $xref->fixDitaRefs) # Fully qualify and validate targets folder
203 {$xref->fixDitaRefs = fullyQualifiedFile($d) ? $d : # Fully qualified target folder name
204 absFromAbsPlusRel($xref->currentFolder, $d); # Get fully qualified target folder if necessary
205 if (!-d $d) # Check targets folder is available
206 {confess "Targets folder does not exist: fixDitaRefs=>$d";
207 }
208 my @d = searchDirectoryTreesForMatchingFiles($d);
209 @d or confess "Targets folder is empty: fixDitaRefs=>$d";
210 }
211  
212 if (!$xref->fixedFolder and $xref->fixDitaRefs || $xref->fixRelocatedRefs) # Fixing references
213 {my $s = join ' and ',
214 ($xref->fixDitaRefs ? "fixDitaRefs" : (),
215 $xref->fixRelocatedRefs ? "fixRelocatedRefs" : ());
216 warn "No fixedFolder attribute specified yet $s specified.\n".
217 "Assuming inputFolder for fixedFolder.\n";
218 $xref->fixedFolder = $xref->inputFolder;
219 }
220  
221 if (my $f = $xref->fixedFolder) # Fixing references in a folder other than the input folder requires us to copy the files across so we can make differential changes
222 {if (my $i = $xref->inputFolder)
223 {if ($f ne $i)
224 {copyFolder($i, $f);
225 }
226 }
227 }
228  
229 $xref->fixedFolderTemp //= fpd(temporaryFolder); # Preserve this temporary value across session instances
230  
231 my @series = # Must be done in series at the start
232 (q(loadInputFiles),
233 q(analyzeInputFiles),
234 q(reportReferencesFromBookMaps), # Used by fixReferences to get bookmap references
235 $xref->deguidize ? q(reportGuidsToFiles) : (), # Used by addNavTitleToMaps
236 q(checkReferences), # Check all the references
237 q(createReportsInParallel1), # Create reports that do not rely on fixed references
238 q(fixReferences), # Fix any failing references
239 q(createReportsInParallel2), # Create reports that rely on fixed references
240 );
241  
242 my @parallel1 = # Create reports in parallel that do not rely on fixed references
243 (q(reportXml1),
244 q(reportXml2),
245 q(reportDuplicateIds),
246 q(reportDuplicateTopicIds),
247 q(reportNoHrefs),
248 q(reportTables),
249 q(reportParseFailed),
250 q(reportAttributeCount),
251 q(reportLtGt),
252 q(reportTagCount),
253 q(reportTagsAndTextsCount),
254 q(reportDocTypeCount),
255 q(reportFileExtensionCount),
256 q(reportFileTypes),
257 q(reportValidationErrors),
258 q(reportGuidHrefs),
259 q(reportExternalXrefs),
260 q(reportTopicDetails),
261 q(reportTopicReuse),
262 q(reportMd5Sum),
263 q(reportOlBody),
264 q(reportHrefUrlEncoding),
265 q(reportFixRefs),
266 q(reportSourceFiles),
267 q(reportOtherMeta),
268 q(createSubjectSchemeMap),
269 q(reportTopicsNotReferencedFromBookMaps),
270 q(reportTableDimensions),
271 q(reportExteriorMaps),
272 q(createClassificationMaps),
273 q(reportIdRefs),
274 q(reportEmptyTopics),
275 q(reportConRefMatching),
276 q(reportPublicIds),
277 q(reportRequiredCleanUps),
278 );
279  
280 if ($xref->addNavTitles) # Add nav titles to bookmaps if requested
281 {push @parallel1, q(addNavTitlesToMaps);
282 }
283  
284 if ($xref->requestAttributeNameAndValueCounts) # Report attribute name and value counts
285 {push @parallel1, q(reportAttributeNameAndValueCounts);
286 }
287  
288 if ($xref->flattenFolder) # Fix file names to the Gearhart-Brenan file naming standard
289 {push @parallel1, q(fixFilesGB)
290 }
291  
292 if ($xref->matchTopics) # Topic matching reports
293 {push @parallel1, q(reportSimilarTopicsByTitle),
294 q(reportSimilarTopicsByVocabulary);
295 }
296  
297 if ($xref->indexWords) # Word indexing required
298 {push @parallel1, q(reportWordsByFile);
299 $xref->indexWordsFolder or confess "Please set indexWordsFolder";
300 }
301  
302 if ($xref->validateUrls) # Validate urls
303 {push @parallel1, q(reportUrls);
304 }
305  
306 push $xref->createReports1->@*, @parallel1; # Create reports that do not rely on references being fixed
307  
308 my @parallel2 = # Create reports in parallel after references fixed
309 (q(removeUnusedIds), # Count and/or remove unused ids
310 q(reportImages), # Images relies on fixed references to locate images
311 );
312  
313 if ($xref->oxygenProjects) # Create oxygen project files if requested
314 {push @parallel2, q(createOxygenProjectMapFiles);
315 }
316  
317 push $xref->createReports2->@*, @parallel2; # Create reports that rely on references being fixed
318  
319 if (1) # Perform phases in series that must be run in series
320 {my @times;
321  
322 for my $phase(@series) # Each phase in series
323 {my $startTime = time;
324 #lll "Xref phase $phase";
325  
326 $xref->$phase; # Execute phase
327  
328 #mmm "Xref phase $phase";
329 push @times, [$phase, sprintf("%12.4f", time - $startTime)]; # Phase time
330 }
331  
332 my $delta = sprintf("%.3f seconds", time - $xref->timeStart); # Time so far
333  
334 formatTables($xref, [sort {$$b[1] <=> $$a[1]} @times], # Update after each phase so we can see progress on long running jobs
335 columns => <<END,
336 Phase Xref processing phase
337 Time_Seconds Time in seconds taken by this processing phase
338 END
339 title => qq(Processing phases elapsed times in descending order),
340 head => <<END,
341 Xref phases took the following times on DDDD
342  
343 Total run time: $delta
344 END
345 file => fpe(q(timing), qw(xref_phases txt))); # Write phase times
346 }
347  
348 formattedTablesReport # Needs update for reports in parallel
349 (title=>q(Reports available),
350 head=><<END,
351 NNNN reports available on DDDD
352  
353 Sorted by title
354 END
355 file=>fpe($xref->reports, qw(reports txt))) if $xref->reports;
356  
357 if (1) # Summarize
358 {my @o;
359 my $save = sub
360 {my ($levels, $field, $plural, $single) = @_;
361 my $n = &countLevels($levels, $xref->{$field});
362 push @o, [$n, $plural] if $n > 1;
363 push @o, [$n, $single // ($plural =~ s(s\Z) ()gsr)] if $n == 1;
364 };
365  
366 # $save->(1, "badConRefsList", q(conrefs));
367 # $save->(1, "badConRefs", q(files with bad conrefs), q(file with bad conrefs));
368 $save->(1, "badGuidHrefs", q(invalid guid hrefs));
369 $save->(1, "badTables", q(tables));
370 $save->(1, "badXml1", q(first lines));
371 $save->(1, "badXml2", q(second lines));
372 # $save->(1, "badXRefsList", q(xrefs));
373 # $save->(1, "badXRefs", q(files with bad xrefs), q(file with bad xrefs));
374 $save->(2, "duplicateIds", q(duplicate ids));
375 $save->(1, "fixedRefsBad", q(refs)); # Unable to resolve these references - L<fixBadRefs> can be used to ameliorate them.
376 $save->(1, "hrefUrlEncoding", q(href url encoding), q(href url encoding));
377 $save->(2, "md5SumDuplicates", q(duplicate files));
378 $save->(1, "missingImageFiles", q(image refs));
379 $save->(1, "missingTopicIds", q(missing topic ids));
380 $save->(1, "notReferenced", q(files not referenced), q(file not referenced));
381 $save->(1, "parseFailed", q(files failed to parse), q(file failed to parse));
382 $save->(1, "duplicateTopicIds", q(duplicate topic ids));
383 $save->(2, "noHref", q(hrefs missing), q(href missing));
384 $save->(2, "validationErrors", q(validation errors)); # Needs testing
385 $save->(2, "xrefBadFormat", q(External xrefs with no format=html));
386 $save->(2, "xrefBadScope", q(External xrefs with no scope=external));
387 $save->(1, "urlsBad", q(urls));
388  
389 my $files = $xref->inputFiles->@*;
390  
391 my $statusLine = $xref->statusLine = @o ? join " ", # Status line
392 "Xref:", join ", ",
393 map {join " ", @$_}
394 sort
395 {return $$a[1] cmp $$b[1] if $$b[0] == $$a[0];
396 $$b[0] <=> $$a[0]
397 }
398 @o : qq(Xref: processed $files files, found no errors);
399  
400 lll $statusLine; # Print Xref results summary with date and time
401  
402 $xref->errors = @o;
403  
404 $xref->statusTable = formatTable
405 ([sort {$$b[0] <=> $$a[0]} @o], [qw(Count Condition)]); # Summary in status form
406 $xref->results = \@o; # Save status line components
407 }
408  
409 $xref->timeEnded = time; # Run ended time
410  
411 formatTables($xref, [[$xref->timeStart, $xref->timeEnded, # Write run times
412 $xref->timeEnded - $xref->timeStart]],
413 columns => <<END,
414 Start_Time Start time of the run
415 End_Time End time of the run
416 Elapsed_Time Xref took this many seconds to run
417 END
418 title => qq(Run times in seconds),
419 head => qq(Xref took the following time to run on DDDD),
420 file => fpe(q(timing), qw(run txt)));
421  
422 formatHtmlTablesIndex($xref->reports, q(), $xref->getFileUrl, 2); # Create an index of html files for use as an initial page of Xref results - this is done in Dita.pm as well but we need it here too so it can be tested.
423  
424 lll "Xref finished on folder: ", $xref->inputFolder unless # Show that we have finished unless in development
425 $xref->inputFolder =~ m(/tmp/);
426  
427 $xref # Return Xref results
428 } # xref2
429  
430 sub xref(%) # Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>.
431 {my (%attributes) = @_; # Cross referencer attribute value pairs
432 my $x = callSubInParallel {xref2(%attributes)}; # Process in alternate process to avoid memory fragmentation
433 newXref %$x # Return blessed results after creating in child process
434 }
435  
436 sub createReportsInParallel($@) #P Create reports in parallel
437 {my ($xref, @reports) = @_; # Cross referencer, reports to be run
438  
439 runInParallel($xref->maximumNumberOfProcesses,
440 sub # Execute each report in parallel
441 {my ($report) = @_;
442 my ($result) = my @result = ($xref->$report); # Check that the value returned by the report is a single hash reference
443  
444 formatHtmlAndTextTablesWaitPids; # Wait for report tables to be formatted in parallel
445  
446 if (scalar(@result) != 1) # Check return from multiverse to universe
447 {confess "Phase $report does not return one result";
448 }
449 if ($result =~ m(hash)s)
450 {confess "Phase $report does not return a hash reference";
451 }
452  
453 $result # Return results from report
454 },
455 sub # Decode results
456 {for my $r(@_)
457 {for my $k(sort keys %$r)
458 {$xref->{$k} = $$r{$k};
459 }
460 }
461 },
462 @reports); # Each report to be run parallel
463 }
464  
465 sub createReportsInParallel1() #P Create reports in parallel that do not require fixed references
466 {my ($xref) = @_; # Cross referencer
467 createReportsInParallel($xref, $xref->createReports1->@*)
468 }
469  
470 sub createReportsInParallel2() #P Create reports in parallel that require fixed references
471 {my ($xref) = @_; # Cross referencer
472 createReportsInParallel($xref, $xref->createReports2->@*)
473 }
474  
475 sub countLevels($$) #P Count has elements to the specified number of levels
476 {my ($l, $h) = @_; # Levels, hash
477 if ($l == 0)
478 {return $h;
479 }
480 if ($l == 1)
481 {return scalar keys @$h if ref($h) =~ m(array)i;
482 return scalar keys %$h if ref($h) =~ m(hash)i;
483 }
484 my $n = 0;
485 if (ref($h) =~ m(hash)i)
486 {$n += &countLevels($l-1, $_) for values %$h;
487 }
488 elsif (ref($h) =~ m(array)i)
489 {$n += &countLevels($l-1, $_) for values @$h;
490 }
491 $n
492 }
493  
494 sub externalReference($) #P Check for an external reference
495 {my ($reference) = @_; # Reference to check
496 $reference =~ m(\A(https?:|mailto:|www))is # Check reference
497 }
498  
499 sub fixingRun($) #P A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source. Returns a useful message describing this state of affairs.
500 {my ($xref) = @_; # Cross referencer
501 my @i;
502 for my $i(qw(fixBadRefs fixDitaRefs fixRelocatedRefs fixXrefsByTitle))
503 {push @i, $i if $xref->{$i};
504 }
505  
506 if (@i) # Fixing run
507 {my $i = join ', ', @i;
508 return <<END;
509  
510 Caution: this is a fixing run with options: $i set to on.
511  
512 This report represents the status quo ante - i.e. what things look like at the
513 start of the run. This is useful information for driving development of Xref
514 but confusing if you only want to improve your document corpus.
515  
516 To get the state of play, a posteriori, request a copy of this report as
517 produced after an Xref run with all the options above set to off.
518  
519 END
520 }
521 else # Reporting run
522 {return '';
523 }
524 }
525  
526 sub loadInputFiles($) #P Load the names of the files to be processed
527 {my ($xref) = @_; # Cross referencer
528  
529 my $in = $xref->inputFiles =
530 [searchDirectoryTreesForMatchingFiles
531 $xref->inputFolder, @{$xref->fileExtensions}];
532  
533 if (!$in or @$in == 0) # Complain if there are no input files to analyze
534 {my $i = $xref->inputFolder;
535 my $e = join " ", @{$xref->fileExtensions};
536 my $x = -d $i ? "The input folder does exist." :
537 "The input folder does NOT exist!";
538 confess join '',
539 "No files with the specified file extensions ",
540 "in the specified input folder:\n",
541 "$e\n$i\n$x\n";
542 }
543  
544 my @images = searchDirectoryTreesForMatchingFiles($xref->inputFolder); # Input files
545 $xref->inputFolderImages = {map {fn($_), $_} @images}; # Image file name which works well for images because the md5 sum in their name is probably unique
546 }
547  
548 sub formatTables($$%) #P Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>.
549 {my ($xref, $data, %options) = @_; # Cross referencer, table to be formatted, options
550  
551 $xref && ref($xref) =~ m(xref)i or cluck "No cross referencer"; # Check parameters
552 $data && ref($data) =~ m(array)i or cluck "No data for table";
553  
554 cluck "No file for table" unless $options{file}; # Check for required options
555 cluck "No columns for table" unless $options{columns};
556 cluck "No title for table" unless $options{title};
557 $options{zero} = 1 if $options{facet} and !$options{zero};
558  
559 formatHtmlAndTextTables
560 ($xref->reports, $xref->html, $xref->getFileUrl,
561 $xref->inputFolder, $data, %options);
562 }
563  
564 sub hashOfCountsToArray($) #P Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables>
565 {my ($hash) = @_; # Hash to be converted
566 my $array = [];
567 for my $key(sort keys %$hash)
568 {push @$array, [$$hash{$key}, $key];
569 }
570 $array
571 }
572  
573 sub reportGuidsToFiles($) #P Map and report guids to files
574 {my ($xref) = @_; # Xref results
575 my @r;
576 if (my $xrefTopicIds = $xref->topicIds)
577 {for my $file(sort keys %{$xrefTopicIds}) # Each input file which will be absolute
578 {if (my $topicId = $xrefTopicIds->{$file}) # Topic Id for file - we report missing topicIds in: reportDuplicateTopicIds
579 {next unless $topicId =~ m(\AGUID-)is;
580 $xref->guidToFile->{$topicId} = $file; # Guid Topic Id to file
581 push @r, [$topicId, $file];
582 }
583 }
584 }
585  
586 formatTables($xref, \@r,
587 columns => <<END,
588 Guid The guid being defined
589 File The file that defines the guid
590 END
591 title =>qq(Guid topic definitions),
592 head =>qq(Xref found NNNN guid topic definitions on DDDD),
593 summarize=>1,
594 file =>fpe(q(lists), qw(guidsToFiles txt)));
595 }
596  
597 sub editXml($$$) #P Edit an xml file retaining any existing XML headers and lint trailers
598 {my ($in, $out, $source) = @_; # Input file, output file, source to write
599  
600 my @s = readFile($in); # Read existing source
601  
602 my @h; # Headers if any present
603 if (@s > 0) # Remove header lines using a very basic parse that is not a general solution
604 {if ($s[0] =~ m(\A\<\?xml)is) # First line
605 {push @h, shift @s;
606 if (@s > 0 and $s[0] =~ m(\A<!DOCTYPE)s) # Second line start
607 {push @h, shift @s;
608 while(@s > 0 and $s[0] !~ m(\A\s*<[a-z])i) # Parse to root tag
609 {push @h, shift @s;
610 }
611 }
612 }
613 }
614  
615 my @l; # Lint data if any
616 if (1)
617 {my $state;
618 for my $s(@s)
619 {if (!$state && $s =~ m(\A\<\!\-\-linted\:)s or $state)
620 {push @l, $s;
621 $state++;
622 }
623 }
624 }
625  
626 owf($out, join '', @h, $source, @l) # Insert new source between old headers and trailers
627 }
628  
629 # Fix a file by moving its hrefs and conrefs to the xtrf attribute unless
630 # deguidization is in effect and the guid can be converted into a valid Dita
631 # reference accessing a file in the input corpus.
632 #
633 # If fixRelocatedRefs is in effect: such references are fixed by assuming that
634 # the files mentioned in broken links have been relocated else where in the
635 # folder structure and can be located by base file name alone.
636 #
637 # If fixXrefsByTitle is in effect apply the Gearhart Title Method: fix broken
638 # xrefs by looking for a unique topic with the same title text as the content of
639 # the xref.
640 #
641 # If fixDitaRefs is in effect we are converting Dita to Dita: relink Dita
642 # references that were valid in the input corpus to make them valid again in the
643 # output corpus even after files have been cut out and renamed to the GB Standard.
644 # The targets/ folder provides the mapping between the input and output corpii.
645  
646 sub fixReferencesInOneFile($$) #P Fix one file by moving unresolved references to the xtrf attribute
647 {my ($xref, $sourceFile) = @_; # Xref results, source file to fix
648 my $fixed = newXref(); # Fix results
649 my $node; # The current node we are working with
650 my $attr; # The current attribute we are working with
651 my $ref; # The current reference we are working with
652 my @bad; $fixed->fixedRefsBad = \@bad; # Hrefs that could not be fixed and so were ameliorated by moving them to @xtrf
653 my @good; $fixed->fixedRefsGood = \@good; # Hrefs that were fixed by resolving a Guid
654  
655 my $refDetails = sub # Save details of a reference
656 {my ($r) = @_;
657 my $s = $xref->targetTopicToInputFiles->{$sourceFile}; # The source file(s) from which each target was obtained
658 [$r, $node->tag, $attr, $ref, $sourceFile, sort keys %$s] # Construct reference details
659 };
660  
661 my $bad = sub # Save details of a bad reference
662 {my ($r) = @_;
663 push @bad, my $R = &$refDetails($r);
664 $R
665 };
666  
667 my $good = sub # Save details of a good reference
668 {my ($target, $r) = @_; # Target file, reason
669 my $R = &$refDetails($r);
670 push @good, [@$R[0..3], $target, @$R[4..$#$R]]; # Insert target at correct location
671 $R
672 };
673  
674 my $fixXrefByTitle = sub # Attempt to fix an xref by using its text content to search for a matching title
675 {return undef unless -t $node eq q(xref); # Only works for xrefs
676  
677 my $xTitle = nws($node->stringContent); # Normalized title from xref node
678 if (my $topics = $xref->titleToFile->{$xTitle}) # Find the topics that match the title text content
679 {my $N = keys %$topics; # Matching topics
680  
681 if ($N == 1) # Unique matching topic - the original Gearhart Title Method
682 {my ($path) = keys %$topics;
683 my $rel = relFromAbsAgainstAbs($path, $sourceFile); # Relative file name
684 $node->href = $rel; # Update xref
685 return &$good($path, q(Fixed by Gearhart Title Method)); # Report the fix made
686 }
687 elsif ($N > 1) # Multiple matches
688 {if (my $l = fileLargestSize(sort keys %$topics)) # Boldly choose the topic with the largest size to resolve the ambiguity on the basis that it is probably the most interesting
689 {my $rel = relFromAbsAgainstAbs($l, $sourceFile); # File name of target topic relative to source file
690 $node->href = $rel; # Update reference
691 return &$good($l, q(Fixed by Gearhart Bold Title Method)); # Report the fix made
692 }
693 }
694 }
695 undef # Failed
696 };
697  
698 my $locateUniqueTopicSourceForTargetFile = sub # Unique source file in the input corpus corresponding to the specified target file else undef. There could be multiple such files because of flattening: if there are then they are all supposed to be identical and so any one of them should do.
699 {my ($targetFile) = @_; # The target file we want to locate a unique source file from
700 my $inputFiles = $xref->targetTopicToInputFiles->{$targetFile}; # Input files corresponding to target file
701 return undef unless $inputFiles; # Only if we have input files corresponding to this target file
702 my ($inputFile) = sort keys %$inputFiles; # The unique source file
703 $inputFile
704 };
705  
706 # Given a bookmap and a href to a topic in in in/ find that topic in out/
707 # - Find source of bookmap
708 # - Find source of topic
709 # - Find target of topic
710 # - If target is a bookmap, substitute it
711 # - If target is a topic, replace it
712 # The original source topic was split into several sub topics as described by a bookmap.
713 # A Dita Book Map reference to such a topic should be replaced by the bookmap content.
714 # Conversely, existing map to map references should be left alone
715  
716 my $fixBookMapDitaRef = sub # Fix a partial dita reference in a bookmap. If the reference is to a single topic then replace the href with the renamed topic. If the reference is to a topic that was cut into multiple sub topics then replace the reference with the bookmap that represents the cut out topic.
717 {return undef unless $xref->fixDitaRefs; # Only works if we have the targets folder information
718  
719 if (my $docType = $xref->docType->{$sourceFile}) # Must be a bookmap
720 {if ($docType !~ m(map\Z)s)
721 {lll "Not a bookmap: $sourceFile";
722 return undef;
723 }
724 }
725 else
726 {lll "Not a file known to xref: $sourceFile";
727 return undef;
728 }
729  
730 if (my $bookMapSource = &$locateUniqueTopicSourceForTargetFile($sourceFile))# Source of the book map
731 {my $sourceTopic = absFromAbsPlusRel($bookMapSource, $ref); # Source topic relative to source bookmap
732  
733 if (my $sourceTarget = $xref->sourceTopicToTargetBookMap->{$sourceTopic}) # Target of source topic via targets/ folder
734 {my $sourceDocType = $sourceTarget->{sourceDocType} // q(); # Source document type
735 my $sourceTargetType = $sourceTarget->{targetType} // q(); # Target document type
736 if ($sourceDocType !~ m(map\Z)s and $sourceTargetType =~ m(\Abookmap\Z))# Replace this chapter or topic with the content of the book map generated to represent a non bookmap topic that was split into several sub topics described by a bookmap
737 {if (my $generatedBookMap = $sourceTarget->{target})
738 {if (my $x = Data::Edit::Xml::new($generatedBookMap)) # Parse the generated bookmap for chapters
739 {if ($x->at_bookmap)
740 {if (my @c = $x->c_chapter) # Chapters
741 {for my $c(reverse @c) # Copy chapters
742 {$node->putFirstCut($c);
743 }
744 $node->unwrap; # Unwrap the referencing topic
745 }
746 }
747 else
748 {lll "Expected bookmap, got:", $x->tag,
749 "sourceDocType=$sourceDocType from file:\n$b";
750 }
751 }
752 return &$good(q(), q(Expanded representative bookmap)); # Record the fix made
753 }
754 else
755 {lll "Generated bookmap does not exist $generatedBookMap";
756 }
757 }
758 else # Not a bookmap so just upgrade href
759 {my $path = $sourceTarget->{target};
760 $node->href = relFromAbsAgainstAbs($path, $sourceFile);
761 return &$good($path, q(Unique target from bookmap)); # Record the fix made
762 }
763 }
764 }
765 else
766 {lll "No source for $sourceFile\n";
767 }
768  
769 undef # Failed
770 };
771  
772 my $checkImageRef = sub # Check whether an image exists or not
773 {my $i = absFromAbsPlusRel($sourceFile, $ref); # Local file name
774 return 1 if -e $i; # Local file exists
775 return 2 if -e wwwDecode($i); # Local file exists
776 undef # Local file exists after decoding % signs
777 };
778  
779 my $fixOnePartialDitaRef = sub # Fix a partial dita reference to an externally cut out topic renamed to the GB Standard where such a reference is just a file name as used in a bookmapref.
780 {my ($ref) = @_; # Partial reference
781 return undef unless $xref->fixDitaRefs; # Fixing dita references not requested
782  
783 my $topicSource = &$locateUniqueTopicSourceForTargetFile($sourceFile); # Unique source file corresponding to the target file else undef
784 return undef unless $topicSource; # The references can not be resolved without a unique source file.
785  
786 my $refIn = absFromAbsPlusRel($topicSource, $ref); # The referenced input file that was present in the input being transformed because we assume that (most of) the input Dita refs were valid
787  
788 if (my $new = $xref->inputFileToTargetTopics->{$refIn}) # The target files new files that were cut out of the referenced input file - there might several such
789 {if (my $referencedTarget = fileLargestSize(sort keys %$new)) # Boldly assume that the largest possible target is the one we want
790 {my $link = relFromAbsAgainstAbs($referencedTarget, $sourceFile); # Create relative link from book map
791 $node->set($attr=>$link);# if $xref->fixBadRefs; # Reset reference - we know fixDitaRefs is true.
792 &$good($link, q(unique target)); # Record successful fix
793 return 1; # Success
794 }
795 }
796 undef # Failed
797 };
798  
799 my $fixOneFullDitaRef = sub # Fix a full dita reference to an externally cut out topic renamed to the GB Standard where such a reference is: file#topicId/label
800 {return undef unless $xref->fixDitaRefs; # Fixing dita references not requested
801 return &$fixOnePartialDitaRef($ref) unless $ref =~ m(#); # Confirm it is a full reference else fix it as a partial reference
802  
803 my $topicSource = &$locateUniqueTopicSourceForTargetFile($sourceFile); # Unique source file corresponding to the target file else undef
804 return undef unless $topicSource; # The references can not be resolved without a unique source file.
805 my ($rf, $rt, $ri) = parseDitaRef($ref, $topicSource); # Parse the dita ref
806  
807 if (my $new = $xref->originalSourceFileAndIdToNewFile->{$rf}{$ri}) # The new files cut out of the original topic source file
808 {my $targetFile = relFromAbsAgainstAbs($new, $sourceFile); # Create relative link from current file
809 if (my $topicId = $xref->topicIds->{$new}) # Topic id for target file
810 {my $href = qq($targetFile#$topicId/$ri); # New href
811 $node->set($attr=>$href);# if $xref->fixBadRefs; # Reset href - we know fixDitaRefs is true.
812 &$good($new, q(Unique target for file ref)); # Record the fix made
813 return 1; # Record the fix made
814 }
815 }
816  
817 if ($xref->allowUniquePartialMatches && $attr !~ m(\Aconref)s) # Partial matching - i.e ignoring the stuff to the right of the # in the reference sometimes produces a unique result.
818 {return &$fixOnePartialDitaRef($ref =~ s(#.*\Z) ()rs); # Try to resolve reference as a partial re
819 }
820  
821 undef # Failed
822 };
823  
824 my $fixRelRef = sub # Attempt to fix a reference broken by relocation
825 {my ($R, $rest) = split m(#)s, $ref, 2; # Get referenced file name
826 if ($R)
827 {my $r = fne($R); # Href file base name
828 if (my $F = $xref->baseFiles->{$r}) # Relocated else where
829 {my @targets = sort keys(%$F); # Relocation targets
830 if (@targets == 1) # Just one such relocation
831 {my $f = relFromAbsAgainstAbs($targets[0], $sourceFile); # Link to it
832 if ($f ne $R)
833 {my $newLink; # Fix if the target is else where
834 if ($rest) # Link has more than one component
835 {$node->set($attr=>($newLink = $f.q(#).$rest)); # Reset link
836 }
837 else # Link has just one component
838 {$node->set($attr=>($newLink = $f)); # Reset link
839 }
840 my $saveRef = $ref; $ref = $newLink; # Try fixing the relocated reference as a dita reference.
841 my $r = &$fixOneFullDitaRef;
842 $ref = $saveRef;
843 return $r;
844 }
845 }
846 }
847 }
848 undef # Failed
849 };
850  
851 my $fixOneRef = sub # Fix one unresolved reference either by ameliorating it or by moving it to the xtrf attribute thereby putting it in M3.
852 {return unless $xref->fixRefs->{$sourceFile}{$ref}; # Fix not requested for this reference
853  
854 if ($xref->deguidize and $ref =~ m(GUID-)is) # On a guid and deguidization allowed so given g1#g2/id convert g1 to a file name by locating the topic with topicId g2.
855 {my @refs = split /\s+/, $ref; # There might be multiple references in the href
856 my @unresolved; # Unresolved targets
857 my @resolved; # Resolved targets
858  
859 for my $subRef(@refs) # Each reference in the reference
860 {my ($guid, $rest) = split /#/, $subRef;
861 if (my $target = $xref->guidToFile->{$guid}) # Target file associated with guid
862 {my $link = relFromAbsAgainstAbs($target, $sourceFile); # Relative link
863 $link .= q(#).$rest if $rest; # Remainder of reference which does not change as it is not file related
864 if (!@resolved) # First resolution
865 {$node->set($attr=>$link); # New href or conref
866 &$good($target, q(Deguidized reference)); # Report fix made
867 }
868 push @resolved, $subRef;
869 }
870 else
871 {push @unresolved, $subRef;
872 }
873 }
874  
875 if (@unresolved and $xref->fixBadRefs) # Unresolved - transfer all references to xtrf so some-one else can try
876 {$node->renameAttr($attr, q(xtrf)); # No target file for guid
877 &$bad(q(No file for guid)); # Report failure
878 }
879 }
880 elsif ($xref->fixRelocatedRefs and &$fixRelRef) # Try to fix as a relocated ref if possible
881 {
882 }
883 elsif ($xref->fixXrefsByTitle and &$fixXrefByTitle) # Try to fix a missing xref by title
884 {
885 }
886 elsif ($xref->fixBadRefs) # Move href to xtrf as no other fix seems possible given that we have already tried to fix it as a guid and it was reportedly not working as a standard dita reference.
887 {$node->renameAttr($attr, q(xtrf)); # No target file for guid
888  
889 if ($xref->changeBadXrefToPh) # Change bad xref to ph if requested
890 {if ($node->at_xref)
891 {$node->change_ph;
892 }
893 }
894 &$bad(q(No such target)); # Report failure
895 }
896 else # ffff - Fix not requested so href left alone
897 {&$bad(q(Not fixable)); # Unable to fix the reference using any known method
898 }
899 };
900  
901 my $x = Data::Edit::Xml::new($sourceFile); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed
902 my $s = -p $x; # Source before any changes
903  
904 $x->by(sub # Check any references encountered on each node, Ameliorate some specific cases. If the reference is still invalid report the discrepancy.
905 {my ($o) = @_; # Current node
906 $node = $o; # Make current node available globally
907 my $t = $node->tag; # Tag
908 if ($t =~ m(\A(appendix|chapter|image|link|mapref|topicref|xref)\Z)is) # Hrefs that need to be fixed
909 {if ($ref = $node->attr($attr = q(href))) # The attribute and reference to ameliorate or fix
910 #if ($t =~ m(\A(appendix|chapter|topicref)\Z)is) # Fix bookmap hrefs
911 {if ($t =~ m(\A(appendix|chapter|mapref|topicref)\Z)is) # Fix bookmap hrefs
912 {&$fixBookMapDitaRef or &$fixOneRef; # Fix references to topics cut into multiple pieces and now represented by a bookmap
913 }
914 elsif ($t =~ m(\Aimage\Z)is) # Check image references
915 {&$checkImageRef or &$fixOneRef; # No additional fixes available yet for images, as so far, the resolution of images is done in thee calling frame work. Hence we only need to check whether the reference is valid and if it is not then the standard techniques can be applied and the results reported as usual.
916 }
917 else # Fix hrefs without the benefit of the targets/ folder
918 {&$fixOneFullDitaRef or &$fixOneRef; # Fix references not in a bookmap
919 }
920 }
921 elsif ($t =~ m(\Axref\Z)s and $xref->fixXrefsByTitle and &$fixXrefByTitle)# Try to fix a missing xref by title
922 {
923 }
924 }
925 if ($ref = $node->attr($attr = q(conref))) # Fix a conref
926 {&$fixOneFullDitaRef or &$fixOneRef;
927 }
928 if ($ref = $node->attr($attr = q(conrefend))) # Fix a conrefend
929 {&$fixOneFullDitaRef or &$fixOneRef;
930 }
931 });
932  
933 if (my $S = -p $x) # Source after any changes
934 {if ($S ne $s) # Write any changes - seems to be slightly faster than not checking
935 {if (onAwsSecondary) # Write output to temporary folder regardless so it can be copied enmasse back to the session leader
936 {my $f = swapFolderPrefix($sourceFile, # Output file name
937 $xref->inputFolder, $xref->fixedFolderTemp);
938 editXml($sourceFile, $f, $S); # Write the fixed file to the fixedFolder retaining headers and trailers
939 }
940 elsif (my $fixedFolder = $xref->fixedFolder) # New output file in fixedFolder
941 {my $f = swapFolderPrefix($sourceFile, $xref->inputFolder, $fixedFolder);# Output file name
942 editXml($sourceFile, $f, $S); # Write the fixed file to the fixedFolder retaining headers and trailers
943 }
944 else
945 {editXml($sourceFile, $sourceFile, $S); # Edit existing xml retaining headers and trailers
946 }
947 }
948 }
949  
950 $fixed # Results of fixing this file
951 } # fixReferencesInOneFile
952  
953 sub fixReferencesParallel($$) #P Fix the references in one file
954 {my ($xref, $file) = @_; # Cross referencer, file to fix
955  
956 newXref; # Recreate Xref LVALUE methods
957  
958 if (my $d = $xref->fixedFolderTemp) # Create the folder to be used for fixed files
959 {makePath($d);
960 }
961  
962 my $x = $xref->fixReferencesInOneFile($file);
963 newXref(fixedRefsGood => $x->fixedRefsGood,
964 fixedRefsBad => $x->fixedRefsBad);
965 };
966  
967 sub fixReferencesResults($@) #P Consolidate the results of fixing references.
968 {my ($xref, @results) = @_; # Cross referencer, results from fixReferencesInParallel
969 newXref; # Recreate LVALUE methods
970  
971 $xref->fixedRefsBad = [];
972 $xref->fixedRefsGood = [];
973 for my $x(@results)
974 {push $xref->fixedRefsBad ->@*, $x->fixedRefsBad ->@*;
975 push $xref->fixedRefsGood->@*, $x->fixedRefsGood->@*;
976 }
977  
978 if (onAwsPrimary) # Recover fixed files from secondary instances
979 {my $s = $xref->fixedFolderTemp;
980 makePath($s);
981 my $t = $xref->fixedFolder // $xref->inputFolder;
982 awsParallelGatherFolder($s); # Recover fixed files from secondaries
983 mergeFolder($s, $t); # Merge fixed files into the target area
984 }
985  
986 $xref
987 }
988  
989 sub fixReferences($) #P Fix just the file containing references using a number of techniques and report those references that cannot be so fixed.
990 {my ($xref) = @_; # Xref results
991 my $startTime = time; # Time each block
992  
993 if (1) # Map titles to files for the Gearhart Title Method
994 {my %titleToFile; # Titles to file
995 if (my $xrefTitle = $xref->title)
996 {for my $file(keys %{$xrefTitle}) # Title for each file
997 {if (my $tag = $xref->docType->{$file}) # Document type for file
998 {if ($tag !~ m(map\Z)s) # Ignore maps as we want the topic in the map not the map.
999 {$titleToFile{nws($xrefTitle->{$file})}{$file}++; # Record title to topic
1000 }
1001 }
1002 }
1003 }
1004  
1005 $xref->titleToFile = \%titleToFile; # Record titles to files
1006  
1007 if (1) # Report titles with duplicated titles
1008 {my @r;
1009  
1010 for my $t(sort keys %titleToFile)
1011 {my %f = %{$titleToFile{$t}};
1012 if (my @f = sort keys %f)
1013 {if (@f > 1)
1014 {push @r, map {[$t, $_]} @f;
1015 }
1016 }
1017 }
1018  
1019 formatTables($xref, \@r,
1020 columns => <<END,
1021 Title Topic title
1022 File Topic file
1023 END
1024 title => qq(Topics with duplicate titles),
1025 head => <<END,
1026 Xref noted NNNN topics have duplicated titles on DDDD
1027 END
1028 clearUpLeft=>1,
1029 file=>(fpe(qw(bad topics_with_duplicated_titles txt))));
1030 }
1031 }
1032  
1033 if (my $d = $xref->fixDitaRefs) # Map where the input files went and where the target files came from
1034 {my $bookmaps = $xref->targetFolderContent = readFiles($d); # The targets produced from each input file
1035  
1036 my @r; # Source to target report
1037 my %sourceToTarget; # Maps a source file to its resulting output topics
1038 my %sourceTopicToTargetBookMap; # Source topic cut into multiple parts produces a bookmap
1039  
1040 for my $source(sort keys %$bookmaps) # Each input file represented in the targets folder
1041 {my $sourceToTarget = eval $$bookmaps{$source}; # Mapping for input file
1042 $@ and confess $@; # Check eval was successful
1043  
1044 my $bookMap = genHash(qq(Bookmap), %$sourceToTarget); # Details of bookmap representing source file after possible cutting out
1045 my $bookMapSource = $bookMap->source; # Input file name from input folder rather than target folder
1046 my $bookMapTarget = $bookMap->{target} // q(); # The target bookmap
1047 my $sourceDocType = $bookMap->{sourceDocType} // q(); # The docType of the source input file if known
1048 my $targetType = $bookMap->{targetType} // q(); # The target type, initially just a bookmap, now extended to include topics and images
1049  
1050 push @r, [$targetType, $sourceDocType, $bookMapSource, $bookMapTarget]; # Report source to targets
1051 $sourceTopicToTargetBookMap{$bookMapSource} = $bookMap; # Source to target details of topic
1052  
1053 if ($sourceDocType =~ m(map\Z)s) # If the input file was a map then its target is a map
1054 {$sourceToTarget{$bookMapSource} = {$bookMapTarget=>1};
1055 }
1056 elsif ($targetType =~ m(\Atopic\Z)is) # If the input file produced a single dita topic then the target is that topic
1057 {$sourceToTarget{$bookMapSource} = {$bookMapTarget=>1};
1058 }
1059 elsif ($targetType =~ m(\Abookmap\Z)i) # If the input file was a topic that was cut into multiple topics then the target is a bookmap
1060 {if (my $refs = $xref->topicsReferencedFromBookMaps->{$bookMapTarget}) # The references from the bookmap the source file became
1061 {$sourceToTarget{$bookMapSource} = $refs; # The output files files this source input file was split into
1062 }
1063 else
1064 {#lll "No references from bookmap $bookMapTarget";
1065 }
1066 }
1067 }
1068  
1069 my %targetToSource; # The source files for each output topic - the reverse of sourceTotarget - output topics that have been flattened will have multiple sources
1070 for my $source(sort keys %sourceToTarget) # Each source input file
1071 {for my $target(sort keys %{$sourceToTarget{$source}}) # Each of the target topics that were derived from this source file
1072 {$targetToSource{$target}{$source}++; # Maps an output topic back to the input files that gave rise to it
1073 }
1074 }
1075  
1076 my %targetToSourceDuplicated; # The target files for which there is more than one source file
1077 for my $target(sort keys %targetToSource) # Each of the target topics that were derived from this source file
1078 {my %s = %{$targetToSource{$target}}; # Each source input file
1079 if (keys %s > 1) # Mapping is not injective
1080 {push @{$targetToSourceDuplicated{$target}}, sort keys %s;
1081 }
1082 }
1083  
1084 my %si; # Source ids: {original source input file}{id} = target file
1085 my $xrefIds = $xref->ids;
1086 for my $targetFile(keys %$xrefIds) # Each target file with an id in it
1087 {if (my $sourceFiles = $targetToSource{$targetFile}) # Originating source files for this target file
1088 {for my $sourceFile(keys %$sourceFiles) # Each originating source files for this target file
1089 {for my $id(keys $xrefIds->{$targetFile}->%*) # Each id in the target file
1090 {$si{$sourceFile}{$id} = $targetFile; # The new file containing the id defined in the source file
1091 }
1092 }
1093 }
1094 }
1095  
1096 $xref->inputFileToTargetTopics = \%sourceToTarget; # The targets for each input file
1097 $xref->targetTopicToInputFiles = \%targetToSource; # The source file from which each target was obtained
1098 $xref->sourceTopicToTargetBookMap = \%sourceTopicToTargetBookMap; # The bookmap representing a cut up topic
1099 $xref->topicFlattening = \%targetToSourceDuplicated; # Topics that arose from flattening several source files
1100 $xref->originalSourceFileAndIdToNewFile = \%si; # Record mapping from original source file and id to the new file containing the id
1101  
1102 formatTables($xref, \@r,
1103 columns => <<END,
1104 Type The type of reference
1105 DocType Document type of the source file
1106 Source Source file
1107 Target Cut out file
1108 END
1109 summarize=>1,
1110 title=>qq(The target topics cut out of the source documents),
1111 head=><<END,
1112 Xref noted NNNN cut out topics on DDDD
1113 END
1114 file=>(fpe(qw(lists source_to_targets txt))));
1115  
1116 if (1) # Report topic flattening
1117 {my @r;
1118 my $s = 0; my $t = 0;
1119 for my $target(sort keys %targetToSourceDuplicated) # Each of the target topics that were derived from this source file
1120 {my @s = @{$targetToSourceDuplicated{$target}}; # Each source input file
1121 push @r, [scalar(@s), $target];
1122 push @r, [q(), q( ).$_] for @s;
1123 push @r, [q()];
1124 ++$t; $s += @s;
1125 }
1126 $xref->topicsFlattened = $s; # Record the number of topics flattened
1127 my $F = $xref->topicFlatteningFactor = $t ? $s / $t : 0; # Topic flattening factor - higher is better
1128 my $f = sprintf("%7.4f", $F);
1129 my $n = @{$xref->inputFiles}; # Number of topics
1130 my $p = sprintf("%7.4f", $n ? 100*$t/$n : 0); # Percentage topics flattened versus total number of topics
1131  
1132 formatTables($xref, \@r,
1133 columns => <<END,
1134 Count Number of sources that created this target
1135 Target The target file flattened out from multiple source files
1136 END
1137 summarize => 1,
1138 title => qq(Topic files flattened from multiple sources),
1139 head => <<END,
1140 Xref noted that $s source topics were reduced to $t target topics on DDDD
1141  
1142 This represents a flattening factor of: $f (higher is better) in the topics that got flattened
1143  
1144 Total number of topics : $n
1145 Number of topics flattened: $t
1146 Percent topics flattened: $p
1147 END
1148 file => fpe(qw(lists topic_flattening txt)));
1149 }
1150 }
1151  
1152 if ($xref->fixRelocatedRefs) # Load base file name to full name but if needed to do relocation fixes
1153 {my %baseFiles; # Map base files back to full files. The base file is the file name shorn of the path - the reason the GB Standard is so important
1154 for my $file(searchDirectoryTreesForMatchingFiles($xref->inputFolder)) # All input files
1155 {my $base = fne $file; # Base file name - the GB Standard name for the file
1156 $baseFiles{$base}{$file}++; # Current location of the file
1157 }
1158 $xref->baseFiles = \%baseFiles;
1159 }
1160  
1161 my @bad; # Hrefs that could not be fixed and so were ameliorated by moving them to @xtrf
1162 my @good; # Hrefs that were fixed by resolving a Guid
1163 if (my @files = sort keys %{$xref->fixRefs}) # Fix files if requested
1164 {awsParallelProcessFiles $xref, # Fix files in parallel
1165 \&fixReferencesParallel, # Fix one file
1166 \&fixReferencesResults, # Consolidate results
1167 [@files];
1168  
1169 @good = $xref->fixedRefsGood->@*; # Results from fixReferencesResults
1170 @bad = $xref->fixedRefsBad ->@*;
1171 }
1172  
1173 @good = sort {join(' ', @$a) cmp join(' ', @$b)} @good;
1174 @bad = sort {join(' ', @$a) cmp join(' ', @$b)} @bad;
1175  
1176 my $fbr = $xref->fixBadRefs; # Are we fixing bad refs?
1177 my $facet = q(Dita references);
1178  
1179 formatTables($xref, $xref->fixedRefsBad = \@bad, # Report references we cannot fix
1180 columns => <<END,
1181 Reason The reason the reference was not fixed
1182 Tag The tag of the node in which the reference failure occurs
1183 Attr The attribute of the node in which the reference failure occurs
1184 Reference The reference not being fixed
1185 File The file in which the reference appears
1186 Source_Files One or more source files that from which this file was derived
1187 END
1188 summarize => 1,
1189 title => q(Invalid references),
1190 facet => $facet, aspectColor => q(red),
1191 head => $fbr ? <<END : <<END2,
1192 Xref moved NNNN invalid references to M3 on DDDD as fixBadRefs=>$fbr was specified
1193 END
1194 Xref was unable to resolve NNNN failing references on DDDD, fixBadRefs=> was not specified
1195 END2
1196 zero => 1,
1197 file => fpe(qw(bad failing_references txt)));
1198  
1199 formatTables($xref, $xref->fixedRefsGood = \@good, # Report hrefs which were failing but were successfully resolved by ingenuity.
1200 columns => <<END,
1201 Method The way that the reference was fixed
1202 Tag The tag of the node on which the reference was fixed
1203 Attr The attribute being fixed - normally href
1204 Ref The reference that is being resolved
1205 Target_File The file the reference resolves to.
1206 File The file in which the reference appears
1207 Source_Files The source files that gave rise to the file containing the reference after file flattening
1208 END
1209 summarize => 1,
1210 title => qq(These failing references were successfully resolved),
1211 facet => $facet, aspectColor => q(green),
1212 head => <<END,
1213 Xref successfully resolved NNNN previously failing references on DDDD
1214 END
1215 zero => 1,
1216 file => fpe(qw(good fixed_references txt)));
1217  
1218 } # fixReferences
1219  
1220 sub fixOneFileGB($$) #P Fix one file to the Gearhart-Brenan standard
1221 {my ($xref, $file) = @_; # Xref results, file to fix
1222 my @r; # Hrefs changed
1223  
1224 my $x = Data::Edit::Xml::new($file); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed
1225  
1226 $x->by(sub # Each node
1227 {my ($o) = @_;
1228 if (my $h = $o->href) # Href encountered
1229 {my ($localFile, $rest) = split /#/, $h, 2; # Split reference
1230 my $fullFile = absFromAbsPlusRel($file, $localFile); # Full name of referenced file
1231 if (my $target = $xref->flattenFiles->{$fullFile}) # Target file name
1232 {$o->href = $target.($rest ? qq(#$rest) : qq()); # Reassemble href
1233 }
1234 else
1235 {push @r, [$h, $file];
1236 }
1237 }
1238 });
1239  
1240 my $target = fpf($xref->flattenFolder, $xref->flattenFiles->{$file}); # Previously assigned GB name. We cannot use the very latest name because other files have to be told about it and in changing them to reflect the latest name we would change their name as well. So close has to be good enough.
1241 editXml($file, $target, -p $x); # Edit xml
1242  
1243 \@r # Return report of items fixed
1244 }
1245  
1246 sub fixFilesGB($) #P Rename files to the L<GBStandard>
1247 {my ($xref) = @_; # Xref results
1248 my @files = grep {!$xref->parseFailed->{$_}} sort @{$xref->inputFiles}; # Fix files that parsed if requested
1249  
1250 my @r; # Fixes made
1251 processFilesInParallel
1252 sub # Each file
1253 {my ($file) = @_; # File to analyze
1254 $xref->fixOneFileGB($file); # Analyze one input file
1255 },
1256 sub {push @r, deSquareArray @_}, @files; # Flatten results
1257  
1258 formatTables($xref, $xref->fixedRefsGB = \@r, # Report results
1259 columns => <<END,
1260 Href The href being fixed
1261 Source The source file containing the href
1262 END
1263 summarize=>1,
1264 title=>qq(Hrefs that can not be renamed to the Gearhart-Brenan file naming standard),
1265 head=><<END,
1266 Xref failed to fix NNNN hrefs to the Gearhart-Brenan file naming standard
1267 END
1268 file=>(my $f = fpe(qw(bad fixedRefsGB txt))));
1269  
1270 {fixedRefsGB => $xref->fixedRefsGB, # From multiverse to universe
1271 }
1272 } # fixFilesGB
1273  
1274 sub analyzeOneFileParallel($$) #P Analyze one input file
1275 {my ($Xref, $iFile) = @_; # Xref request, File to analyze
1276  
1277 my $xref = newXref(); # Cross referencer for this file
1278 # $xref->sourceFile = $iFile; # File analyzed
1279 my %maxZoomIn = $Xref->maxZoomIn ? %{$Xref->maxZoomIn} : (); # Regular expressions from maxZoomIn to look for text
1280 my %maxZoomOut; # Text elements that match a maxZoomIn regular expression
1281 my %countAttrNames; # Attribute names
1282 my %countAttrNamesAndValues; # Attribute names and values
1283 my %countTagNames; # Tag names
1284 my $changes; # Changes made to the file
1285 my $tags; my $texts; # Number of tags and text elements
1286  
1287 my $source = readFile($iFile); # Source of file so we can gets its GB Standard name
1288  
1289 my $x = eval {Data::Edit::Xml::new($iFile)}; # Parse xml - at this point if the caller is interested in line numbers they should have added them.
1290  
1291 if ($@) # Check we were able to parse the xml
1292 {$xref->parseFailed->{$iFile}++;
1293 return $xref;
1294 }
1295  
1296 my $md5 = $xref->md5Sum->{$iFile} = -M $x; # Md5 sum for parse tree
1297  
1298 if ($Xref->flattenFolder)
1299 {$xref->flattenFiles->{$iFile} = # Record correspondence between existing file and its GB Standard file name
1300 Dita::GB::Standard::gbStandardFileName($source, fe($iFile), md5=>$md5);
1301 }
1302  
1303 my $saveReference = sub # Save a reference so it can be integrity checked later
1304 {my ($ref) = @_; # Reference
1305 return if externalReference($ref); # Looks like an external reference
1306 $xref->references->{$iFile}{$ref}++; # Save reference
1307 };
1308  
1309 my $isADitaMap = $x->isADitaMap; # Map
1310  
1311 $x->by(sub # Each node
1312 {my ($o) = @_;
1313  
1314 # my $content = sub #P First few characters of content on one line to avoid triggering multi table layouts
1315 # {my ($o) = @_; # String
1316 # nws($o->stringContent, improvementLength); # Length of improvement
1317 # };
1318  
1319 my $tag = -t $o; # Element tag
1320 if ($tag eq q(CDATA)) {++$texts} else {++$tags} # Count texts and tags
1321  
1322 if (my $h = $o->href) # Check href
1323 {if ($h =~ m(\s)s and externalReference($h)) # Check href for url encoding needed
1324 {$xref->{hrefUrlEncoding}{$iFile}{$o->lineLocation} = $h;
1325 }
1326 if ($xref->{deguidize} and $h =~ m(\bguid-)is) # Deguidizing a href that looks as if it might have a guid in it
1327 {$xref->{fixRefs}{$iFile}{$h}++
1328 }
1329 &$saveReference($h);
1330 }
1331  
1332 if (my $conref = $o->attr(q(conref))) # Conref
1333 {my $saveConRef = sub # Save a conref
1334 {my ($conRef) = @_; # Conref
1335 $xref->{conRefs}{$iFile}{$conRef}{$tag}++;
1336 };
1337  
1338 &$saveConRef($conref);
1339 &$saveReference($conref);
1340  
1341 if (my $conref = $o->attr(q(conrefend))) # Conref end
1342 {&$saveConRef($conref);
1343 &$saveReference($conref);
1344 }
1345 }
1346  
1347 if (my $i = $o->id) # Id definitions
1348 {$xref->{ids}{$iFile}{$i}++;
1349 push $xref->{idTags}{$iFile}{$i}->@*, $tag; # Tags for each id in the file
1350 }
1351  
1352 if ($tag eq q(xref)) # Xrefs but not to the web
1353 {if (my $h = $o->href)
1354 {if (externalReference($h)) # Check attributes on external links
1355 {if ($o->attrX(q(scope)) !~ m(\Aexternal\Z)s)
1356 {$xref->{xrefBadScope}{$iFile}{$h} = -A $o;
1357 }
1358 if ($o->attrX(q(format)) !~ m(\Ahtml\Z)s)
1359 {$xref->{xrefBadFormat}{$iFile}{$h} = -A $o;
1360 }
1361 }
1362 elsif ($h =~ m(\Aguid-)is) # Href is a guid
1363 {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation];
1364 }
1365 else #if ($o->attrX_format =~ m(\Adita)i) # Check xref has format=dita
1366 {$xref->{xRefs}{$iFile}{$h}{$o->stringText}++;
1367 }
1368 }
1369 else
1370 {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href
1371 $xref->{fixRefs}{$iFile}{q()}++ # Try and fix by the Gearhart Title method augmented by the Monroe Map Method
1372 }
1373 }
1374 elsif ($isADitaMap and $tag =~ m(\A(appendix|chapter|link|mapref|notices|topicref)\Z)is) # References from bookmaps at 2019.11.10 22:58:24 as mapref can be used in a topic
1375 {if (my $h = $o->href)
1376 {if ($h =~ m(\Aguid-)is) # Href is a guid
1377 {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation];
1378 }
1379 else
1380 {$xref->{bookMapRefs}{$iFile}{$h}{$o->attr_navtitle//$o->stringText}++;
1381 }
1382 }
1383 elsif ($tag ne q(notices)) # Notices is often positioned in a bookmap and left empty for author convenience
1384 {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href
1385 }
1386 }
1387 elsif ($tag eq q(image)) # Images
1388 {if (my $h = $o->href)
1389 {if ($h =~ m(\Aguid-)is) # Href is a guid
1390 {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation]; # Resolve image later
1391 }
1392 else
1393 {$xref->{images}{$iFile}{$h}++;
1394 }
1395 $xref->{imagesReferencedFromTopics}{$iFile}{$h}++; # Image referenced from a topic
1396 }
1397 else
1398 {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href
1399 }
1400 }
1401 elsif ($tag eq q(required-cleanup)) # Required cleanup
1402 {$xref->{requiredCleanUp}{$iFile}{nws($o->stringContent)}++;
1403 }
1404 elsif ($tag eq q(title) and $o->parent == $x) # Title
1405 {$xref->{title}{$iFile} = $o->stringContent; # Topic Id
1406 }
1407 elsif ($tag eq q(mainbooktitle)) # Title for bookmaps
1408 {$xref->{title}{$iFile} //= $o->stringText;
1409 }
1410 elsif ($tag eq q(author)) # Author
1411 {$xref->{author}{$iFile} = $o->stringContent;
1412 }
1413 elsif ($tag eq q(ol)) # Ol
1414 {if (my $p = $o->parent)
1415 {if ($p->tag =~ m(body\Z)s)
1416 {$xref->{olBody}{$iFile}++;
1417 }
1418 }
1419 }
1420 elsif ($tag eq q(tgroup)) # Tgroup cols
1421 {my $error = sub # Table error message
1422 {push @{$xref->{badTables}},
1423 [join('', @_), $tag, $o->lineLocation, $iFile];
1424 };
1425  
1426 my $stats = $o->ditaTGroupStatistics; # Statistics for table
1427 my $cols = $stats->colsAttribute;
1428 my $maxCols = max($stats->maxHead//0, $stats->maxBody//0);
1429 my $maxColsMP = max($stats->maxHeadMinusPadding//0,
1430 $stats->maxBodyMinusPadding//0);
1431 if (($stats->maxHead//0) == $maxCols && # The right combination of body and header
1432 ($stats->minHead//0) == $maxCols &&
1433 ($stats->maxBody//0) == $maxCols &&
1434 ($stats->minBody//0) == $maxCols &&
1435 $stats->colSpec == $maxCols
1436 or !defined($stats->maxHead) && # No headers but everything else looks good
1437 ($stats->maxBody//0) == $maxCols &&
1438 ($stats->minBody//0) == $maxCols &&
1439 $stats->colSpec == $maxCols)
1440 {if (!$cols) # Check for cols attribute
1441 {$error->(qq(No cols attribute, should be $maxCols));
1442 }
1443 elsif ($cols ne $maxCols) # Cols present but wrong
1444 {$error->(qq(Cols attribute is $cols but should be $maxCols));
1445 }
1446 }
1447 elsif ($maxColsMP > (my $actual = $stats->maxHead//0)) # Not enough headers
1448 {$error->(qq(Not enough headers, $actual vs $maxColsMP));
1449 }
1450 else
1451 {$error->(qq(Column padding required));
1452 }
1453 if ($maxCols and my $rows = $stats->rows) # Count table sizes
1454 {$xref->{tableDimensions}{$iFile}{$maxCols}{$rows}++;
1455 }
1456 }
1457 elsif ($tag =~ m(\Aothermeta\Z)) # Other meta tags
1458 {my $c = $o->attrX(q(content));
1459 my $n = $o->attrX(q(name));
1460 $xref->{otherMeta}{$iFile}{$n}{$c}++;
1461 }
1462 elsif ($tag =~ m(body\Z) and $o->isAllBlankText) # Empty body
1463 {$xref->{emptyTopics}{$iFile}++;
1464 }
1465 elsif ($o->isText) # Not much interest - obscured at the start of this 'if' statement - Check text for interesting constructs
1466 {my $t = $o->text;
1467 my @l = $t =~ m(&lt;(.*?)&gt;)g;
1468 for my $l(@l)
1469 {$xref->{ltgt}{$iFile}{$l}++;
1470 }
1471 if (keys %maxZoomIn) # Search for text using Micaela's Max Zoom In Method
1472 {for my $name(sort keys %maxZoomIn) # Each regular expression to check
1473 {my $re = $maxZoomIn{$name};
1474 if ($t =~ m($re)is)
1475 {$maxZoomOut{$name}++
1476 }
1477 }
1478 }
1479 }
1480  
1481 if (my $a = $o->{attributes}) # Count tags, attributes, attribute values
1482 {for my $k(sort keys %$a)
1483 {$countAttrNames{$k}++;
1484 if (my $v = $$a{$k})
1485 {$countAttrNamesAndValues{$k}{$v}++;
1486 }
1487 }
1488 $countTagNames{$tag}++;
1489 }
1490 });
1491  
1492 $xref->maxZoomOut->{$iFile} = \%maxZoomOut; # Save max zoom
1493  
1494 my $docType = parseXmlDocType($source); # Get DocType details
1495  
1496 $xref->attributeCount ->{$iFile} = \%countAttrNames; # Attribute names
1497 $xref->attributeNamesAndValuesCount->{$iFile} = \%countAttrNamesAndValues; # Attribute names and values
1498 $xref->tagCount ->{$iFile} = \%countTagNames; # Tag names
1499 $xref->baseTag ->{$iFile} = $x->tag; # Tag on base node
1500 $xref->docType ->{$iFile} = $x->tag; # Document type
1501 $xref->publicId ->{$iFile} = $docType->{publicId}; # Public id on Doctype
1502 $xref->tags ->{$iFile} = $tags; # Number of tags
1503 $xref->texts ->{$iFile} = $texts; # Number of texts
1504 $xref->topicIds ->{$iFile} = $x->id; # Topic Id
1505 $xref->vocabulary ->{$iFile} = $x->stringTagsAndText # Text of topic minus attributes
1506 if !$isADitaMap and $Xref->indexWords||$Xref->matchTopics; # Maps tend not to have any matchable text in them.
1507  
1508 if (my @urls = $source =~ m(["'](https?://[^"']*?)["'])g) # Urls found in file
1509 {$xref->urls->{$iFile} = {map {$_=>1} @urls};
1510 }
1511  
1512 if (1) # Check xml headers and lint errors
1513 {my @h = split /\n/, my $s = readFile($iFile);
1514 if (!$h[0] or $h[0] !~ m(\A<\?xml version=\"1.0\" encoding=\"UTF-8\"\?>\Z))
1515 {$xref->badXml1->{$iFile}++;
1516 }
1517 my $tag = $x->tag;
1518 if (!$h[1] or $h[1] !~ m(\A<!DOCTYPE $tag PUBLIC "-//))
1519 {$xref->badXml2->{$iFile}++;
1520 }
1521  
1522 $xref->validationErrors->{$iFile}++ if $s =~ m(<!--compressedErrors:)s; # File has validation errors
1523 }
1524  
1525 $xref
1526 } # analyzeOneFileParallel
1527  
1528 sub analyzeOneFileResults($@) #P Merge a list of cross reference results into the first cross referencer in the list
1529 {my ($xref, @x) = @_; # Cross referencer to merge into, other cross referencers
1530  
1531 newXref; # Create LVALUE methods
1532  
1533 my @fields = ( # Fields to be merged
1534 q(attributeCount),
1535 q(attributeNamesAndValuesCount),
1536 q(author),
1537 q(badXml1),
1538 q(badXml2),
1539 q(baseTag),
1540 q(conRefs),
1541 q(docType),
1542 q(emptyTopics),
1543 q(fixRefs),
1544 q(flattenFiles),
1545 q(guidHrefs),
1546 q(hrefUrlEncoding),
1547 q(ids),
1548 q(idTags),
1549 q(images),
1550 q(imagesReferencedFromTopics),
1551 q(ltgt),
1552 q(maxZoomOut),
1553 q(md5Sum),
1554 q(noHref),
1555 q(olBody),
1556 q(otherMeta),
1557 q(parseFailed),
1558 q(publicId),
1559 q(references),
1560 q(requiredCleanUp),
1561 q(tableDimensions),
1562 q(tagCount),
1563 q(tags),
1564 q(texts),
1565 q(title),
1566 q(topicIds),
1567 q(bookMapRefs),
1568 q(targetTopicToInputFiles),
1569 q(urls),
1570 q(validationErrors),
1571 q(vocabulary),
1572 q(xrefBadFormat),
1573 q(xrefBadScope),
1574 q(xRefs),
1575 );
1576  
1577 my $ip = awsCurrentIp;
1578 my @times; # Time for each merge
1579  
1580 if (@x > 10) # Merge in parallel if there are enough items to merge - 10 tests Xref but allows final merge to occur in series
1581 {my $fields = @fields;
1582 my $q = newProcessStarter($xref->maximumNumberOfProcesses); # Process starter
1583 $q->processingTitle = q(Xref Analyze Merge on ip: $ip);
1584 $q->totalToBeStarted = $fields;
1585  
1586 if (my $reports = $xref->reports) # Log file for merge
1587 {$q->processingLogFile = fpe($reports, qw(log xref analyzeMerge txt));
1588 }
1589  
1590 for my $field(@fields) # Merge hashes by file names which are unique - ffff
1591 {$q->start(sub
1592 {my $startTime = time;
1593 my $target = $xref->{$field} //= {}; # Field to be merged
1594 for my $x(@x) # Merge results from each file analyzed
1595 {if (my $xf = $x->{$field})
1596 {for my $f(keys %$xf) # Each file analyzed
1597 {$target->{$f} = $xf->{$f} # Merge
1598 }
1599 }
1600 }
1601 [$field, $xref, time - $startTime] # Return results as a reference
1602 });
1603 }
1604 my @merge = $q->finish; # Load results
1605  
1606 for my $m(@merge)
1607 {my ($f, $x, $t) = @$m;
1608 $xref->{$f} = $x->{$f};
1609 push @times, [$f, $t];
1610 }
1611 }
1612 else # Merge in series
1613 {for my $field(@fields) # Merge hashes by file names which are unique - ffff
1614 {my $startTime = time;
1615 my $target = $xref->{$field} //= {}; # Field to be merged
1616 for my $x(@x) # Merge results from each file analyzed
1617 {if (my $xf = $x->{$field})
1618 {for my $f(keys %$xf) # Each file analyzed
1619 {$target->{$f} = $xf->{$f} # Merge
1620 }
1621 }
1622 }
1623 push @times, [$field, time - $startTime] # Save merge times
1624 }
1625 }
1626  
1627 formatTables($xref, [sort {$$b[1] <=> $$a[1]} @times],
1628 columns => <<END,
1629 Field Xref field merge
1630 Time Time in seconds to merge this field
1631 END
1632 title=>qq(Field merging elapsed times in descending order),
1633 head =>qq(Xref field merging took the following times on DDDD),
1634 file =>fpe(q(timing), qw(merges txt)));
1635  
1636 for my $field( # Merge arrays
1637 qw(badTables))
1638 {for my $x(@x) # mmmm Merge results from each file analyzed
1639 {next unless my $xf = $x->{$field};
1640 push @{$xref->{$field}}, @$xf;
1641 }
1642 }
1643  
1644 $xref
1645 } # analyzeOneFileResults
1646  
1647 sub analyzeInputFiles($) #P Analyze the input files
1648 {my ($xref) = @_; # Cross referencer
1649  
1650 awsParallelProcessFiles
1651 ($xref,
1652 \&analyzeOneFileParallel,
1653 \&analyzeOneFileResults,
1654 $xref->inputFiles);
1655  
1656 } # analyzeInputFiles
1657  
1658 # 63.128s Salesforce restructure on one instance
1659  
1660 sub reportIdRefs($) #P Report the number of times each id is referenced
1661 {my ($xref) = @_; # Cross referencer
1662  
1663 my %n; # Ids that might not be referenced
1664 if (my $xrefIds = $xref->ids)
1665 {for my $f(sort keys %{$xrefIds}) # Each input file
1666 {for my $i(sort keys %{$xrefIds->{$f}}) # Each id in each input file
1667 {$n{$f}{$i}++; # Ids that might not be referenced
1668 }
1669 }
1670 }
1671  
1672 my %c; # Count of how often each id in each file was referenced
1673 if (my $xrefReferences = $xref->references)
1674 {for my $f(sort keys %{$xrefReferences}) # Each input file
1675 {for my $r(sort keys %{$xrefReferences->{$f}}) # Each reference from each input file
1676 {my ($rf, $rt, $ri) = parseDitaRef($r, $f); # Parse reference
1677 if ($ri) # If this is an id not a topic id
1678 { $c{$rf}{$ri} += $xrefReferences->{$f}{$r}; # Count id references
1679 delete $n{$rf}{$ri}; # Remove ids that get referenced
1680 }
1681 }
1682 }
1683 }
1684  
1685 my @n; my $notTopicIds = 0; # Ids not referenced
1686 for my $f(sort keys %n) # Each input file
1687 {my $topicId = $xref->topicIds->{$f} // q();
1688 for my $i(sort keys %{$n{$f}}) # Each unreferenced id in each input file
1689 {my $t = $i eq $topicId ? "**" : q(); # Show ids which are topic ids
1690 ++$notTopicIds unless $t; # Count non topic ids that are not referenced
1691 push @n, [$t, $i, $f]; # Report as topicId or not, id, file
1692 }
1693 }
1694  
1695 my @c;
1696 for my $f(sort keys %c) # Each input file
1697 {for my $i(sort keys %{$c{$f}}) # Each referenced id in each input file
1698 {push @c, [$c{$f}{$i}, $i, $f]; # Report as count, id, file
1699 }
1700 }
1701  
1702 my $facet = q(Ids referenced);
1703  
1704 formatTables($xref, \@n,
1705 columns => <<END,
1706 TopicId If this id is a topic id then this column will have ** in it.
1707 Id The id that has not been referenced
1708 File The file containing the id that has not been referenced
1709 END
1710 title => qq(Ids that have not been referenced),
1711 facet => $facet, aspectColor=>q(red),
1712 head => <<END,
1713 Xref found $notTopicIds unreferenced ids on DDDD
1714  
1715 The $notTopicIds ids below which are not marked with ** are not referenced by
1716 any other file in this corpus and thus should be considered for removal.
1717  
1718 END
1719 summarize => 1, csv => 1,
1720 file => fpe(qw(bad ids_not_referenced txt)));
1721  
1722 formatTables($xref, \@c,
1723 columns => <<END,
1724 Count The number of times this id is referenced
1725 Id The id in question
1726 File The file in which the id appears
1727 END
1728 title => qq(Ids referenced),
1729 facet => $facet, aspectColor=>q(green),
1730 head => <<END,
1731 Xref found NNNN ids are being referenced on DDDD
1732  
1733 END
1734 summarize => 1, csv => 1,
1735 file => fpe(qw(good id_reference_counts txt)));
1736  
1737 {idReferencedCount => \%c, # From multiverse to universe
1738 idNotReferenced => \%n,
1739 }
1740 } # reportIdRefs
1741  
1742 sub removeUnusedIds($) #P Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference. This is a very conservative approach which acknowledges that writers might be looking for an id if they mention it in a reference.
1743 {my ($xref) = @_; # Cross referencer
1744  
1745 my %keep; # {id}++ : id to be kept as it occurs in a reference some where
1746 if (my $xrefReferences = $xref->references)
1747 {for my $f(sort keys $xrefReferences->%*) # Each input file with references
1748 {for my $r(sort keys $xrefReferences->{$f}->%*) # Each id in each input file with references
1749 {my (undef, undef, $i) = parseDitaRef($r); # Ids to keep as they might be referenced now or soon
1750 $keep{$i}++
1751 }
1752 }
1753 }
1754  
1755 my %remove; # {file}{id}++ : id to be removed from a file
1756 my %ids; # {id}++ : id removed from all files as a consequence of removing it from each file
1757 if (my $xrefIds = $xref->ids)
1758 {for my $f(sort keys $xrefIds->%*) # Each input file
1759 {for my $i(sort keys $xrefIds->{$f}->%*) # Each id in each input file
1760 {next if $keep{$i}; # Id that can be deleted as it is never referenced in any form
1761 $remove{$f}{$i}++;
1762 $ids {$i}++
1763 }
1764 }
1765 }
1766  
1767 my $disposition = $xref->deleteUnusedIds ? # Disposition of deletable ids
1768 q(The deleteUnusedIds option was set on so these ids have been removed.) :
1769 q(The deleteUnusedIds option was set off so these ids have been retained.);
1770  
1771 formatTables($xref, [map {[$_]} sort keys %ids],
1772 columns => <<END,
1773 Id An unreferenced id
1774 END
1775 title => qq(Ids removed),
1776 head => <<END,
1777 Xref found NNNN unreferenced ids on DDDD
1778  
1779 $disposition
1780 END
1781 summarize => 1,
1782 file => fpe(qw(lists ids_removed txt)));
1783  
1784 processFilesInParallel
1785 (sub
1786 {my ($f) = @_; # File with ids to be removed
1787 my %i = %{$remove{$f}}; # Ids to remove
1788 my @remove; # [id, file] : removed this id in this file
1789 if (my $x = Data::Edit::Xml::new($f)) # Parse file with ids to be removed
1790 {$x->by(sub
1791 {my ($o) = @_;
1792 if ($o != $x and my $i = $o->id) # Keep topic ids
1793 {if ($i{$i})
1794 {$o->id = undef;
1795 push @remove, [$i, $o->tag, $f]; # Report id removed
1796 }
1797 }
1798 });
1799 editXml($f, $f, -p $x) if @remove;
1800 }
1801 [@remove]
1802 },
1803  
1804 sub # Consolidate removal report
1805 {my (@removed) = @_;
1806  
1807 my @r;
1808 for my $removed(@removed) # Consolidate each batch of removals
1809 {push @r, @$removed if $removed;
1810 }
1811  
1812 formatTables($xref, \@r,
1813 columns => <<END,
1814 Id The id removed
1815 Tag The tag the removed id was on
1816 File The file in which the id appeared
1817 END
1818 title => qq(Ids removed from files),
1819 head => <<END,
1820 Xref removed NNNN unreferenced ids from files on DDDD
1821  
1822 Ids are only removed if the id is not mentioned in any reference in the entire
1823 corpus regardless of what file the reference is to. This is to allow for the
1824 possibility of fixing possibly broken references in the future.
1825  
1826 END
1827 summarize => 1,
1828 file => fpe(qw(lists ids_removed_from_files txt)));
1829 }, sort keys %remove) if $xref->deleteUnusedIds; # Remove unused ids unless except on the root element
1830  
1831 {idsRemoved => \%ids, # Multiverse to universe
1832 }
1833 } # removeUnusedIds
1834  
1835 sub reportEmptyTopics($) #P Report empty topics
1836 {my ($xref) = @_; # Cross referencer
1837  
1838 my @E; # Ids that might not be referenced
1839 for my $f(sort keys %{$xref->emptyTopics}) # Each empty topic
1840 {my $d = $xref->docType->{$f};
1841 my $t = $xref->title->{$f} // q();
1842 push @E, [$d, $t, $f];
1843 }
1844  
1845 my @e = sort {$$a[1] cmp $$b[1]} @E;
1846  
1847 formatTables($xref, \@e,
1848 columns => <<END,
1849 TopicType The type of topic thatis empty
1850 Title The title of the empty topic
1851 File The source file containing the empty topic
1852 END
1853 title => <<END,
1854 Empty Topics
1855 END
1856 head => <<END,
1857 Xref found NNNN empty topics on DDDD
1858  
1859 END
1860 summarize => 1,
1861 csv => 1,
1862 file => fpe(qw(lists empty_topics txt)));
1863  
1864 { # From multiverse to universe
1865 }
1866 } # reportEmptyTopics
1867  
1868 sub reportDuplicateIds($) #P Report duplicate ids
1869 {my ($xref) = @_; # Cross referencer
1870  
1871 my @dups; # Duplicate ids definitions
1872 my %dups;
1873 my %active; my %removed; # Active duplicate ids, removable ids
1874 if (my $xrefIds = $xref->ids)
1875 {for my $f(sort keys %$xrefIds) # Each input file
1876 {for my $i(sort keys $xrefIds->{$f}->%*) # Each id in the file
1877 {my $count = $xrefIds->{$f}{$i}; # Number of definitions of this id in the file
1878 if ($count > 1) # Duplicate definition
1879 {if ($xref->idsRemoved->{$i}) # Duplicated and being referred to
1880 {$removed{$i}++;
1881 }
1882 else
1883 {$active{$i}++;
1884 }
1885 push @dups, [$i, $count, $active{$i} ? q(**) : q(), $f]; # Save details of duplicate definition
1886 $dups{$f}{$i} = $count;
1887 }
1888 }
1889 }
1890 }
1891  
1892 my $A = keys %active; # Number of possibly active duplicate ids
1893 my $R = keys %removed; # Number of ids removed
1894  
1895 my $r = $xref->deleteUnusedIds ? <<END : # Explain active vs inactive ids
1896 $R unused ids are being deleted as they are not active (used in a reference
1897 regardless of the file component of the reference) and Xref was invoked with:
1898  
1899 deleteUnusedIds => 1
1900  
1901 The $A active ids that cannot be removed are marked in this report with **.
1902 END
1903 <<END;
1904 $R unused ids could be deleted as they are not active (used in a reference
1905 regardless of the file component of the reference). To remove these ids from
1906 the entire corpus invoke Xref with:
1907  
1908 deleteUnusedIds => 1
1909  
1910 The $A active ids that cannot be so removed are marked in this report with **.
1911 END
1912  
1913 formatTables($xref, \@dups,
1914 columns => <<END,
1915 Id The id that has been duplicated
1916 Count The number of times the id was duplicated
1917 Active ** if this id is possibly being referred to from elsewhere in the corpus.
1918 File The file in which the duplication occurs
1919 END
1920 title=>qq(Duplicate id definitions within files),
1921 head=><<END,
1922 Xref found NNNN duplicate id definitions within files on DDDD
1923  
1924 $A of these ids are possibly being referred to - see rows containing **.
1925  
1926 Duplicate topic ids are reported in ../bad/topicIds.txt.
1927  
1928 $r
1929 END
1930 summarize=>1, csv=>1,
1931 file=>(my $f = fpe(qw(bad duplicateIds txt))));
1932  
1933 {duplicateIds => \%dups,
1934 } # From multiverse to universe
1935 } # reportDuplicateIds
1936  
1937 sub reportDuplicateTopicIds($) #P Report duplicate topic ids
1938 {my ($xref) = @_; # Cross referencer
1939  
1940 my %dups; # Duplicate topic ids definitions
1941 my @dups; # Duplicate topic ids definitions report
1942 my @miss; # Missing topic id definitions report
1943 if (my $xrefTopicIds = $xref->topicIds) # Each input file
1944 {for my $file(sort keys %{$xrefTopicIds}) # Each input file
1945 {if (my $i = $xrefTopicIds->{$file}) # Topic Id
1946 {if (my $d = $dups{$i}) # Duplicate topic id
1947 {push @dups, [$i, $file, $d]; # Save details of duplicate definition
1948 }
1949 else
1950 {$dups{$i} = $file; # Save topic id
1951 }
1952 }
1953 elsif ($xref->docType->{$file} !~ m(map\Z)s) # Maps are not required to have topics ids
1954 {push @miss, [$file]; # Missing topic id
1955 }
1956 }
1957 }
1958  
1959 my $dups = $xref->duplicateTopicIds = {map {$$_[0]=>$_} @dups}; # All duplicates
1960 my $miss = $xref->missingTopicIds = {map {$$_[0]=>$_} @miss}; # All missing
1961  
1962 formatTables($xref, \@dups, columns => <<END, # Duplicate topic ids report
1963 TopicId The topic id that has been duplicated in File1 and File2
1964 File1 The first file in which the duplicated topic id appears
1965 File2 The second file in which the duplicated topic id appears
1966 END
1967 title => qq(Duplicate topic id definitions),
1968 head => <<END,
1969 Xref found NNNN duplicate topic id definitions on DDDD
1970 END
1971 file=>(fpe(qw(bad duplicate_topics_ids txt))));
1972  
1973 formatTables($xref, \@miss, columns => <<END, # Missing topic ids report
1974 File A file containing a topic with no topic id
1975 END
1976 title=>qq(Topics without ids),
1977 head=><<END,
1978 Xref found NNNN topics that have no topic id on DDDD
1979  
1980 END
1981 file=>(fpe(qw(bad topic_id_missing txt))));
1982  
1983 {duplicateTopicIds => $dups,
1984 missingTopicIds => $miss,
1985 }
1986 } # reportDuplicateTopicIds
1987  
1988 sub reportNoHrefs($) #P Report locations where an href was expected but not found
1989 {my ($xref) = @_; # Cross referencer
1990 my @t;
1991 if (my $xrefNoHref = $xref->noHref)
1992 {for my $file(sort keys %{$xrefNoHref}) # Each input file
1993 {push @t, @{$xrefNoHref->{$file}}; # Missing href details
1994 }
1995 }
1996  
1997 formatTables($xref, \@t,
1998 columns => <<END,
1999 Tag A tag that should have an xref.
2000 Location The location of the tag that should have an xref.
2001 File The source file containing the tag
2002 END
2003 title=>qq(Missing hrefs),
2004 head=><<END,
2005 Xref found NNNN tags that should have href attributes but did not on DDDD
2006 END
2007 file=>(fpe(qw(bad missing_href_attributes txt))));
2008 {} # From multiverse to universe
2009 } # reportNoHrefs
2010  
2011 sub checkReferences($) #P Check each reference, report bad references and mark them for fixing.
2012 {my ($xref) = @_; # Cross referencer
2013  
2014 my @bad; # Bad references
2015 my @good; # Good references
2016  
2017 if (my $xrefReferences = $xref->references)
2018 {for my $file(sort keys %$xrefReferences) # Each input file which will be absolute
2019 {for my $ref (sort keys $xrefReferences->{$file}->%*) # Each href in the file which will be relative
2020 {if (my $r = &oneBadRef($xref, $file, $ref)) # Check reference
2021 {push @bad, $r;
2022 $xref->fixRefs->{$file}{$ref}++; # Request fix attempt for this reference
2023 $xref->badReferencesCount++; # Number of bad references encountered
2024 }
2025 else # Good references
2026 {push @good, [$ref, $file]
2027 }
2028 }
2029 }
2030 }
2031  
2032 my $facet = q(References between topics at start);
2033  
2034 formatTables($xref, \@bad, # Report the failing references
2035 columns => <<END,
2036 Reason The reason why the reference failed to resolve
2037 Reference The href in the source file
2038 Ref_File The file containing the referenced topic relative to the referencing file
2039 Ref_TopicId The topic id of the referenced topic
2040 Ref_Id The id of the statement in the referenced topic
2041 Source_TopicId The topic id of the referencing file
2042 Target_TopicId The topic id of the referenced file
2043 Source_File The referencing source file
2044 Target_File The referenced target file
2045 END
2046 title => qq(Bad references at start),
2047 facet => $facet, aspectColor=>q(red),
2048 head => <<END,
2049 Xref found NNNN bad references on DDDD at the start of processing
2050  
2051 Depending on the options chosen, Xref might fix or ameliorate these references
2052 so that the actual number of references with problems is lower than this. The
2053 best way to find out how many references still need fixing after doing an Xref
2054 run with fixing options enabled is to do another run with all of these options
2055 disabled. The braver elements will be asking why we do not account for these
2056 improvements in flight - that would significantly increase code complexity
2057 while taking a lot of effort to validate yet produce no significant
2058 improvements - so it is much easier not to when a second run suffices.
2059 END
2060 csv => 1, wide =>1, summarize=>1,
2061 file => fpe(q(bad), q(references), q(txt)));
2062  
2063 formatTables($xref, \@good, # Report good references
2064 columns => <<END,
2065 Href A good href attribute
2066 Source_File The referencing source file
2067 END
2068 title => qq(Good references at start),
2069 facet => $facet, aspectColor=>q(green),
2070 head => <<END,
2071 Xref found NNNN good references on DDDD at the start of processing.
2072 END
2073 csv => 1, wide =>1, summarize=>1,
2074 file => fpe(q(good), q(references), q(txt)));
2075 # {} # From multiverse to universe 2019.08.29 - no need - we are in series
2076 } # checkReferences
2077  
2078 sub reportGuidHrefs($) #P Report on guid hrefs
2079 {my ($xref) = @_; # Cross referencer
2080  
2081 my %guidToFile; # Map guids to files
2082 if (my $xrefTopicIds = $xref->topicIds)
2083 {for my $file(sort keys %{$xrefTopicIds}) # Each input file containing a topic id
2084 {my $id = $xrefTopicIds->{$file}; # Each href in the file which will start with guid
2085 next unless defined $id;
2086 next unless $id =~ m(\bguid-)is; # Check guid appears somewhere in href
2087 $guidToFile{$id} = $file; # We report duplicates in reportDuplicateTopicIds
2088 }
2089 }
2090  
2091 my @bad; my @good; # Good and bad guid hrefs
2092 for my $file(sort keys %{$xref->guidHrefs}) # Each input file which will be absolute
2093 {my $sourceTopicId = $xref->topicIds->{$file};
2094 for my $href(sort keys %{$xref->guidHrefs->{$file}}) # Each href in the file which will start with guid
2095 {my ($tag, $lineLocation) = @{$xref->guidHrefs->{$file}{$href}}; # Tag of node and location in source file of node doing the referencing
2096 # 2019.08.29 The following line does not appear to be needed - it is happening to late to affect anything
2097 $xref->fixRefs->{$file}{$href}++ unless $xref->fixRefs->{$file}{$href}; # Avoid double counting - all guid hrefs will be fixed if we are fixing hrefs as both good and bad will fail.
2098  
2099 if ($href =~ m(#)) # Href with #
2100 {my ($guid, $topic, $id) = split m(#|\/), $href, 3; # Guid, topic, remainder
2101 my $targetFile = $guidToFile{$guid}; # Locate file defining guid
2102  
2103 if (!defined $targetFile) # No definition of this guid
2104 {push @bad, # Report missing guid
2105 ["No such guid defined", $tag, $href, $lineLocation, q(),
2106 $sourceTopicId, $targetFile, $file];
2107 next;
2108 }
2109  
2110 my $targetFileId = $xref->topicIds->{$targetFile} // ''; # Actual id in target file
2111  
2112 my $bad = sub
2113 {push @bad,
2114 [@_, $tag, $href, $lineLocation, $targetFileId, $sourceTopicId,
2115 $targetFile, $file];
2116 };
2117  
2118 my $good = sub
2119 {push @good,
2120 [$href, $tag, $lineLocation, $targetFile, $file];
2121 };
2122  
2123 if (!-e $targetFile) # Existence of file
2124 {$bad->(q(No such file));
2125 }
2126 elsif (defined $topic) # Topic defined so it must be an xref
2127 {if ($topic ne $guid)
2128 {$bad->(q(Guid does not match topic id));
2129 }
2130 elsif (defined $id)
2131 {if (my $i = $xref->ids->{$targetFile}{$id}) # Check id exists in target file
2132 {if ($i == 1)
2133 {&$good;
2134 }
2135 else
2136 {$bad->(q(Duplicate id in topic));
2137 }
2138 }
2139 $bad->(q(No such id in topic));
2140 }
2141 else
2142 {&$good;
2143 }
2144 }
2145 else
2146 {&$good;
2147 }
2148 }
2149 elsif ($tag eq q(image)) # Image reference
2150 {my $guid = $href =~ s(guid|-) ()igsr;
2151 if (my $image = $xref->inputFolderImages->{$guid})
2152 {push @good, [$tag, $href, $lineLocation, $image, $file];
2153 }
2154 else
2155 {push @bad, [qq(No such image guid defined), $tag, $href,
2156 $lineLocation, q(), $sourceTopicId, q(), $file];
2157 }
2158 }
2159 else # No # in href and not an image so it must be a bookmap element
2160 {my $targetFile = $guidToFile{$href};
2161 if (!defined $targetFile) # No such guid
2162 {push @bad, [qq(No such guid defined), $tag, $href,
2163 $lineLocation, q(), $sourceTopicId, q(), $file];
2164 }
2165 elsif (!-e $targetFile) # Actual file name
2166 {push @bad, my $p = [qq(No such file), $tag, $href,
2167 $lineLocation, q(), $sourceTopicId, $targetFile, $file];
2168 }
2169 elsif ($xref->fixBadRefs) # The file exists and we want to fix such references
2170 {# 2019.08.29 the following line happens too late to be useful
2171 $xref->fixRefs->{$file}{$href}++;
2172 }
2173 else
2174 {push @good, [$tag, $href, $lineLocation, $targetFile, $file];
2175 }
2176 }
2177 }
2178 }
2179  
2180 $xref->badGuidHrefs = {map {$$_[7]=>$_} @bad}; # Bad references
2181  
2182 my $in = $xref->inputFolder//'';
2183 formatTables($xref, \@bad,
2184 columns => <<END,
2185 Reason The reason why the href failed to resolve
2186 Tag The tag of the node doing the referencing
2187 Href The href of the node doing the referencing
2188 Line_Location The line location where the href occurred in the source file
2189 Target_Topic_Id The actual id of the topic in the target file
2190 Source_Topic_Id The topic id in the source file
2191 Target_File The target file
2192 Source_file The source file containing the reference
2193 END
2194 title =>qq(Unresolved GUID hrefs),
2195 head =>qq(Xref found NNNN unresolved GUID hrefs on DDDD),
2196 summarize=>1,
2197 wide =>1,
2198 file =>(fpe(q(bad), qw(guidHrefs txt))));
2199  
2200 formatTables($xref, \@good,
2201 columns => <<END,
2202 Tag The tag containing the href
2203 Href The href of the node doing the referencing
2204 Line_Location The line location where the href occurred in the source file
2205 Source_File The source file containing the reference
2206 Target_File The target file
2207 END
2208 title =>qq(Resolved GUID hrefs),
2209 head =>qq(Xref found NNNN Resolved GUID hrefs on DDDD),
2210 file =>(fpe(q(good), qw(guidHrefs txt))));
2211  
2212 {badGuidHrefs => $xref->badGuidHrefs, # From multiverse to universe
2213 }
2214 } # reportGuidHrefs
2215  
2216 sub reportImages($) #P Reports on images and references to images
2217 {my ($xref) = @_; # Cross referencer
2218  
2219 my %bad; # Missing images
2220 for my $fail($xref->fixedRefsBad->@*) # References that failed
2221 {my ($reason, $type, $attr, $ref, $file) = @$fail;
2222 next unless $type =~ m(\Aimage\Z)is;
2223 my $i = absFromAbsPlusRel($file, $ref);
2224 $bad{$i}++;
2225 }
2226 my @bad = map {[$bad{$_}, $_]} sort keys %bad;
2227  
2228 my $facet = q(Image files);
2229  
2230 formatTables($xref, \@bad, # Report missing images references
2231 columns => <<END,
2232 Count The number of unresolved references to this missing image
2233 Image_File The file name of the missing image
2234 END
2235 title => qq(Missing images),
2236 facet => $facet, aspectColor => qw(red),
2237 head => qq(Xref found NNNN missing images on DDDD),
2238 summarize => 1, zero => 1,
2239 file => fpe(qw(bad missing_images txt)));
2240  
2241  
2242 my %good; # Good images
2243 for my $topic(sort keys $xref->images->%*) # Topics containing images
2244 {for my $ref (sort keys $xref->images->{$topic}->%*) # Image references
2245 {my $i = absFromAbsPlusRel($topic, $ref);
2246 if (!$bad{$i})
2247 {$good{$i}++;
2248 }
2249 }
2250 }
2251  
2252 my @good = map {[$good{$_}, $_]} sort keys %good; # Report good images
2253  
2254 formatTables($xref, \@good,
2255 columns => <<END,
2256 Count The number of references to this image
2257 Image_File The file name of the image
2258 END
2259 title => qq(Image files),
2260 facet => $facet, aspectColor => qw(green),
2261 head=>qq(Xref found NNNN image files on DDDD),
2262 summarize => 1, zero => 1,
2263 file=>(fpe(qw(good image_files txt))));
2264  
2265 {missingImageFiles => \%bad,
2266 goodImageFiles => \%good}
2267  
2268 } # reportImages
2269  
2270 sub reportParseFailed($) #P Report failed parses
2271 {my ($xref) = @_; # Cross referencer
2272  
2273 formatTables($xref,
2274 hashOfCountsToArray($xref->parseFailed),
2275 columns => <<END,
2276 Source The file that failed to parse as an absolute file path
2277 END
2278 title=>qq(Files failed to parse),
2279 head=>qq(Xref found NNNN files failed to parse on DDDD),
2280 file=>(my $f = fpe(qw(bad parseFailed txt))));
2281 {} # From multiverse to universe
2282 } # reportParseFailed
2283  
2284 sub reportXml1($) #P Report bad xml on line 1
2285 {my ($xref) = @_; # Cross referencer
2286  
2287 formatTables($xref, [map {[$_]} sort keys %{$xref->badXml1}],
2288 columns => <<END,
2289 Source The source file containing bad xml on line
2290 END
2291 title=>qq(Bad Xml line 1),
2292 head=>qq(Xref found NNNN Files with the incorrect xml on line 1 on DDDD),
2293 file=>(my $f = fpe(qw(bad xmlLine1 txt))));
2294 {} # From multiverse to universe
2295 } # reportXml1
2296  
2297 sub reportXml2($) #P Report bad xml on line 2
2298 {my ($xref) = @_; # Cross referencer
2299  
2300 formatTables($xref, [map {[$_]} sort keys %{$xref->badXml2}],
2301 columns => <<END,
2302 Source The source file containing bad xml on line
2303 END
2304 title=>qq(Bad Xml line 2),
2305 head=>qq(Xref found NNNN Files with the incorrect xml on line 2 on DDDD),
2306 file=>(my $f = fpe(qw(bad xmlLine2 txt))));
2307 {} # From multiverse to universe
2308 } # reportXml2
2309  
2310 sub reportDocTypeCount($) #P Report doc type count
2311 {my ($xref) = @_; # Cross referencer
2312  
2313 my %d;
2314 if (my $xrefDocType = $xref->docType)
2315 {for my $f(sort keys %{$xrefDocType})
2316 {my $d = $xrefDocType->{$f};
2317 $d{$d}++
2318 }
2319 }
2320  
2321 formatTables($xref,
2322 hashOfCountsToArray(\%d),
2323 columns => <<END,
2324 DocType The root element tag of this document
2325 END
2326 title=>qq(Document types),
2327 head=>qq(Xref found NNNN different doc types on DDDD),
2328 file=>(fpe(qw(count docTypes txt))));
2329 {} # From multiverse to universe
2330 } # reportDocTypeCount
2331  
2332 sub reportTagCount($) #P Report tag counts
2333 {my ($xref) = @_; # Cross referencer
2334  
2335 my %d;
2336 if (my $xrefTagCount = $xref->tagCount)
2337 {for my $f(sort keys %{$xrefTagCount})
2338 {for my $t(sort keys %{$xrefTagCount->{$f}})
2339 {my $d = $xrefTagCount->{$f}{$t};
2340 $d{$t} += $d;
2341 }
2342 }
2343 }
2344  
2345 formatTables($xref, hashOfCountsToArray(\%d),
2346 columns => <<END,
2347 Tag The tag name of the xml node
2348 Count The number of times this tag name occurs
2349 END
2350 title=>qq(Tags),
2351 head=>qq(Xref found NNNN different tags on DDDD),
2352 file=>(fpe(qw(count tags txt))));
2353 {} # From multiverse to universe
2354 } # reportTagCount
2355  
2356 sub reportTagsAndTextsCount($) #P Report tags and texts counts
2357 {my ($xref) = @_; # Cross referencer
2358  
2359 my $tags = 1; $tags += $xref->tags ->{$_}||0 for keys %{$xref->tags};
2360 my $texts = 1; $texts += $xref->texts->{$_}||0 for keys %{$xref->texts};
2361  
2362 my @t;
2363 push @t, [q(Tags), $tags];
2364 push @t, [q(Texts), $texts];
2365 my $ratio = $xref->tagsTextsRatio = $tags/$texts;
2366 push @t, [q(Tags to Texts), sprintf("%7.4f", $ratio)];
2367  
2368  
2369 formatTables($xref, \@t,
2370 columns => <<END,
2371 Item A tag or some text
2372 Count The number of times the tag or text occurs
2373 END
2374 title => q(Tags to Texts Ratio),
2375 head => q(Xref found the following tag and text counts on DDDD),
2376 file => (fpe(qw(count tagsAndTexts txt))));
2377  
2378 {tagsTextsRatio => $xref->tagsTextsRatio, # From multiverse to universe
2379 }
2380 } # reportTagsAndTextsCount
2381  
2382 sub reportLtGt($) #P Report items found between &lt; and &gt;
2383 {my ($xref) = @_; # Cross referencer
2384  
2385 my %d;
2386 if (my $xrefLtgt = $xref->ltgt)
2387 {for my $f(sort keys %{$xrefLtgt})
2388 {for my $t(sort keys %{$xrefLtgt->{$f}})
2389 {$d{$t} += $xrefLtgt->{$f}{$t};
2390 }
2391 }
2392 }
2393  
2394 formatTables($xref, [map {[$d{$_}, nws($_)]} sort keys %d],
2395 columns => <<END,
2396 Count The number of times this text was found
2397 Text The text found between &lt; and &gt;. The white space has been normalized to make better use of the display.
2398 END
2399 title=>qq(Text found between &lt; and &gt;),
2400 head=><<END,
2401 Xref found NNNN different text items between &lt; and &gt; on DDDD
2402 END
2403 file=>(fpe(qw(count ltgt txt))));
2404 {} # From multiverse to universe
2405 } # reportLtGt
2406  
2407 sub reportAttributeCount($) #P Report attribute counts
2408 {my ($xref) = @_; # Cross referencer
2409  
2410 my %d;
2411 if (my $xrefAttributeCount = $xref->attributeCount)
2412 {for my $f(sort keys %{$xrefAttributeCount})
2413 {for my $t(sort keys %{$xrefAttributeCount->{$f}})
2414 {my $d = $xrefAttributeCount->{$f}{$t};
2415 $d{$t} += $d;
2416 }
2417 }
2418 }
2419  
2420 my @c;
2421 for my $t(sort keys %d)
2422 {push @c, [$d{$t}, $t];
2423 }
2424  
2425 formatTables($xref, [@c], # Attribute count report
2426 columns => <<END,
2427 Count The number of times this attribute appears
2428 Attribute The attribute name being counted
2429 END
2430 title=>qq(Attributes),
2431 head=>qq(Xref found NNNN different attributes on DDDD),
2432 file=>(my $f = fpe(qw(count attributes txt))));
2433 {} # From multiverse to universe
2434 } # reportAttributeCount
2435  
2436 sub reportAttributeNameAndValueCounts($) #P Report attribute value counts
2437 {my ($xref) = @_; # Cross referencer
2438  
2439 my %d;
2440 if (my $xrefAttributeNamesAndValuesCount = $xref->attributeNamesAndValuesCount)
2441 {for my $f(sort keys %{$xrefAttributeNamesAndValuesCount})
2442 {for my $a(sort keys %{$xrefAttributeNamesAndValuesCount->{$f}})
2443 {for my $v(sort keys %{$xrefAttributeNamesAndValuesCount->{$f}{$a}})
2444 {my $c = $xrefAttributeNamesAndValuesCount->{$f}{$a}{$v};
2445 $d{$a}{$v} += $c;
2446 }
2447 }
2448 }
2449 }
2450  
2451 my @D;
2452 for my $a(sort keys %d)
2453 {for my $v(sort keys %{$d{$a}})
2454 {push @D, [$d{$a}{$v}, firstNChars($v, 128), $a]; # Otherwise the report can get surprisingly wide
2455 }
2456 }
2457  
2458 my @d = sort {$$a[2] cmp $$b[2]}
2459 sort {$$b[0] <=> $$a[0]} @D;
2460  
2461  
2462 formatTables($xref, \@d,
2463 columns => <<END,
2464 Count The number of times this value occurs
2465 Value The value being counted
2466 Attribute The attribute on which the value appears
2467 END
2468 summarize => 1,
2469 title => qq(Attribute value counts),
2470 head => qq(Xref found NNNN attribute value combinations on DDDD),
2471 file => (fpe(qw(count attributeNamesAndValues txt))));
2472 {} # From multiverse to universe
2473 } # reportAttributeNameAndValueCounts
2474  
2475 sub reportValidationErrors($) #P Report the files known to have validation errors
2476 {my ($xref) = @_; # Cross referencer
2477  
2478 my $e = $xref->validationErrors;
2479  
2480 formatTables($xref, [map {[$$e{$_}, $_]} sort keys %$e], #
2481 columns => <<END,
2482 Count The number of valiation errors in the file
2483 File A file with xml validation errors
2484 END
2485 title=>qq(Topics with xml validation errors),
2486 head=><<END,
2487 Xref found NNNN topics with xml validation errors on DDDD
2488 END
2489 file=>(fpe(qw(bad validationErrors txt))));
2490 {} # From multiverse to universe
2491 } # reportValidationErrors
2492  
2493 sub reportTables($) #P Report on tables that have problems
2494 {my ($xref) = @_; # Cross referencer
2495  
2496 formatTables($xref, $xref->badTables,
2497 columns => <<END,
2498 Reason Reason bookmap failed
2499 Attributes The tag and attributes of the table element in question
2500 Location The location at which the error was detected
2501 Source_File The file in which the error was detected
2502 END
2503 title=>qq(Tables with errors),
2504 head=><<END,
2505 Xref found NNNN table errors on DDDD
2506 END
2507 summarize=>1,
2508 file=>(fpe(qw(bad tables txt))));
2509 {} # From multiverse to universe
2510 } # reportTables
2511  
2512 sub reportFileExtensionCount($) #P Report file extension counts
2513 {my ($xref) = @_; # Cross referencer
2514  
2515 my $folder = $xref->inputFolder;
2516 formatTables($xref,
2517 hashOfCountsToArray(countFileExtensions($folder)),
2518 columns => <<END,
2519 Ext An extension that appears in the input corpus
2520 Count The number of times the extension appears in the corpus.
2521 END
2522 title=>qq(File extensions),
2523 head=><<END,
2524 Xref found NNNN different file extensions on DDDD in folder: $folder
2525 END
2526 file=>(fpe(qw(count fileExtensions txt))));
2527 {} # From multiverse to universe
2528 } # reportFileExtensionCount
2529  
2530 sub reportFileTypes($) #P Report file type counts - takes too long in series
2531 {my ($xref) = @_; # Cross referencer
2532  
2533 my $ft = countFileTypes($xref->inputFolder, $xref->maximumNumberOfProcesses); # Count file types
2534  
2535 formatTables($xref,
2536 hashOfCountsToArray($ft),
2537 columns=><<END,
2538 Type A file type that appears in the input corpus
2539 Count The number of times the file type appears in the corpus.
2540 END
2541 title=>qq(Files types),
2542 head=><<END,
2543 Xref found NNNN different file types on DDDD
2544 END
2545 file=>(my $f = fpe(qw(count fileTypes txt))));
2546 {} # From multiverse to universe
2547 } # reportFileTypes
2548  
2549 sub reportExternalXrefs($) #P Report external xrefs missing other attributes
2550 {my ($xref) = @_; # Cross referencer
2551  
2552 my @s;
2553 if (my $xrefXrefBadScope = $xref->xrefBadScope)
2554 {for my $f(sort keys %{$xrefXrefBadScope})
2555 {my $sourceTopicId = $xref->topicIds->{$f};
2556 for my $h(sort keys %{$xrefXrefBadScope->{$f}})
2557 {my $s = $xrefXrefBadScope->{$f}{$h};
2558 push @s, [q(Bad scope attribute), $h, $s, $sourceTopicId, $f];
2559 }
2560 }
2561 }
2562  
2563 if (my $xrefXrefBadFormat = $xref->xrefBadFormat)
2564 {for my $f(sort keys %{$xrefXrefBadFormat})
2565 {my $sourceTopicId = $xref->topicIds->{$f};
2566 for my $h(sort keys %{$xrefXrefBadFormat->{$f}})
2567 {my $s = $xrefXrefBadFormat->{$f}{$h};
2568 push @s, [q(Bad format attribute), $h, $s, $sourceTopicId, $f];
2569 }
2570 }
2571 }
2572  
2573 formatTables($xref, \@s,
2574 columns => <<END,
2575 Reason The reason why the xref is unsatisfactory
2576 Href The href attribute of the xref in question
2577 Xref_Statement The xref statement in question
2578 Source_Topic_Id The topic id of the source file containing file containing the bad external xref
2579 File The file containing the xref statement in question
2580 END
2581 title=>qq(Bad external xrefs),
2582 head=>qq(Xref found bad external xrefs on DDDD),
2583 file=>(my $f = fpe(qw(bad externalXrefs txt))));
2584 {} # From multiverse to universe
2585 } # reportExternalXrefs
2586  
2587 sub reportMaxZoomOut($) #P Text located via Max Zoom In
2588 {my ($xref) = @_; # Cross referencer
2589 return {} unless my $names = $xref->maxZoomIn; # No point if maxZoomIn was not specified
2590  
2591 my @names = (qw(File_Name Title), sort keys %$names); # Column Headers
2592 my $columns = join "\n", @names; # Column Headers as lines suitable for format tables
2593 my %names = map {$names[$_]=>$_} keys @names; # Assign regular expression names to columns in the output table/csv
2594  
2595 my @f;
2596 if (my $xrefMaxZoomOut = $xref->maxZoomOut // {})
2597 {for my $f(sort keys %$xrefMaxZoomOut) # One row per file processed showing which regular expression names matched
2598 {my @n = ($f, $xref->title->{$f});
2599 my $c = 0;
2600 for my $n(sort keys %{$xrefMaxZoomOut->{$f}})
2601 {$n[$names{$n}] += $xrefMaxZoomOut->{$f}{$n};
2602 ++$c;
2603 }
2604 push @f, [@n] if $c; # Only save a row if it has something in it
2605 }
2606 }
2607  
2608 if (my $xrefMaxZoomOut = $xref->maxZoomOut // {})
2609 {for my $f(sort keys %$xrefMaxZoomOut)
2610 {my $t = $xref->title->{$f};
2611 my $d = $xrefMaxZoomOut->{$f};
2612 $xrefMaxZoomOut->{$f} = {title=>$t, data=>$d};
2613 }
2614 }
2615  
2616 formatTables($xref, [sort {$$a[0] cmp $$b[0]} @f], # Sort by file name
2617 columns => $columns,
2618 title => qq(Max Zoom In Matches),
2619 head => <<END,
2620 Xref found NNNN file matches on DDDD
2621 END
2622 file =>(fpe(qw(lists maxZoom txt))),
2623 summarize => 1);
2624  
2625 dumpFile(fpe(qw(lists maxZoom data)), $xref->maxZoomOut); # Dump the search results
2626  
2627 {maxZoomOut => $xref->maxZoomOut, # From multiverse to universe
2628 }
2629 } # reportMaxZoomOut
2630  
2631 sub reportTopicDetails($) #P Things that occur once in each file
2632 {my ($xref) = @_; # Cross referencer
2633  
2634 my @t;
2635 for my $f(sort @{$xref->inputFiles})
2636 {push @t, [$xref->docType ->{$f}//q(),
2637 $xref->topicIds->{$f}//q(),
2638 $xref->author ->{$f}//q(),
2639 $xref->title ->{$f}//q(),
2640 $f,
2641 ];
2642 }
2643  
2644 formatTables($xref, \@t,
2645 columns => <<END,
2646 Tag The outermost tag
2647 Id The id on the outermost tag
2648 Author The author of the topic
2649 Title The title of the topic
2650 File The source file name as a relative file name
2651 END
2652 title=>qq(Topics),
2653 head=><<END,
2654 Xref found NNNN topics on DDDD
2655 END
2656 file=>(fpe(qw(lists topics txt))),
2657 summarize=>1);
2658 {} # From multiverse to universe
2659 } # reportTopicDetails
2660  
2661 sub reportTopicReuse($) #P Count how frequently each topic is reused
2662 {my ($xref) = @_; # Cross referencer
2663  
2664 my %t;
2665 if (my $xrefBookMapRefs = $xref->bookMapRefs)
2666 {for my $f(sort keys %{$xrefBookMapRefs})
2667 {for my $t(sort keys %{$xrefBookMapRefs->{$f}})
2668 {my $file = absFromAbsPlusRel($f, $t);
2669 $t{$file}{$f}++;
2670 }
2671 }
2672 }
2673  
2674 for my $t(keys %t) # Eliminate bookmaprefs that are used only once
2675 {if (keys (%{$t{$t}}) <= 1)
2676 {delete $t{$t};
2677 }
2678 }
2679  
2680 my @t;
2681 for my $t(keys %t) # Target
2682 {for my $s(keys %{$t{$t}}) # Source
2683 {push @t, [scalar(keys %{$t{$t}}), $t{$t}{$s}, $t, $s];
2684 }
2685 }
2686  
2687 my $t = [sort {$a->[0] <=> $b->[0]} # Order report
2688 sort {$a->[2] cmp $b->[2]} @t];
2689  
2690 for my $i(keys @$t) # Deduplicate first column from third column
2691 {next unless $i;
2692 my $a = $t->[$i-1];
2693 my $b = $t->[$i];
2694 $b->[0] = '' if $a->[2] eq $b->[2];
2695 }
2696  
2697 formatTables($xref, $t,
2698 columns => <<END,
2699 Reuse The number of times the target topic is reused over all topics
2700 Count The number of times the target topic is reused in the source topic
2701 Target The topic that is being reused == the target of reuse
2702 Source The topic that is referencing the reused topic
2703 END
2704 title=>qq(Topic Reuses),
2705 head=><<END,
2706 Xref found NNNN topics that are currently being reused on DDDD
2707 END
2708 file=>(fpe(qw(lists topicReuse txt))),
2709 zero=>1, # Reuse is very unlikely because the matching criteria is the MD5 sum
2710 summarize=>1);
2711 {} # From multiverse to universe
2712 } # reportTopicReuse
2713  
2714 # References might need fixing either because they are invalid or because we are
2715 # deguidizing
2716  
2717 sub reportFixRefs($) #P Report of hrefs that need to be fixed
2718 {my ($xref) = @_; # Cross referencer
2719  
2720 my @r;
2721 if (my $xrefFixRefs = $xref->fixRefs)
2722 {for my $f(sort keys %{$xrefFixRefs})
2723 {for my $h(sort keys %{$xrefFixRefs->{$f}})
2724 {push @r, [$h, $f];
2725 }
2726 }
2727 }
2728  
2729 formatTables($xref, \@r,
2730 columns => <<END,
2731 Reference The reference to be fixed
2732 Source The topic that contains the reference
2733 END
2734 title=>qq(References to fix),
2735 head=><<END,
2736 Xref found NNNN hrefs that should be fixed on DDDD
2737 END
2738 file=>(fpe(qw(lists fixRefs txt))),
2739 zero=>1,
2740 summarize=>1);
2741 {} # From multiverse to universe
2742 } # reportFixRefs
2743  
2744 sub reportSourceFiles($) #P Source file for each topic
2745 {my ($xref) = @_; # Cross referencer
2746 my @r;
2747 if (my $xrefTargetTopicToInputFiles = $xref->targetTopicToInputFiles)
2748 {for my $f(sort keys %{$xrefTargetTopicToInputFiles}) # File
2749 {my $s = $xrefTargetTopicToInputFiles->{$f}; # Source file for topic
2750 push @r, [$f, join ', ', sort keys %$s] if $s;
2751 }
2752 }
2753  
2754 formatTables($xref, [sort {$$a[0]cmp $$b[0]} @r],
2755 columns => <<END,
2756 Topic The topic file
2757 Source The source file(s) from which the topic was obtained
2758 END
2759 title=>qq(Source file for each topic),
2760 head=><<END,
2761 Xref found the source files for NNNN topics on DDDD
2762 END
2763 file=>(fpe(qw(lists source_file_for_each_topic txt))),
2764 summarize=>1);
2765 {} # From multiverse to universe
2766 } # reportSourceFiles
2767  
2768 sub reportReferencesFromBookMaps($) #P Topics and images referenced from bookmaps
2769 {my ($xref) = @_; # Cross referencer
2770 my %bi; # Bookmap to image
2771 my %bt; # Bookmap to topics
2772 my %ib; # Images to bookmaps
2773 my %tb; # Topics to bookmaps
2774 my @bi; # Bookmap to image report
2775 my @bt; # Bookmap to topic report
2776 my @ib; # Images to bookmaps reports
2777 my @tb; # Topics to bookmaps reports
2778  
2779 my $imageRefsFromTopic = sub # Image references from a topic
2780 {my ($b, $t) = @_; # Book map, topic
2781 for my $I(sort keys %{$xref->imagesReferencedFromTopics->{$t}}) # Image href
2782 {my $i = absFromAbsPlusRel($t, $I);
2783 push @bi, my $d = [$I, -e $i ? 1 : '', $i, $t, $b];
2784 $bi{$b}{$i}++; # Images from bookmap
2785 $ib{$i}{$b}++; # Images to bookmaps
2786 }
2787 };
2788  
2789 if (my $xrefBookMapRefs = $xref->bookMapRefs) # Book map as that is the only kind of file containing a topic ref
2790 {for my $b(sort keys %{$xrefBookMapRefs}) # Book map as that is the only kind of file containing a topic ref
2791 {for my $T(sort keys %{$xrefBookMapRefs->{$b}}) # Topic href
2792 {my $t = absFromAbsPlusRel($b, $T);
2793  
2794 push @bt, [$T, (-e $T ? 1 : q()), $t, $b]; # Report book map to topic
2795 $bt{$b}{$t}++; # Book maps to topics
2796 $tb{$t}{$b}++; # Topics to book maps
2797  
2798 &$imageRefsFromTopic($b, $t);
2799 }
2800  
2801 for my $C(sort keys %{$xref->conRefs->{$b}}) # Conref
2802 {my ($t) = parseDitaRef($C, $b);
2803 &$imageRefsFromTopic($b, $t);
2804 }
2805 }
2806 }
2807  
2808 for my $i(sort keys %ib) # Image
2809 {for my $b(sort keys %{$ib{$i}}) # Bookmap
2810 {my $c = $ib{$i}{$b};
2811 push @ib, [$c, $i, $b];
2812 }
2813 }
2814  
2815 for my $t(sort keys %tb) # Topic
2816 {for my $b(sort keys %{$tb{$t}}) # Bookmap
2817 {my $c = $tb{$t}{$b};
2818 push @tb, [$c, $t, $b];
2819 }
2820 }
2821 # ALL FILES (below) ARE FULLY QUALIFIED!
2822 $xref->topicsToReferringBookMaps = \%tb; # Topics to referring bookmaps
2823 $xref->topicsReferencedFromBookMaps = \%bt; # Topics referenced from bookmaps
2824 $xref->imagesReferencedFromBookMaps = \%bi; # Images referenced from bookmaps
2825 $xref->imagesToRefferingBookMaps = \%ib; # Images to bookmaps
2826  
2827 formatTables($xref, \@bi,
2828 columns => <<END,
2829 Href The href that contains an image reference
2830 Exists Whether the referenced image exists or not
2831 Image The name of the image file
2832 Topic The topic that referenced the image
2833 Bookmap The book map that referenced the topic
2834 END
2835 title=>qq(Images referenced from bookmaps),
2836 head=><<END,
2837 Xref found NNNN images referenced from bookmaps via topics on DDDD
2838 END
2839 file=>(fpe(qw(lists images_from_bookmaps txt))),
2840 zero=>1,
2841 summarize=>1);
2842  
2843 formatTables($xref, \@bt,
2844 columns => <<END,
2845 Reference The topic reference
2846 Exists Whether the referenced topic exists or not
2847 Topic The topic that referenced the image
2848 Bookmap The book map that referenced the topic
2849 END
2850 title=>qq(Topics referenced from bookmaps),
2851 head=><<END,
2852 Xref found NNNN topics referenced from bookmaps via topics on DDDD
2853 END
2854 file=>(fpe(qw(lists bookmaps_to_topics txt))),
2855 zero=>1,
2856 summarize=>1);
2857  
2858 formatTables($xref, \@tb,
2859 columns => <<END,
2860 Count The number of times this topic is reffered to by this bookmap
2861 Topic The topic in question
2862 Bookmap A bookmap that refers to this topic
2863 END
2864 title=>qq(Topics to referring bookmaps),
2865 head=><<END,
2866 Xref found NNNN topics referred to from bookmaps on DDDD
2867 END
2868 file=>(fpe(qw(lists topics_to_bookmaps txt))),
2869 zero=>1,
2870 summarize=>1);
2871  
2872 formatTables($xref, \@ib,
2873 columns => <<END,
2874 Count The number of times this image is reffered to by this bookmap
2875 Image The image in question
2876 Bookmap A bookmap that refers to this image
2877 END
2878 title=>qq(Images to referring bookmaps),
2879 head=><<END,
2880 Xref found NNNN images referred to from bookmaps on DDDD
2881 END
2882 file=>(fpe(qw(lists images_to_bookmaps txt))),
2883 zero=>1,
2884 summarize=>1);
2885 } # reportReferencesFromBookMaps
2886  
2887 sub reportExteriorMaps($) #P Maps that are not referenced by any other map
2888 {my ($xref) = @_; # Cross referencer
2889  
2890 my %r; # Map reference count
2891 if (my $xrefTopicsReferencedFromBookMaps = $xref->topicsReferencedFromBookMaps)
2892 {for my $b(sort keys %{$xrefTopicsReferencedFromBookMaps}) # Each map
2893 {$r{$b}++; # Save map
2894 for my $t(sort keys %{$xrefTopicsReferencedFromBookMaps->{$b}}) # Each topic reference from the map
2895 {if (my $d = $xref->docType->{$t}) # Document type
2896 {if ($d =~ m(map\Z)s) # Its a map
2897 {$r{$t}--; # It has been referenced
2898 }
2899 }
2900 }
2901 }
2902 }
2903  
2904 my %u; # Unreferenced maps
2905 for my $b(sort keys %r) # Each map
2906 {$u{$b}++ if $r{$b} > 0; # Unreferenced map
2907 }
2908  
2909 formatTables($xref,
2910 hashOfCountsToArray(\%u),
2911 columns => <<END,
2912 Bookmap An exterior bookmap
2913 END
2914 title=>qq(Exterior bookmaps),
2915 head=><<END,
2916 Xref found NNNN exterior bookmaps on DDDD
2917  
2918 An exterior map is one that is not referenced from any other map in the corpus.
2919 END
2920 file=>(fpe(qw(lists exterior_maps txt))),
2921 summarize=>1);
2922  
2923 {exteriorMaps => \%u, # Exterior maps == maps that are not referenced by another map
2924 } # From multiverse to universe
2925 } # reportExteriorMaps
2926  
2927 sub reportTopicsNotReferencedFromBookMaps($) #P Topics not referenced from bookmaps
2928 {my ($xref) = @_; # Cross referencer
2929  
2930 my %r; # Topics referenced from a bookmap
2931 if (my $xrefTopicsReferencedFromBookMaps = $xref->topicsReferencedFromBookMaps)
2932 {for my $b(sort keys %{$xrefTopicsReferencedFromBookMaps}) # Book maps
2933 {for my $f(sort keys %{$xrefTopicsReferencedFromBookMaps->{$b}}) # Topics referenced from bookmaps
2934 {$r{$f}++; # Topic reference count
2935 }
2936 }
2937 }
2938  
2939 my @n; # Topic not referenced from a bookmap
2940 if (my $xrefDocType = $xref->docType)
2941 {for my $f(sort keys %{$xrefDocType}) # Input xml files
2942 {next if $xrefDocType ->{$f} =~ m(map\Z)s;
2943 push @n, $f unless $r{$f}; # Topic not referenced from a bookmap
2944 }
2945 }
2946  
2947 my $facet = q(Topics referenced from bookmaps); # Facet
2948  
2949 my $n = $xref->topicsNotReferencedFromBookMaps = {map {$_=>1} @n}; #Topics not referenced by any bookmap
2950  
2951 formatTables($xref,
2952 [map {[$r{$_}, swapFilePrefix($_, $xref->inputFolder)]} sort keys %r],
2953 columns => <<END,
2954 Count Number of referring bookmaps
2955 Topic Topic referenced from one or more bookmaps
2956 END
2957 title => qq(Topics referenced by bookmaps),
2958 facet => $facet, aspectColor=>q(green),
2959 head => <<END,
2960 Xref found NNNN topics that are referenced by one or more bookmaps on DDDD
2961 END
2962 file => fpe(qw(good topics_referenced_from_bookmaps txt)),
2963 summarize => 1);
2964  
2965 formatTables($xref,
2966 [map {[swapFilePrefix($_, $xref->inputFolder)]} sort {$a cmp $b} @n],
2967 columns => <<END,
2968 Topic Unreferenced topic file name
2969 END
2970 title => qq(Topics not referenced by any bookmap),
2971 facet => $facet, aspectColor=>q(red),
2972 head => <<END,
2973 Xref found NNNN topics that are not referenced by any bookmaps on DDDD
2974 END
2975 file => fpe(qw(bad topics_not_referenced_from_bookmaps txt)),
2976 summarize => 1);
2977  
2978 {topicsNotReferencedFromBookMaps=>$n
2979 }
2980 } # reportTopicsNotReferencedFromBookMaps
2981  
2982  
2983 sub reportTableDimensions($) #P Report table dimensions
2984 {my ($xref) = @_; # Cross referencer
2985  
2986 my %d = %{$xref->tableDimensions};
2987 my %r; # Topic not referenced from a bookmap
2988 for my $f(keys %d) # Files with tables
2989 {for my $c(keys %{$d{$f}}) # Table columns
2990 {for my $r(keys %{$d{$f}{$c}}) # Table rows
2991 {push @{$r{$c}{$r}}, $f;
2992 }
2993 }
2994 }
2995  
2996 my @d; # Topic not referenced from a bookmap
2997 for my $c(sort {$a <=> $b} keys %r) # Table columns
2998 {for my $r(sort {$a <=> $b} keys %{$r{$c}}) # Table rows
2999 {my ($f, @f) = @{$r{$c}{$r}};
3000 push @d, [$c, $r, $f];
3001 push @d, [q(), q(), $_] for @f;
3002 }
3003 }
3004  
3005 formatTables($xref, [@d],
3006 columns => <<END,
3007 Columns Number of columns in table
3008 Rows Number of rows in table
3009 File A file that contains a table with this many columns and rows
3010 END
3011 title=>qq(Table dimensions),
3012 head=><<END,
3013 Xref found NNNN table dimensions on DDDD
3014 END
3015 file=>(fpe(qw(count table_dimensions txt))),
3016 summarize=>1);
3017  
3018 {}
3019 } # reportTableDimensions
3020  
3021 sub reportOtherMeta($) #P Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map.
3022 {my ($xref) = @_; # Cross referencer
3023  
3024 my %t; my %b; my %B; # Othermeta at topic level, othermeta to migrate to bookmap, othermeta already in bookmap
3025 for my $b(sort keys $xref->bookMapRefs->%*)
3026 {for my $n(sort keys $xref->otherMeta->{$b}->%*) # Name of othermeta
3027 {for my $c(sort keys $xref->otherMeta->{$b}->{$n}->%*) # Content of other meta
3028 {$b{$b}{$n}{$c}++;
3029 $B{$b}{$n}{$c}++;
3030 }
3031 }
3032  
3033 for my $r(sort keys $xref->bookMapRefs->{$b}->%*) # Each topic reached from the bookmap
3034 {my $t = absFromAbsPlusRel($b, $r); # Topic references are relative
3035 if (my $o = $xref->otherMeta->{$t})
3036 {for my $n(sort keys $o->%*)
3037 {for my $c(sort keys $o->{$n}->%*)
3038 {$t{$t}{$n}{$c}{$b}++; # Othermeta by topic
3039 $b{$b}{$n}{$c}++; # Put topic othermeta in bookmap
3040 }
3041 }
3042 }
3043 }
3044 }
3045  
3046 if (1) # Bookmaps and topics with duplicate othermeta should be empty
3047 {my @d;
3048 for my $b(sort keys %B) # Bookmaps
3049 {for my $n(sort keys $B{$b}->%*)
3050 {if (my $N = keys $B{$b}{$n}->%*)
3051 {if ($N > 1)
3052 {my ($c, @c) = sort keys $B{$b}{$n}->%*;
3053 push @d, [$b, $n, $N, $c], map {[(q()) x 3, $_]} @c;
3054 }
3055 }
3056 }
3057 }
3058  
3059 for my $t(sort keys %t) # Topics
3060 {for my $n(sort keys $t{$t}->%*)
3061 {if (my $N = keys $t{$t}{$n}->%*)
3062 {if ($N > 1)
3063 {my ($c, @c) = sort keys $t{$t}{$n}->%*;
3064 push @d, [$t, $n, $N, $c], map {[(q()) x 3, $_]} @c;
3065 }
3066 }
3067 }
3068 }
3069  
3070 formatTables($xref, $xref->otherMetaDuplicatesSeparately = [@d],
3071 columns => <<END,
3072 Source Bookmaps or topic files with duplicate othermeta data
3073 Name Duplicated othermeta name field
3074 Count Number of duplicates
3075 Content Othermeta content
3076 END
3077 title=>q(Duplicate othermeta data in bookmaps and topics considered separately),
3078 head =>q(Found NNNN duplicate othermeta items on DDDD),
3079 clearUpLeft => -1, summarize=>1,
3080 file =>fpe(qw(other_meta duplicates_separately txt)));
3081 }
3082  
3083 if (1) # Report duplicate othermeta in bookmaps with called topics othermeta included
3084 {my @d;
3085  
3086 for my $b(sort keys %b) # Bookmaps
3087 {for my $n(sort keys $b{$b}->%*)
3088 {if (my $N = keys $b{$b}{$n}->%*)
3089 {if ($N > 1)
3090 {my ($c, @c) = sort keys $b{$b}{$n}->%*;
3091 push @d, [$b, $n, $N, $c], map {[(q()) x 3, $_]} @c;
3092 }
3093 }
3094 }
3095 }
3096  
3097 formatTables($xref, $xref->otherMetaDuplicatesCombined = [@d],
3098 columns => <<END,
3099 Source Bookmap with duplicate othermeta with called topics othermeta included
3100 Name Duplicated othermeta name field
3101 Count Number of duplicates
3102 Content Othermeta content
3103 END
3104 title=>q(Duplicate othermeta in bookmaps with called topic othermeta included),
3105 head =>q(Found NNNN duplicate othermeta items on DDDD),
3106 clearUpLeft => -1, summarize=>1,
3107 file =>fpe(qw(other_meta duplicates txt)));
3108 }
3109  
3110 my %o; # Topic overrides
3111 for my $t(sort keys %t) # Find topic othermeta which must override bookmap othermeta
3112 {for my $n(sort keys $t{$t}->%*)
3113 {for my $c(sort keys $t{$t}{$n}->%*)
3114 {for my $b(sort keys $t{$t}{$n}{$c}->%*)
3115 {if (!$b{$b}{$n}{$c} or keys($b{$b}{$n}->%*) != 1) # Override the othermeta from the bookmap unless the bookmap agrees that there is only value for this name
3116 {$o{$t}{$n}{$c}{$b}++;
3117 }
3118 }
3119 }
3120 }
3121 }
3122  
3123 if (1) # Keep in topic because we cannot push the meta data to all the calling bookmaps
3124 {my @k;
3125 for my $t(sort keys %o)
3126 {for my $n(sort keys $o{$t}->%*)
3127 {for my $c(sort keys $o{$t}{$n}->%*)
3128 {push @k, [$t, $n, $c, sort keys $o{$t}{$n}{$c}->%*]; # The bookmaps that cause the over ride
3129 }
3130 }
3131 }
3132  
3133 formatTables($xref, $xref->otherMetaRemainWithTopic = [@k],
3134 columns => <<END,
3135 Topic Topic file name
3136 Name Othermeta name field to be retained
3137 Content Othermeta name content to be retained
3138 Bookmaps One or more bookmaps that prevented the migration of this othermeta to the calling bookmaps
3139 END
3140 title=>q(Othermeta kept in topics because calling bookmaps disagree),
3141 clearUpLeft => -1, summarize=>1,
3142 head =>qq(Found NNNN othermeta items that must remain in topic on DDDD),
3143 file =>fpe(qw(other_meta must_remain_in_topic txt)));
3144 }
3145  
3146 if (1) # Report othermeta pushed to the bookmaps
3147 {my @p;
3148 for my $t(sort keys %t)
3149 {for my $n(sort keys $t{$t}->%*)
3150 {for my $c(sort keys $t{$t}{$n}->%*)
3151 {for my $b(sort keys $t{$t}{$n}{$c}->%*)
3152 {push @p, [$t, $n, $c, $b] unless $o{$t}{$n}{$c}{$b} # Can be pushed to bookmap
3153 }
3154 }
3155 }
3156 }
3157  
3158 formatTables($xref, $xref->otherMetaPushToBookMap = [@p],
3159 columns => <<END,
3160 Topic Topic file name
3161 Name Othermeta name field to be retained
3162 Content Othermeta name content to be retained
3163 Bookmap The bookmap data othermeta can be migrated to
3164 END
3165 title=>q(Othermeta data that can be pushed to the calling bookmaps),
3166 clearUpLeft => -1, summarize=>1,
3167 head =>qq(Found NNNN othermeta items that can be pushed to the calling bookmaps on DDDD),
3168 file =>fpe(qw(other_meta push_to_book_maps txt)));
3169 }
3170  
3171 if (1) # Report bookmap othermeta before topic othermeta has been included
3172 {my @b;
3173  
3174 for my $b(sort keys %B)
3175 {for my $n(sort keys $B{$b}->%*)
3176 {for my $c(sort keys $B{$b}{$n}->%*)
3177 {push @b, [$b, $n, scalar(keys $B{$b}{$n}->%*), $c];
3178  
3179 }
3180 }
3181 }
3182  
3183 formatTables($xref, $xref->otherMetaBookMapsBeforeTopicIncludes = [@b],
3184 columns => <<END,
3185 Bookmap Bookmap file name
3186 Name Bookmap othermeta name
3187 Count Number of distinct values for this othermeta name in this bookmap
3188 Content Othermeta content for this name
3189 END
3190 title=> q(Bookmap othermeta before topic othermeta has been included),
3191 clearUpLeft => -1, summarize=>1,
3192 head => qq(Xref found NNNN Bookmap othermeta tags before topic othermeta was included),
3193 file => fpe(qw(other_meta book_maps_before_topics_included txt)));
3194 }
3195  
3196 if (1) # Report bookmap othermeta after topic othermeta has been included
3197 {my @b;
3198  
3199 for my $b(sort keys %b)
3200 {for my $n(sort keys $b{$b}->%*)
3201 {for my $c(sort keys $b{$b}{$n}->%*)
3202 {push @b, [$b, $n, scalar(keys $b{$b}{$n}->%*), $c];
3203 }
3204 }
3205 }
3206  
3207 formatTables($xref, $xref->otherMetaBookMapsAfterTopicIncludes = [@b],
3208 columns => <<END,
3209 Bookmap Bookmap file name
3210 Name Bookmap or topic othermeta name
3211 Count Number of distinct values for this othermeta name
3212 Content Othermeta content for this name
3213 END
3214 title=> q(Bookmap othermeta data after topic othermeta has been included),
3215 clearUpLeft => -1, summarize=>1,
3216 head => qq(Xref found NNNN Bookmap othermeta tags after topic othermeta was included on DDDD),
3217 file =>fpe(qw(other_meta book_maps_after_topics_included txt)));
3218 }
3219  
3220 {otherMetaDuplicatesSeparately => $xref->otherMetaDuplicatesSeparately,
3221 otherMetaDuplicatesCombined => $xref->otherMetaDuplicatesCombined,
3222 otherMetaRemainWithTopic => $xref->otherMetaRemainWithTopic,
3223 otherMetaPushToBookMap => $xref->otherMetaPushToBookMap,
3224 otherMetaBookMapsBeforeTopicIncludes => $xref->otherMetaBookMapsBeforeTopicIncludes,
3225 otherMetaBookMapsAfterTopicIncludes => $xref->otherMetaBookMapsAfterTopicIncludes,
3226 }
3227  
3228 } # reportOtherMeta
3229  
3230 sub createSubjectSchemeMap($) #P Create a subject scheme map from othermeta
3231 {my ($xref) = @_; # Cross referencer
3232  
3233 my %o; # Consolidated other meta
3234 for my $f(sort keys $xref->otherMeta->%*) # Each file containing othermeta
3235 {for my $n(sort keys $xref->otherMeta->{$f}->%*) # Name of othermeta
3236 {for my $c(sort keys $xref->otherMeta->{$f}->{$n}->%*) # Content of other meta
3237 {$o{$n}{$c}++;
3238 }
3239 }
3240 }
3241  
3242 if (my $out = $xref->subjectSchemeMap) # SubjectSchemeMap requested by supplying an output folder for the maps
3243 {my $makeNavTitle = sub # Create a nav title using the Gearhart NavTitle Method
3244 {my ($c) = @_; # Othermeta content
3245 ucfirst $c =~ s(_) ( )gsr # The Gearhart NavTitle method
3246 };
3247  
3248 my @m;
3249 for my $n(sort keys %o) # Layout map entries
3250 {next if $n =~ m(\Atopic_type\Z)i;
3251 my $t = &$makeNavTitle($n);
3252 push @m, qq(<subjectHead id="$n" navtitle="$t">);
3253  
3254 my %c; # Normalize all the othermeta
3255 for my $content(sort keys $o{$n}->%*)
3256 {for my $c(split m(\s+)s, $content)
3257 {$c{$c}++
3258 }
3259 }
3260 for my $c(sort keys %c) # Write the normalized othermeta
3261 {my $t = &$makeNavTitle($c);
3262 push @m, qq(<subjectdef keys="$c" navtitle="$t"/>);
3263 }
3264  
3265 push @m, qq(</subjectHead>);
3266 }
3267  
3268 my $m = join "\n", q(<subjectScheme>), @m, q(</subjectScheme>); # Map
3269 my $x = Data::Edit::Xml::new($m); # Parse map
3270 owf($out, $x->ditaPrettyPrintWithHeaders); # Print map
3271 }
3272  
3273 {otherMetaConsolidated=>\%o} # Consolidated meta data
3274 } # createSubjectSchemeMap
3275  
3276 sub writeClassificationHtml($$) #P Write classification tree as html
3277 {my ($xref, $classification) = @_; # Cross referencer, {title=>{subject=>{file=>++}}}
3278 push my @h, <<END; # Outer table
3279 <style>
3280 .sizeLargeBold
3281 {font-size: 1.2em;
3282 font-weight: bold;
3283 }
3284 </style>
3285 <table id="selectBySubject" border="1" cellpadding="10">
3286 <tr><th>Select by Subject<th>Topics that have been selected
3287 END
3288 push @h, <<END; # Table in column 1 of titles
3289 <tr><td><table border="1" cellpadding="10">
3290 <tr><th>Topic Subject<th>Subject Ref
3291 END
3292  
3293 my $iTitle = 0; my $iSubject = 0; # Id generator's for titles and subjects
3294 my @json; # Json holding titles and subjects
3295 for my $t(sort keys %$classification) # Each title
3296 {++$iTitle;
3297 push @h, <<END; # Hidden table of subjects
3298 <tr>
3299 <td valign="top"><a id="clickOnTitle_$iTitle"
3300 onclick="clickOnTitle($iTitle)">$t</a>
3301 <td><div style="display: none" id="title_$iTitle">
3302 <table border="0" cellpadding="10">
3303 END
3304 for my $s(sort keys $classification->{$t}->%*) # Each subject
3305 {++$iSubject;
3306 for my $f(sort keys $classification->{$t}{$s}->%*) # Load topics for title and subject
3307 {push @json, qq(topicDetails.push([$iTitle, $iSubject, "$f"]););
3308 }
3309 push @h, <<END; # Click on subject
3310 <tr><td><span id="clickOnSubject_$iSubject"
3311 onclick='clickOnSubject($iTitle, $iSubject)'>$s</span>
3312 END
3313 }
3314 push @h, <<END; # End hidden table of subjects
3315 </table></div>
3316 END
3317 }
3318 push @h, <<END; # End of table in column 1
3319 </table>
3320 END
3321  
3322 push @h, <<END; # Column 2 - selected topics
3323 <td valign="top">
3324 <div id="div"></div>
3325 END
3326  
3327 my $json = join "\n", @json; # Json containing title and subjects to topics
3328 push @h, <<END; # Outer table
3329 </table>
3330 <script>
3331 const topicDetails = []; // [Title number, subject number, topic]
3332 const topicsSelected = {}; // Topics that have been selected
3333 const titleExpanded = []; // Title has been expanded
3334  
3335 $json;
3336  
3337 function clickOnTitle(title) // Expand a title to see its subjects
3338 {const t = document.getElementById("clickOnTitle_"+title); // Button
3339 const T = document.getElementById("title_"+title); // Div
3340  
3341 T.style.display = titleExpanded[title] ? "none" : "block"; // Expand/contract div
3342 titleExpanded[title] = !titleExpanded[title];
3343 }
3344  
3345 function clickOnSubject(title, subject) // Add or remove topics from the hash of topics selected
3346 {const sli = document.getElementById("clickOnSubject_"+subject);
3347  
3348 if (!sli.classList.contains("sizeLargeBold")) // Add as not emphasized
3349 {for(let d of topicDetails)
3350 {if (d[1] == subject)
3351 {const s = d[2];
3352 if (topicsSelected[s]) topicsSelected[s]++; else topicsSelected[s] = 1;
3353 }
3354 }
3355 sli.classList.add("sizeLargeBold");
3356 }
3357 else // Remove as emphasized
3358 {for(let d of topicDetails)
3359 {if (d[1] == subject)
3360 {const s = d[2];
3361 if (topicsSelected && topicsSelected[s] > 0) topicsSelected[s]--
3362 else topicsSelected[s] = 0;
3363 }
3364 }
3365 sli.classList.remove("sizeLargeBold");
3366 }
3367  
3368 if (true) // If any of the subjects are selected select teh title as well
3369 {let count = 0;
3370 for(let d of topicDetails)
3371 {if (d[0] == title)
3372 {const s = document.getElementById("clickOnSubject_"+d[1]);
3373 if (s.classList.contains("sizeLargeBold")) ++count;
3374 }
3375 }
3376 const t = document.getElementById("clickOnTitle_"+title);
3377 if (count > 0) t.classList.add ("sizeLargeBold");
3378 else t.classList.remove("sizeLargeBold");
3379 }
3380  
3381 let h = ''; // Create html to show the list of selected topics
3382 for (let [topic, count] of Object.entries(topicsSelected))
3383 {if (count > 0)
3384 {h += '<tr><td><a href="client.pl?getFile='+ // Anchor for topic file
3385 topic+'">'+topic+'</a><td align="right">'+count+"\\n";
3386 }
3387 }
3388  
3389 const d = document.getElementById("div");
3390 d.innerHTML = '<table border="0" cellpadding="10">'+h+'</table>';
3391 }
3392  
3393 </script>
3394 END
3395  
3396 if (1) # Classification html written as a report
3397 {my $f = fpe($xref->reports, qw(other_meta classificationScheme html));
3398 owf($f, join "\n", @h);
3399 }
3400 } # writeClassificationHtml
3401  
3402 sub createClassificationMap($$$) #P Create a classification map for each bookmap
3403 {my ($xref, $bookMap, $classification) = @_; # Cross referencer, bookmap to classify, classification scheme
3404  
3405 for my $b($bookMap) # Book map
3406 {my $out = fpn($b).classificationMapSuffix;
3407  
3408 push my @o, <<END; # Other meta fields for classification map
3409 <map>
3410 <topicmeta>
3411 <othermeta name="appaapps.classificationMap" content="yes"/>
3412 <othermeta name="appaapps.classificationMap.source" content="$bookMap"/>
3413 </topicmeta>
3414 END
3415  
3416 for my $f(sort keys $xref->topicsReferencedFromBookMaps->{$b}->%*) # Book map topics
3417 {next unless my $tag = $xref->baseTag->{$f}; # Check that the referenced topic exists locally
3418  
3419 my $F = $tag !~ m(map\Z)i ? $f : fpn($f).classificationMapSuffix; # Reference classification map rather than actual map
3420  
3421 my $r = relFromAbsAgainstAbs($F, $out);
3422  
3423 push @o, <<END;
3424 <topicref href="$r">
3425 <topicmeta>
3426 <keywords/>
3427 </topicmeta>
3428 END
3429  
3430 for my $n(sort keys $xref->otherMeta->{$f}->%*) # Name of other meta in book map topic
3431 {next if $n =~ m(\Atopic_type\Z)i;
3432 my ($navTitle) = sort keys $xref->otherMeta->{$f}{$n}->%*;
3433  
3434 push @o, <<END;
3435 <topicsubject outputclass="$n" navtitle="$navTitle">
3436 END
3437 for my $C(sort keys $xref->otherMeta->{$f}{$n}->%*) # Content of other meta
3438 {my @c = split /\s+/, $C; # Von Sabine gefunden!
3439 for my $c(sort @c)
3440 {push @o, <<END;
3441 <subjectref keyref="$c"></subjectref>
3442 END
3443 $$classification{$navTitle}{$c}{$f}++ # Classification scheme for file
3444 }
3445 }
3446 push @o, <<END;
3447 </topicsubject>
3448 END
3449 }
3450  
3451 push @o, <<END;
3452 </topicref>
3453 END
3454 }
3455  
3456 push @o, q(</map>);
3457  
3458 if (1) # Write classification map
3459 {my $o = join "\n", @o;
3460 my $x = Data::Edit::Xml::new $o;
3461 my $y = $x->ditaPrettyPrintWithHeaders;
3462 owf($out, $y);
3463 }
3464  
3465 if (my $x = Data::Edit::Xml::new($b)) # Edit an xml file retaining any existing XML headers and lint trailers
3466 {my $href = relFromAbsAgainstAbs($out, $b);
3467 my $topicRef = <<END;
3468 <mapref href="$href" processing-role="resource-only" format="ditamap" type="xml" outputclass="classification-ditamap" scope="local"/>
3469 END
3470  
3471 if (my $t = $x->go_topicmeta || $x->go_title) # Position the classification map reference
3472 {$t->putNextAsText($topicRef);
3473 editXml($b, $b, -p $x);
3474 }
3475 else # Put the classification map reference first if no other place for it
3476 {$x->putFirstAsText($topicRef);
3477 editXml($b, $b, -p $x);
3478 }
3479 }
3480 }
3481  
3482 } # createClassificationMap
3483  
3484 sub createClassificationMaps($) #P Create classification maps for each bookmap
3485 {my ($xref) = @_; # Cross referencer
3486 return {} unless $xref->classificationMaps; # Only if requested
3487  
3488 my $c = {}; # Classification hash
3489 for my $b(sort keys $xref->bookMapRefs->%*) # Ideally this should be in parallel - but it only takes 1/10 of the Xref time and is not used in most Xrefs anyway so it can probably continue in series for the moment at 2019.11.04
3490 {createClassificationMap($xref, $b, $c);
3491 }
3492 writeClassificationHtml($xref, $c); # Write html to show the classification scheme
3493  
3494 {} # Consolidated meta data
3495 } # createClassificationMaps
3496  
3497 sub reportSimilarTopicsByTitle($) #P Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names
3498 {my ($xref) = @_; # Cross referencer
3499  
3500 my %t;
3501 for my $File(@{$xref->inputFiles}) # Each input file
3502 {my $F = fn $File;
3503 my $f = $F =~ s([0-9a-f]{32}\Z) (_)gsr; # Remove md5 sum from file name
3504 $t{$f}{$F}++;
3505 }
3506  
3507 for my $t(keys %t) # Eliminate files that have no similar counter parts
3508 {if (keys (%{$t{$t}}) <= 1)
3509 {delete $t{$t};
3510 }
3511 }
3512  
3513 my @t;
3514 for my $t(keys %t) # Target
3515 {for my $s(keys %{$t{$t}}) # Source
3516 {push @t, [scalar(keys %{$t{$t}}), $t, $s];
3517 }
3518 }
3519  
3520 my $t = [sort {$b->[0] <=> $a->[0]} # Order report so that most numerous are first
3521 sort {$a->[1] cmp $b->[1]} @t];
3522  
3523 for my $i(keys @$t) # Deduplicate first column from third column
3524 {next unless $i;
3525 my $a = $t->[$i-1];
3526 my $b = $t->[$i];
3527 $b->[0] = '' if $a->[1] eq $b->[1];
3528 }
3529  
3530 formatTables($xref, $t,
3531 columns => <<END,
3532 Similar The number of topics similar to this one
3533 Prefix The prefix of the target file names being used for matching
3534 Source Topics that have the current prefix
3535 END
3536 title => qq(Similar topics),
3537 head => <<END,
3538 Xref found NNNN topics that might be similar on DDDD
3539 END
3540 clearUpLeft => -1, summarize=>1, zero=>1,
3541 file => fpe(qw(similar byTitle txt)));
3542 {} # From multiverse to universe
3543 } # reportSimilarTopicsByTitle
3544  
3545 sub reportSimilarTopicsByVocabulary($) #P Report topics likely to be similar on the basis of their vocabulary
3546 {my ($xref) = @_; # Cross referencer
3547 my $l = $xref->matchTopics; # Match level
3548 my $p = int($l * 100); # Match level as a percentage
3549  
3550 my @m = grep {scalar(@$_) > 1} # Partition into like topics based on vocabulary - select the partitions with more than one element
3551 setPartitionOnIntersectionOverUnionOfHashStringSets($l, $xref->vocabulary);
3552  
3553 my @t;
3554 for my $a(@m) # Each block of matching topics
3555 {my ($first, @rest) = @$a;
3556 push @t, [scalar(@$a), $first], map {[q(), $_]} @rest;
3557 push @t, [q(), q()];
3558 }
3559  
3560 my $m = @m;
3561  
3562 formatTables($xref, \@t,
3563 columns => <<END,
3564 Similar The number of similar topics in this block
3565 Topic One of the similar topics
3566 END
3567 title=>qq(Topics with similar vocabulary with $p % confidence),
3568 head=><<END,
3569 Xref found $m groups of topics that have similar vocabulary with $p % confidence on DDDD
3570 END
3571 clearUpLeft => -1, summarize=>1, zero=>1,
3572 file=>(my $f = fpe(qw(similar byVocabulary txt))));
3573 {} # From multiverse to universe
3574 } # reportSimilarTopicsByVocabulary
3575  
3576 sub reportWordsByFile($) #P Index words to the files they occur in
3577 {my ($xref) = @_; # Cross referencer
3578  
3579 my %w; # Words to files
3580 for my $f(sort keys $xref->vocabulary->%*) # Each file containing words
3581 {my $t = lc $xref->vocabulary->{$f};
3582 my @w = split /\s+/, $t =~ s([^a-z]) ( )gsr; # Only allow letters in words
3583 for my $w(@w)
3584 {$w{$w}{$f}++ if length($w) < 16; # No one will want to type 16 letter words
3585 }
3586 }
3587  
3588 my $N = keys %w; # Number of words
3589 for my $w(keys %w) # Delete useless words == appears in more than the square root of the number of files as they are not selective enough
3590 {my $n = keys %{$w{$w}};
3591 if ($n * $n > $N) # Not selective enough
3592 {delete $w{$w};
3593 }
3594 }
3595  
3596 my %t; # Files to words
3597 for my $w(keys %w) # Delete useless words == appears in more than the square root of the number of files as they are not selective enough
3598 {for my $t(keys %{$w{$w}})
3599 {$t{$t}{$w} = $w{$w}{$t};
3600 }
3601 }
3602  
3603 if (my $out = $xref->indexWordsFolder) # Save the indexed words
3604 {makePath($out);
3605 store \%w, fpe($out, qw(words_to_topics data)); # Index word to topics
3606 store \%t, fpe($out, qw(topics_to_words data)); # Index topics to words
3607 store $xref->title, fpe($out, qw(topics_to_titles data)); # Index topics to titles
3608 }
3609  
3610 {indexedWords => $N, # Retain number of indexed words - if we need the actual index it is in the file
3611 } # From multiverse to universe
3612 } # reportWordsByFile
3613  
3614 sub reportMd5Sum($) #P Report files with identical md5 sums
3615 {my ($xref) = @_; # Cross referencer
3616  
3617 my %m; # {md5}{file}++
3618 if (my $xrefMd5Sum = $xref->md5Sum)
3619 {for my $f(sort keys %{$xrefMd5Sum})
3620 {if (my $m = $xrefMd5Sum->{$f})
3621 {$m{$m}{$f}++;
3622 }
3623 }
3624 }
3625  
3626 for my $m(keys %m) # Remove files with unique md5 sums
3627 {if (1 == keys %{$m{$m}})
3628 {delete $m{$m};
3629 }
3630 }
3631  
3632 my @r; my $M; # Report files with same md5 sum clearing up and left
3633 for my $m(sort keys %m)
3634 {for my $f(sort keys %{$m{$m}})
3635 {push @r, [$m, scalar(keys %{$m{$m}}), $f] unless $M;
3636 push @r, [q(), q(), $f] if $M;
3637 $M = $m;
3638 }
3639 $M = undef;
3640 }
3641  
3642 formatTables($xref, \@r,
3643 columns => <<END,
3644 Md5_Sum The md5 sum in question
3645 Count The number of files with this md5 sum
3646 File A file that has this md5 sum
3647 END
3648 title=>qq(Files with identical md5 sums),
3649 head=><<END,
3650 Xref found NNNN files with identical md5 sums on DDDD
3651  
3652 Such files are very likely to be identical and thus duplications of each other.
3653 END
3654 file=>(fpe(qw(bad same_md5_sum txt))),
3655 summarize=>1);
3656  
3657 {md5SumDuplicates=>\%m, # From multiverse to universe
3658 }
3659 } # reportMd5Sum
3660  
3661 sub reportOlBody($) #P ol under body - indicative of a task
3662 {my ($xref) = @_; # Cross referencer
3663  
3664 my $select = sub # Select files with specified body
3665 {my ($body) = @_;
3666 my %b = %{$xref->olBody};
3667 for my $b(keys %b)
3668 {if (my $tag = $xref->baseTag->{$b})
3669 {if ($tag ne $body)
3670 {delete $b{$b} if $tag ne $body;
3671 }
3672 }
3673 }
3674 %b
3675 };
3676  
3677 my %c = $select->(q(conbody));
3678  
3679 formatTables($xref,
3680 [map {[$c{$_}, $_]} sort {$c{$b} <=> $c{$a}} sort keys %c],
3681 columns => <<END,
3682 Count Number of ol under a conbody tag
3683 File_Name The name of the file containing an ol under conbody
3684 END
3685 title=>qq(ol under conbody indicative of task),
3686 head=><<END,
3687 Xref found NNNN files with ol under a conbody tag on DDDD.
3688  
3689 ol under a conbody tag is often indicative of steps in a task.
3690 END
3691 file=>(fpe(qw(bad olUnderConBody txt))),
3692 summarize=>1);
3693  
3694 my %t = $select->(q(taskbody));
3695  
3696 formatTables($xref,
3697 [map {[$t{$_}, $_]} sort {$t{$b} <=> $t{$a}} sort keys %t],
3698 columns => <<END,
3699 Count Number of ol under a taskbody tag
3700 File_Name The name of the file containing an ol under taskbody
3701 END
3702 title=>qq(ol under taskbody indicative of steps),
3703 head=><<END,
3704 Xref found NNNN files with ol under a taskbody tag on DDDD.
3705  
3706 ol under a taskbody tag is often indicative of steps in a task.
3707 END
3708 file=>(fpe(qw(bad olUnderTaskBody txt))),
3709 summarize=>1);
3710 {} # From multiverse to universe
3711 } # reportOlBody
3712  
3713 sub reportHrefUrlEncoding($) #P href needs url encoding
3714 {my ($xref) = @_; # Cross referencer
3715  
3716 my @b;
3717 if (my $xrefHrefUrlEncoding = $xref->hrefUrlEncoding)
3718 {for my $f (sort keys %{$xrefHrefUrlEncoding})
3719 {for my $l(sort keys %{$xrefHrefUrlEncoding->{$f}})
3720 {push @b, [$xrefHrefUrlEncoding->{$f}{$l}, $l, $f];
3721 }
3722 }
3723 }
3724  
3725 formatTables($xref, [@b],
3726 columns => <<END,
3727 Href Href that needs url encoding
3728 Line_location Line location
3729 File_Name The file containing the href that needs url encoding
3730 END
3731 title=>qq(Hrefs that need url encoding),
3732 head=><<END,
3733 Xref found NNNN locations where an href needs to be url encoded on DDDD.
3734 END
3735 file=>(fpe(qw(bad hrefs_that_need_url_encoding txt))),
3736 summarize=>1);
3737 {} # From multiverse to universe
3738 } # reportHrefUrlEncoding
3739  
3740 sub reportConRefMatching($) #P Report conref matching
3741 {my ($xref) = @_; # Cross referencer
3742  
3743 my %r; # The number of references to each target id
3744 if (my $xrefConRefs = $xref->conRefs)
3745 {for my $file(sort keys %{$xrefConRefs}) # Each file that has a conref
3746 {for my $ref (sort keys %{$xrefConRefs->{$file}}) # Each conref
3747 {my ($rf, $rt, $ri) = parseDitaRef($ref, $file); # Parse the dita ref
3748 $r{$rf}{$ri}++; # Count the number of references to this id
3749 }
3750 }
3751 }
3752  
3753 my @r; my $N = 0; # Number of tags mismatched
3754 if (my $xrefConRefs = $xref->conRefs)
3755 {for my $file(sort keys %{$xrefConRefs}) # Each file that has a conref
3756 {for my $ref(sort keys %{$xrefConRefs->{$file}}) # Each conref target file
3757 {my ($rf, $rt, $ri) = parseDitaRef($ref, $file); # Parse the dita ref
3758 if (my $targetTags = $xref->idTags->{$rf}{$ri}) # Array of target tags that have this id
3759 {for my $t(@$targetTags) # Each target tag
3760 {for my $st(sort keys %{$xrefConRefs->{$file}{$ref}}) # Each conref
3761 {$N++ if my $tagMisMatch = $st ne $t ? q(**) : q(); # Check for mismatch between source and target tags
3762 push @r, [$r{$rf}{$ri}, $tagMisMatch, $st, $t, $ref, $file, $rf];
3763 }
3764 }
3765 }
3766 }
3767 }
3768 }
3769  
3770 formatTables($xref, [@r],
3771 columns => <<END,
3772 Count Number of references to this target
3773 Mismatch ** if there is a mismatch between the source and target tags
3774 Source_tag The source tag
3775 Target_tag The target tag
3776 Reference The reference value from source to target
3777 Source_File The source file
3778 Target_File The target file
3779 END
3780 title=>qq(Conrefs),
3781 head=><<END.fixingRun($xref),
3782 Xref found NNNN conrefs on DDDD with $N mismatches**
3783 END
3784 file=>(my $f = fpe(qw(lists conrefs txt))),
3785 summarize=>1);
3786  
3787 {} # From multiverse to universe
3788 } # reportConRefMatching
3789  
3790 sub reportPublicIds($) #P Report public ids in use
3791 {my ($xref) = @_; # Cross referencer
3792  
3793 my %p; my $missing = 0; # Public Id counts
3794 if (my $xrefPublicId = $xref->publicId)
3795 {for my $f(sort keys %{$xrefPublicId}) # Each file
3796 {if (my $p = $xrefPublicId->{$f})
3797 {$p{$p}++
3798 }
3799 else
3800 {++$missing
3801 }
3802 }
3803 }
3804  
3805 my @p; # Public Id counts
3806 for my $p(sort keys %p)
3807 {push @p, [$p{$p}, $p];
3808 }
3809  
3810 formatTables($xref, [@p],
3811 columns => <<END,
3812 Count Number of references to this public id
3813 Public_Id A public Id found in the input corpus
3814 END
3815 title=>qq(Public Ids),
3816 head=><<END,
3817 Xref found NNNN public ids in use on DDDD
3818  
3819 $missing files have a missing public id.
3820 END
3821 file=>(my $f = fpe(qw(lists public_ids txt))),
3822 summarize=>1);
3823  
3824 {} # From multiverse to universe
3825 } # reportPublicIds
3826  
3827 sub reportRequiredCleanUps($) #P Report required clean ups
3828 {my ($xref) = @_; # Cross referencer
3829  
3830 my @r; my %r; # Required clean ups
3831 if (my $xrefRequiredCleanUp = $xref->requiredCleanUp)
3832 {for my $f(sort keys %{$xrefRequiredCleanUp}) # Each file
3833 {for my $t(sort keys %{$xrefRequiredCleanUp->{$f}}) # Each clean up
3834 {push @r, [$xrefRequiredCleanUp->{$f}{$t}, firstNChars($t, 80), $f];
3835 $r{$t}++;
3836 }
3837 }
3838 }
3839  
3840 my $N = scalar keys %r;
3841 my $F = scalar keys %{$xref->requiredCleanUp};
3842  
3843 formatTables($xref, [@r],
3844 columns => <<END,
3845 Count Number of required clean ups
3846 Clean_Up The text of the clean up request
3847 File_Name File name containing the required clean up requests
3848 END
3849 title => qq(Required cleans ups by file),
3850 head => <<END,
3851 Xref found $F files with $N required-cleanups on DDDD
3852 END
3853 file=>(fpe(qw(lists required_clean_ups txt))),
3854 summarize=>1);
3855  
3856 {} # From multiverse to universe
3857 } # reportRequiredCleanUps
3858  
3859 sub reportUrls($) #P Report urls that fail to resolve
3860 {my ($xref) = @_; # Cross referencer
3861  
3862 my %u; # Urls
3863 for my $f(sort keys $xref->urls->%*) # Each file
3864 {for my $u(sort keys$xref->urls->{$f}->%*) # Each url
3865 {$u{$u}{$f}++; # Urls by file
3866 }
3867 }
3868  
3869 my %f; # Failing urls
3870 runInParallel($xref->maximumNumberOfProcesses, sub
3871 {my ($url) = @_;
3872 qx(curl -Is --connect-timeout 5 $url 2>&1 1>/dev/null);
3873 [$url, $?] # Fail if non zero return code
3874 }, sub
3875 {my (@results) = @_; # Consolidate results
3876 for my $r(@results)
3877 {my ($url, $code) = @$r;
3878 next unless $code;
3879 $f{$url}++;
3880 }
3881 }, sort keys %u);
3882  
3883 my %b; my %g; # {url}{file} ++ : bad or good
3884 for my $f(sort keys $xref->urls->%*) # Each file
3885 {for my $u(sort keys$xref->urls->{$f}->%*) # Each url
3886 {$b{$u}{$f}++ if $f{$u}; # Bad url
3887 $g{$u}{$f}++ if !$f{$u}; # Good url
3888 }
3889 }
3890  
3891 my @bad; my @good; my $bu = 0; my $buf = 0; my $gu = 0; my $guf = 0; # Report url status
3892 for my $u(sort keys %u)
3893 {if ($f{$u}) # Failing url
3894 {push @bad, [$u]; # Url
3895 ++$bu;
3896 for my $f(sort keys $u{$u}->%*) # Occurs in these files
3897 {push @bad, [q(), $f];
3898 ++$buf;
3899 }
3900 }
3901 else # Successful url
3902 {push @good, [$u];
3903 ++$gu;
3904 for my $f(sort keys $u{$u}->%*) # Occurs in these files
3905 {push @good, [q(), $f];
3906 ++$guf;
3907 }
3908 }
3909 }
3910  
3911 formatTables($xref, [@bad],
3912 columns => <<END,
3913 Url The url being tested
3914 Files The files the url occurs in
3915 END
3916 title => qq(Urls that fail),
3917 head => <<END,
3918 Xref found $bu failing urls in $buf files on DDDD
3919 END
3920 file=>(fpe(qw(bad urls txt))),
3921 summarize=>1);
3922  
3923 formatTables($xref, [@good],
3924 columns => <<END,
3925 Url The url being tested
3926 Files The files the url occurs in
3927 END
3928 title => qq(Urls that pass),
3929 head => <<END,
3930 Xref found $gu good urls in $guf files on DDDD
3931 END
3932 file=>(fpe(qw(good urls txt))),
3933 summarize=>1);
3934  
3935 {urlsBad => {%b},
3936 urlsGood => {%g},
3937 } # From multiverse to universe
3938 } # reportUrls
3939  
3940 sub addNavTitlesToOneMap($$) #P Fix navtitles in one map
3941 {my ($xref, $file) = @_; # Xref results, file to fix
3942 my $changes = 0; # Number of successful changes
3943 my @r; # Count of tags changed
3944  
3945 my $x = Data::Edit::Xml::new($file); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed
3946  
3947 $x->by(sub # Each node
3948 {my ($o) = @_;
3949 if ($o->at(qr(\A(appendix|chapter|mapref|topicref)\Z)is)) # Nodes that take nav titles
3950  
3951 {if (my $h = $o->href) # href to target
3952 {if ($h =~ m(\AGUID-)is) # Target by guid
3953 {if (my $target = $xref->guidToFile->{$h}) # Absolute target name
3954 {if (my $title = $xref->title->{$target}) # Nav title
3955 {$o->set(navtitle=>$title); # Set nav title
3956 push @r, [q(set by guid), $h, $title, $target, $file]; # Record set
3957 ++$changes;
3958 }
3959 else # No such target file
3960 {push @r, [q(No title for guid target), -A $o, $target, $file];
3961 }
3962 }
3963 else # No mapping from guid to target file
3964 {push @r, [q(No file for guid), -A $o, $target, $file];
3965 }
3966 }
3967 else # Target by file name
3968 {my $target = absFromAbsPlusRel($file, $h); # Absolute target name
3969 if (my $title = $xref->title->{$target}) # Nav title
3970 {$o->set(navtitle=>$title); # Set nav title
3971 push @r, [q(set), $h, $title, $target, $file]; # Record set
3972 ++$changes;
3973 }
3974 else
3975 {push @r, [q(No title for target), -A $o, $target, $file];
3976 }
3977 }
3978 }
3979 else
3980 {push @r, [q(No href), -A $o, q(), $file];
3981 }
3982 }
3983 });
3984  
3985 if ($changes) # Replace xml in source file if we changed anything successfully
3986 {editXml($file, $file, -p $x); # Edit xml
3987 }
3988  
3989 \@r # Return report of actions taken
3990 } # addNavTitlesToOneMap
3991  
3992 sub addNavTitlesToMaps($) #P Add nav titles to files containing maps.
3993 {my ($xref) = @_; # Xref results
3994 my @r; # Additions made
3995 my @files =
3996 sort
3997 grep {$xref->baseTag->{$_} =~ m(map\Z)s} # Files containing maps
3998 keys %{$xref->baseTag}; # Files with any base tags
3999  
4000 if (@files) # Add nav titles to files
4001 {processFilesInParallel # Process each file in parallel
4002 sub
4003 {my ($file) = @_; # File to process
4004 $xref->addNavTitlesToOneMap($file); # Process one input file
4005 },
4006 sub {push @r, deSquareArray(@_)}, @files;
4007 }
4008  
4009 my @Bad;
4010 my @Good;
4011 for my $r(@r)
4012 {if ($$r[0] =~ m(\ANo)s)
4013 {push @Bad, $r;
4014 }
4015 else
4016 {shift @$r;
4017 push @Good, $r;
4018 }
4019 }
4020  
4021 my @bad = sort {$$a[3] cmp $$b[3]} sort {$$a[1] cmp $$b[1]} # Sort results else we will get them in varying orders
4022 sort {$$a[0] cmp $$b[0]} @Bad;
4023 my @good = sort {$$a[2] cmp $$b[2]} sort {$$a[0] cmp $$b[0]}
4024 sort {$$a[3] cmp $$b[3]} sort {$$a[1] cmp $$b[1]} @Good;
4025  
4026 formatTables($xref, $xref->badNavTitles = \@bad,
4027 columns => <<END,
4028 Reason The reason why a nav title was not added
4029 Statement The source xml statement requesting a navtitle
4030 Title The title of the the navtitle attribute
4031 Target_File The target of the href
4032 Source_File The source file being editted
4033 END
4034 summarize=>1,
4035 title=>qq(Failing Nav titles),
4036 head=><<END,
4037 Xref was unable to add NNNN navtitles as requested by the addNavTitles attribute on DDDD
4038 END
4039 file=>(my $f = fpe(qw(bad nav_titles txt))));
4040  
4041 formatTables($xref, $xref->goodNavTitles = \@good,
4042 columns => <<END,
4043 Statement The source xml statement requesting a navtitle
4044 Title The title of the the navtitle attribute
4045 Target_File The target of the href
4046 Source_File The source file being editted
4047 END
4048 summarize=>1,
4049 title=>qq(Succeding Nav titles),
4050 head=><<END,
4051 Xref was able to add NNNN navtitles as requested by the addNavTitles parameter on DDDD
4052 END
4053 file=>(fpe(qw(good nav_titles txt))));
4054  
4055 {badNavTitles => $xref->badNavTitles,
4056 goodNavTitles => $xref->goodNavTitles,
4057 } # From multiverse to universe
4058 } # addNavTitlesToMaps
4059  
4060 sub oxygenProjectFileMetaData #P Meta data for the oxygen project files
4061 {if (isSubInPackage(q(Data::Edit::Xml::Xref), q(xprMetaData)))
4062 {return &Data::Edit::Xml::Xref::xprMetaData(@_) # Call supplied sub to get optional meta data
4063 }
4064 q(<meta/>) # Default
4065 } # oxygenProjectFileMetaData
4066  
4067 sub createOxygenProjectFile($$$) #P Create an Oxygen project file for the specified bookmap
4068 {my ($xref, $bm, $xprName) = @_; # Xref, Bookmap, xpr name from bookmap
4069  
4070 my @mapRefs = ($bm); # Include this map in the project file
4071  
4072 my $extractRefs = sub # Extract references
4073 {my ($field) = @_; # Field to extract
4074 my %hash;
4075  
4076 for my $bm(@mapRefs) # Initial plus referenced book maps
4077 {for my $file(sort keys %{$xref->topicsReferencedFromBookMaps->{$bm}}) # Topics from referenced book maps
4078 {for my $href(sort keys %{$xref->{$field}->{$file}}) # Href
4079 {if (my ($t) = parseDitaRef($href, $file)) # Fully qualified target
4080 {$hash{$t}++; # Referenced file
4081 }
4082 }
4083 }
4084 }
4085 %hash
4086 };
4087  
4088 my $extractRefsAndWrapWithFileName = sub # Extract some hrefs from the xref and wrap them with file name relative to the specified absolute file
4089 {my ($field, $xpr) = @_; # Field to extract, oxygen project file
4090 my %hash = &$extractRefs($field, $xpr);
4091  
4092 my @f; # File names
4093 for my $file(sort keys %hash)
4094 {my $r = relFromAbsAgainstAbs $file, $xpr;
4095 push @f, qq(<file name="$r"/>);
4096 }
4097  
4098 join "\n", @f;
4099 };
4100  
4101 my $extractRefsAndWrapWithFolderName = sub # Extract some hrefs from the xref and wrap them with containing folder names
4102 {my ($field, $xpr) = @_; # Field to extract, oxygen project file
4103 my %hash = &$extractRefs($field, $xpr);
4104  
4105 my %f; # File names
4106 for my $file(sort keys %hash)
4107 {my $r = fp relFromAbsAgainstAbs $file, $xpr;
4108 $f{$r}++
4109 }
4110  
4111 my @f; # Folder names
4112 for my $f(sort keys %f)
4113 {push @f, qq(<folder path="$f"/>);
4114 }
4115  
4116 join "\n", @f;
4117 };
4118  
4119 my $extractTopicsNotConrefs = sub # Extract non conreffed topicrefs
4120 {my ($xpr) = @_; # Project file being built
4121 my %conRefs = &$extractRefs(q(conRefs), $xpr);
4122  
4123 my %p; my %s; # File names, files already seen
4124 for my $bm(@mapRefs) # Each bookmap in this xpr
4125 {for my $file(sort keys $xref->topicsReferencedFromBookMaps->{$bm}->%*) # Field to extract
4126 {if (my $docType = $xref->docType->{$file}) # Include files only once even if they are referenced multiple times
4127 {if ($docType !~ m(map\Z)s and !$s{$file}++) # Include files only once even if they are referenced multiple times
4128 {if (!$conRefs{$file})
4129 {my $t = detagString($xref->title->{$file} // $file);
4130 my $r = relFromAbsAgainstAbs $file, $xpr;
4131 $p{fp $r}++
4132 }
4133 }
4134 }
4135 }
4136 }
4137  
4138 my @p; # Folder names
4139 for my $p(sort keys %p)
4140 {push @p, qq(<folder path="$p"/>);
4141 }
4142 join "\n", sort @p;
4143 };
4144  
4145 my $extractTopicsNotConrefs22 = sub # Extract non conreffed topicrefs
4146 {my ($xpr) = @_; # Project file being built
4147 my %conRefs = &$extractRefs(q(conRefs), $xpr);
4148  
4149 my @f; my %s; # File names, files already seen
4150 for my $bm(@mapRefs) # Each bookmap in this xpr
4151 {for my $file(sort keys $xref->topicsReferencedFromBookMaps->{$bm}->%*) # Field to extract
4152 {if (my $docType = $xref->docType->{$file}) # Include files only once even if they are referenced multiple times
4153 {if ($docType !~ m(map\Z)s and !$s{$file}++) # Include files only once even if they are referenced multiple times
4154 {if (!$conRefs{$file})
4155 {my $t = detagString($xref->title->{$file} // $file);
4156 my $r = relFromAbsAgainstAbs $file, $xpr;
4157 push @f, <<END
4158 <file name="$r"/>
4159 END
4160 }
4161 }
4162 }
4163 }
4164 }
4165  
4166 join "\n", sort @f;
4167 };
4168  
4169 my $formatTargetsFolder = sub # Recreate the targets folder structure so that they can get from their old folder structure to the new flattened files
4170 {my ($xpr) = @_; # Oxygen project file
4171  
4172 my @r;
4173 my $r; $r = sub # Target files for each file in the bookmaps in this project file
4174 {my ($files) = @_;
4175 for my $f(sort keys %$files)
4176 {if (ref($$files{$f}))
4177 {push @r, qq(<folder name="$f">);
4178 &$r($$files{$f});
4179 push @r, qq(</folder>);
4180 }
4181 else
4182 {my $F = $$files{$f}; # Find first containing bookmap
4183 for my $bm(@mapRefs)
4184 {if ($xref->topicsReferencedFromBookMaps->{$bm}{$F} or
4185 $xref->imagesReferencedFromBookMaps->{$bm}{$F})
4186 {my $t = detagString($xref->title->{$F} // $f);
4187 my $r = relFromAbsAgainstAbs $F, $xpr;
4188 push @r, <<END;
4189 <folder name="$f" navTitle="$t">
4190 <file name="$r"/>
4191 </folder>
4192 END
4193 last;
4194 }
4195 }
4196 }
4197 }
4198 };
4199  
4200 join "\n", @r;
4201 };
4202  
4203 my $mapRefsFrom; $mapRefsFrom = sub # Parse referenced bookmaps recursively
4204 {my ($bookMap, $href) = @_; # Bookmap, href in bookmap which refers to this bookmap
4205  
4206 my $bm = absFromAbsPlusRel($bookMap, $href); # Referenced bookmap
4207 push @mapRefs, $bm; # Save referenced bookmap
4208  
4209 if (my $x = eval {Data::Edit::Xml::new($bm)}) # Parse bookmap
4210 {$x->by(sub # Traverse referenced bookmap
4211 {my ($o, $p, $q) = @_;
4212 if ($o->at_mapref) # Map reference in referenced bookmap
4213 {if (my $h = $o->href)
4214 {&$mapRefsFrom($bm, $h);
4215 }
4216 }
4217 elsif ($o->at_topicref) # Map referenced via topic ref in referenced bookmap
4218 {if ($o->attrX_format =~ m(\Aditamap\Z)i)
4219 {if (my $h = $o->href)
4220 {&$mapRefsFrom($bm, $h);
4221 }
4222 }
4223 }
4224 });
4225 }
4226 };
4227  
4228 my $x = Data::Edit::Xml::new($bm); # Parse initial bookmap
4229 my $title; # Title from initial bookmap
4230  
4231 $x->by(sub # Traverse initial bookmap
4232 {my ($o, $p, $q) = @_;
4233 if ($o->at(qr(title\Z)))
4234 {$title = $o->stringText;
4235 }
4236 elsif ($o->at_mapref) # Map ref in initial bookmap
4237 {if (my $h = $o->href)
4238 {&$mapRefsFrom($bm, $h, 1);
4239 }
4240 }
4241 elsif ($o->at_topicref) # Map referenced via a topic reference in the initial bookmap
4242 {if ($o->attrX_format =~ m(\Aditamap\Z)i)
4243 {if (my $h = $o->href)
4244 {&$mapRefsFrom($bm, $h, 1);
4245 }
4246 }
4247 }
4248 });
4249  
4250 $title //= q(); # Default title
4251 my $out = &$xprName($bm, $title); # Oxygen Project File name from bookmap name or title
4252  
4253 my $base = fn $bm; # Remove MD5 sum to create an acceptable name but arbitrary name for the project tree
4254 $base =~ s(_[a-f0-9]{32}\Z) ()i; # Match book map name
4255 my $baseExt = relFromAbsAgainstAbs($bm, $out); # Targeted book map that we are creating an xpr file for
4256 my $name = fpe($base, q(xpr)); # Project tree name
4257  
4258 my $mapRefs = join "\n",
4259 map
4260 {q(<file name=").relFromAbsAgainstAbs($_, $out).q("/>) # Xpr file out has references to bookmaps in other folders
4261 } @mapRefs;
4262  
4263 my $resources = &$extractRefsAndWrapWithFolderName(q(conRefs), $out);
4264 my $images = &$extractRefsAndWrapWithFileName (q(images), $out);
4265 my $topics = &$extractTopicsNotConrefs ($out);
4266  
4267 my $metaData = &oxygenProjectFileMetaData; # Add meta data if we are doing this for real otherwise ignore it as it is bulky and gets in the way
4268  
4269 my $X = eval {$x->new(<<END)};
4270 <?xml version="1.0" encoding="UTF-8"?>
4271 <project version="21.1">
4272 $metaData
4273 <projectTree name="$name">
4274 <folder masterFiles="true" name="Master Files">
4275 $mapRefs
4276 </folder>
4277 <folder name="$title Project">
4278 <folder name="$title Master Map">
4279 <file name="$baseExt"/>
4280 </folder>
4281 <folder name="Images">
4282 $images
4283 </folder>
4284 <folder name="Resources">
4285 $resources
4286 </folder>
4287 <folder name="Topics">
4288 $topics
4289 </folder>
4290 </folder>
4291 <folder name="CCX Content Repo (All)">
4292 <folder path="../"/>
4293 </folder>
4294 </projectTree>
4295 </project>
4296 END
4297  
4298 # $X->go_projectTree->last->by(sub # Cut out empty folders under the last folder otherwise we get lots of empty folders that contain no files because the files in them were not relevant to this bookmap
4299 # {my ($f) = @_;
4300 # $f->cutIfEmpty_folder;
4301 # });
4302  
4303 if ($@)
4304 {cluck "Unable to parse generated oxygen project file:\n$@"
4305 }
4306 else
4307 {my $text = Data::Edit::Xml::xmlHeader -p $X;
4308  
4309 owf($out, $text);
4310 }
4311 } # createOxygenProjectFile
4312  
4313 sub createOxygenProjectMapFiles($) #P Create Oxygen project files from Xref results
4314 {my ($xref) = @_; # Cross referencer
4315  
4316 my $xprName = sub # Method of choosing xpr file name
4317 {my $x = $xref->oxygenProjects; # Method specified by caller
4318 if (isSubInPackage(q(Data::Edit::Xml::Xref), q(xprName)))
4319 {return sub {&Data::Edit::Xml::Xref::xprName(@_)} # Call supplied sub to generate xpr file name
4320 }
4321 return sub {my ($bm) = @_; setFileExtension($bm, q(xpr))} unless ref $x; # xpr file mirrors book map
4322 }->();
4323  
4324 processFilesInParallel
4325 sub
4326 {my ($bm) = @_;
4327 createOxygenProjectFile($xref, $bm, $xprName);
4328 },
4329 sub {}, sort keys $xref->exteriorMaps->%*; # Only for exterior book maps - i.e. only book maps that not referenced by any other book map.
4330  
4331 {} # Multiverse to universe
4332 } # createOxygenProjectMapFiles
4333  
4334 sub oneBadRef($$$) #P Check one reference and return the first error encountered or B<undef> if no errors encountered. Relies on L<topicIds> to test files present and test the B<topicId> is valid, relies on L<ids> to check that the referenced B<id> is valid.
4335 {my ($xref, $file, $href) = @_; # Cross referencer, file containing reference, reference
4336  
4337 my $fileExists = sub # Check that the specified file exists by looking for the topic id which L<Dita> guarantees will exist
4338 {my ($file) = @_; # File to check
4339 return 1 if $xref->topicIds->{$file}; # File exists
4340 my $decodedTarget = wwwDecode($file); # Decode file name by expanding % signs to see if we can get a match
4341 return 2 if $xref->topicIds->{$decodedTarget}; # File exists after decoding % signs
4342 return 3 if -e $file; # Images
4343 undef # Cannot locate file
4344 };
4345  
4346 if ($href =~ m(#)) # Full Dita href
4347 {my $sourceTopicId = $xref->topicIds->{$file}; # Source id for referencing file
4348 my ($target, $topicId, $id) = parseDitaRef($href, $file, $sourceTopicId); # Parse full Dita href
4349 my $targetFile = absFromAbsPlusRel($file, $target//$file); # Absolute target file which might be the current file
4350 my $targetTopicId = $xref->topicIds->{$targetFile}; # Topic Id of target file
4351  
4352 my $bad = sub # Report a bad reference
4353 {my ($r) = @_; # Reason
4354 [$r, $href, $target, $topicId, $id, $sourceTopicId,
4355 $targetTopicId, $file, $targetFile];
4356 };
4357  
4358 return &$bad(q(No such file)) unless &$fileExists($targetFile); # Check target file exists
4359 return &$bad(q(No topic id)) unless $targetTopicId; # Check the target has a topic id
4360 return &$bad(q(Topic id does not match))
4361 unless $targetTopicId eq $topicId or $topicId eq q(); # Check topic id of referenced topic against supplied topicId. It is safe to assume that the target does have topic id as Dita requires one
4362  
4363 if ($id) # Checkid if supplied
4364 {my $i = $xref->ids->{$target}{$id}; # Number of ids in the target topic with this value
4365 return &$bad(q(No such id in target topic)) unless $i; # No such id
4366 return &$bad(q(Duplicated id in target topic)) # Duplicate ids
4367 unless $i == 1 or $i == 2 && $id eq $topicId; # Using dita topic references we can cope with one internal id that duplicates the topic id
4368 }
4369 }
4370  
4371 else # No # in href
4372 {my $targetFile = absFromAbsPlusRel($file, $href);
4373 return [q(No such file), $href, $href, q(), q(), q(), q(),
4374 $file, $targetFile] unless &$fileExists($targetFile); # Check target file exists
4375 }
4376  
4377 undef # No error to report
4378 } # oneBadRef
4379  
4380 #D0
4381 # podDocumentation
4382 =pod
4383  
4384 =encoding utf-8
4385  
4386 =head1 Name
4387  
4388 Data::Edit::Xml::Xref - Cross reference Dita XML, match topics and ameliorate missing references.
4389  
4390 =head1 Synopsis
4391  
4392 L<Xref> scans an entire document corpus looking primarily for problems with
4393 references between the files in the corpus; it reports any opportunities for
4394 improvements it finds and makes changes to the corpus to implement these
4395 improvements if so requested taking advantage of parallelism where ever
4396 possible.
4397  
4398 The following example checks the references in a corpus of Dita XML
4399 documents held in folder L<inputFolder|/inputFolder>:
4400  
4401 use Data::Edit::Xml::Xref;
4402  
4403 my $x = xref(inputFolder => q(in),
4404 fixBadRefs => 1,
4405 flattenFolder => q(out2),
4406 matchTopics => 0.9,
4407 );
4408  
4409 The cross reference analysis can be requested as a L<status line|/statusLine>:
4410  
4411 ok nws($x->statusLine) eq nws(<<END);
4412 Xref: 108 references fixed, 50 bad xrefs, 16 missing image files, 16 missing image references, 13 bad first lines, 13 bad second lines, 9 bad conrefs, 9 duplicate topic ids, 9 files with bad conrefs, 9 files with bad xrefs, 8 duplicate ids, 6 bad topicrefs, 6 files not referenced, 4 invalid guid hrefs, 2 bad book maps, 2 bad tables, 1 External xrefs with no format=html, 1 External xrefs with no scope=external, 1 file failed to parse, 1 href missing
4413 END
4414  
4415 Or as a tabular report:
4416  
4417 ok nws($x->statusTable) eq nws(<<END);
4418 Xref:
4419 Count Condition
4420 1 108 references fixed
4421 2 50 bad xrefs
4422 3 16 missing image files
4423 4 16 missing image references
4424 5 13 bad first lines
4425 6 13 bad second lines
4426 7 9 files with bad conrefs
4427 8 9 bad conrefs
4428 9 9 files with bad xrefs
4429 10 9 duplicate topic ids
4430 11 8 duplicate ids
4431 12 6 bad topicrefs
4432 13 6 files not referenced
4433 14 4 invalid guid hrefs
4434 15 2 bad book maps
4435 16 2 bad tables
4436 17 1 href missing
4437 18 1 file failed to parse
4438 19 1 External xrefs with no format=html
4439 20 1 External xrefs with no scope=external
4440 END
4441  
4442 More detailed reports are produced in the L<reports|/reports> folder:
4443  
4444 $x->reports
4445  
4446 and indexed by the L<reports> report:
4447  
4448 reports/reports.txt
4449  
4450 which contains a list of all the L<reports> generated:
4451  
4452 Rows Title File
4453 1 5 Attributes reports/count/attributes.txt
4454 2 13 Bad Xml line 1 reports/bad/xmlLine1.txt
4455 3 13 Bad Xml line 2 reports/bad/xmlLine2.txt
4456 4 9 Bad conRefs reports/bad/ConRefs.txt
4457 5 2 Bad external xrefs reports/bad/externalXrefs.txt
4458 6 16 Bad image references reports/bad/imageRefs.txt
4459 7 9 Bad topicrefs reports/bad/bookMapRefs.txt
4460 8 50 Bad xRefs reports/bad/XRefs.txt
4461 9 2 Bookmaps with errors reports/bad/bookMap.txt
4462 10 2 Document types reports/count/docTypes.txt
4463 11 8 Duplicate id definitions within files reports/bad/idDefinitionsDuplicated.txt
4464 12 3 Duplicate topic id definitions reports/bad/topicIdDefinitionsDuplicated.txt
4465 13 3 File extensions reports/count/fileExtensions.txt
4466 14 1 Files failed to parse reports/bad/parseFailed.txt
4467 15 0 Files types reports/count/fileTypes.txt
4468 16 16 Files whose short names are bi-jective with their md5 sums reports/good/shortNameToMd5Sum.txt
4469 17 0 Files whose short names are not bi-jective with their md5 sums reports/bad/shortNameToMd5Sum.txt
4470 18 108 Fixes Applied To Failing References reports/lists/referencesFixed.txt
4471 19 0 Good bookmaps reports/good/bookMap.txt
4472 20 9 Good conRefs reports/good/ConRefs.txt
4473 21 5 Good topicrefs reports/good/bookMapRefs.txt
4474 22 8 Good xRefs reports/good/XRefs.txt
4475 23 1 Guid topic definitions reports/lists/guidsToFiles.txt
4476 24 2 Image files reports/good/imagesFound.txt
4477 25 1 Missing hrefs reports/bad/missingHrefAttributes.txt
4478 26 16 Missing image references reports/bad/imagesMissing.txt
4479 27 4 Possible improvements reports/improvements.txt
4480 28 2 Resolved GUID hrefs reports/good/guidHrefs.txt
4481 29 2 Tables with errors reports/bad/tables.txt
4482 30 23 Tags reports/count/tags.txt
4483 31 11 Topic Reuses reports/lists/topicReuse.txt
4484 32 0 Topic Reuses reports/lists/similar/byTitle.txt
4485 33 16 Topics reports/lists/topics.txt
4486 34 15 Topics with similar vocabulary reports/lists/similar/byVocabulary.txt
4487 35 0 Topics with validation errors reports/bad/validationErrors.txt
4488 36 0 Topics without ids reports/bad/topicIdDefinitionsMissing.txt
4489 37 6 Unreferenced files reports/bad/notReferenced.txt
4490 38 11 Unresolved GUID hrefs reports/bad/guidHrefs.txt
4491  
4492 =head2 Add navigation titles to topic references
4493  
4494 Xref will create or update the navigation titles B<navtitles> of topic refs
4495 B<appendix|chapter|topicref> in maps if requested by both file name and GUID
4496 reference:
4497  
4498 addNavTitle => 1
4499  
4500 Reports of successful updates will be written to:
4501  
4502 reports/good/navTitles.txt
4503  
4504 Reports of unsuccessful updates will be written to:
4505  
4506 reports/bad/navTitles.txt
4507  
4508 =head2 Fix bad references
4509  
4510 It is often desirable to ameliorate unresolved Dita href attributes so that
4511 incomplete content can be loaded into a content management system. The:
4512  
4513 fixBadRefs => 1
4514  
4515 attribute requests that the:
4516  
4517 conref and href
4518  
4519 attributes be renamed to:
4520  
4521 xtrf
4522  
4523 if the B<conref> or B<href> attribute specification cannot be resolved in the
4524 current corpus by other methods of fixing failing references such as:
4525 L<fixDitaRefs>, L<fixRelocatedRefs> or L<fixXrefsByTitle>.
4526  
4527 This feature designed by L<mailto:mim@cpan.org>.
4528  
4529 =head2 Deguidize
4530  
4531 Some content management systems use guids, some content management systems use
4532 file names as their means of identifying content. When moving from a guid to a
4533 file name content management system it might be necessary to replace the guids
4534 representing file names with the actual underlying file names. If the
4535  
4536 deguidize => 1
4537  
4538 parameter is set to true, Xref will replace any such file guids with the
4539 underlying file name if it is present in the content being cross referenced.
4540  
4541 =head2 File flattening
4542  
4543 It is often desirable to flatten or reflatten the topic files in a corpus so
4544 that they can coexist in a single folder of a content management system without
4545 colliding with each other.
4546  
4547 The presence of the input attribute:
4548  
4549 flattenFolder => folder-to-flatten-files-into
4550  
4551 causes topic files to be flattened into the named folder using the
4552 L<GBStandard> to generate the flattened file names. Xref will then update all
4553 L<Dita> references to match these new file names. If the L<flattenFolder>
4554 folder is the same as the L<inputFolder> then the input files are flattened in
4555 place.
4556  
4557 =head2 Locating relocated files
4558  
4559 File references in B<conref> or B<hrefs> that have a unique valid base file
4560 name and an invalid path can be fixed by setting the input attribute:
4561  
4562 fixRelocatedRefs => 1
4563  
4564 to a true value to request that Xref should replace the incorrect paths to the
4565 unique bases file names with the correct path.
4566  
4567 If coded in conjunction with the B<fixBadRefs> input attribute this will cause
4568 Xref to first try and fix any missing xrefs, any that still fail to resolve
4569 will then be ameliorated by moving them to the B<xtrf> attribute.
4570  
4571 =head2 Fix Xrefs by Title
4572  
4573 L<Dita> B<xref> tags with broken or missing B<href> attributes can sometimes be
4574 fixed by matching the text content of the B<xref> with the titles of topics.
4575  
4576 If:
4577  
4578 fixXrefsByTitle => 1
4579  
4580 is specified, L<Xref> will locate possible targets for a broken B<href> by
4581 matching the white space normalized L<Data::Table::Text::nws> of the text
4582 content of the B<xref> with the similarly normalized title of each topic that
4583 is referenced by any book map that refers to the topic containing the B<xref>.
4584  
4585 If a single matching candidate is located then it will be used to update the
4586 B<href> attribute of the B<xref>.
4587  
4588 =head2 Fix References in Dita To Dita Conversions
4589  
4590 When converting a L<Dita> input source corpus to L<Dita> the referenced topics
4591 are usually renamed and flattened via the L<GBStandard>. If enabled:
4592  
4593 fixDitaRefs => targets/
4594  
4595 updates valid L<Dita> references in the input corpus with the latest name for
4596 the referenced topic to make links that were valid in the input corpus valid in
4597 the output corpus as well.
4598  
4599 The B<targets/> folder should contain the same set of file names as the
4600 original input corpus, each such file should contain the name of a B<bookmap>
4601 topic present in the B<inputFolder=> whose B<chapter> and B<topicref>s identify
4602 the new names of the files cut out and flattened from the existing input
4603 corpus.
4604  
4605 The creation of the B<target/> folder is usually done by some other piece of
4606 software such as L<Data::Edit::Xml::To::Dita> as it is too complex and
4607 laborious to be performed reliably by hand. No validation of the contents of
4608 this folder is performed as it is assumed that it has been created reliably in
4609 software.
4610  
4611 =head2 Topic Matching
4612  
4613 Topics can be matched on title and vocabulary to assist authors in finding
4614 similar topics by specifying the:
4615  
4616 matchTopics => 0.9
4617  
4618 attribute where the value of this attribute is the confidence level between 0
4619 and 1.
4620  
4621 Topic matching produces the reports:
4622  
4623 reports/lists/similar/byTitle.txt
4624 reports/lists/similar/byVocabulary.txt
4625  
4626 Topic matching might take some time for large input folders.
4627  
4628 =head3 Title matching
4629  
4630 This report can be found at:
4631  
4632 reports/lists/similar/byTitle.txt
4633  
4634 Title sorts topics by their titles so that topic with similar titles can be
4635 easily located:
4636  
4637 Similar Prefix Source
4638 1 14 c_Notices__ c_Notices_5614e96c7a3eaf3dfefc4a455398361b
4639 2 c_Notices__ c_Notices_14a9f467215dea879d417de884c21e6d
4640 3 c_Notices__ c_Notices_19011759a2f768d76581dc3bba170a44
4641 4 c_Notices__ c_Notices_aa741e6223e6cf8bc1a5ebdcf0ba867c
4642 5 c_Notices__ c_Notices_f0009b28c3c273094efded5fac32b83f
4643 6 c_Notices__ c_Notices_b1480ac1af812da3945239271c579bb1
4644 7 c_Notices__ c_Notices_5f3aa15d024f0b6068bd8072d4942f6d
4645 8 c_Notices__ c_Notices_17c1f39e8d70c765e1fbb6c495bedb03
4646 9 c_Notices__ c_Notices_7ea35477554f979b3045feb369b69359
4647 10 c_Notices__ c_Notices_4f200259663703065d247b35d5500e0e
4648 11 c_Notices__ c_Notices_e3f2eb03c23491c5e96b08424322e423
4649 12 c_Notices__ c_Notices_06b7e9b0329740fc2b50fedfecbc5a94
4650 13 c_Notices__ c_Notices_550a0d84dfc94982343f58f84d1c11c2
4651 14 c_Notices__ c_Notices_fa7e563d8153668db9ed098d0fe6357b
4652 15 3 c_Overview__ c_Overview_f9e554ee9be499368841260344815f58
4653 16 c_Overview__ c_Overview_f234dc10ea3f4229d0e1ab4ad5e8f5fe
4654 17 c_Overview__ c_Overview_96121d7bcd41cf8be318b96da0049e73
4655  
4656 =head3 Vocabulary matching
4657  
4658 This report can be found at:
4659  
4660 reports/lists/similar/byVocabulary.txt
4661  
4662 Vocabulary matching compares the vocabulary of pairs of topics: topics with
4663 similar vocabularies within the confidence level specified are reported
4664 together:
4665  
4666 Similar Topic
4667 1 8 in/1.dita
4668 2 in/2.dita
4669 3 in/3.dita
4670 4 in/4.dita
4671 5 in/5.dita
4672 6 in/6.dita
4673 7 in/7.dita
4674 8 in/8.dita
4675 9
4676 10 2 in/map/bookmap.ditamap
4677 11 in/map/bookmap2.ditamap
4678 12
4679 13 2 in/act4. dita
4680 14 in/act5.dita
4681  
4682 =head2 Url checking
4683  
4684 Xref will check urls by fetching their headers with L<curl> if the
4685  
4686 validateUrls=>1
4687  
4688 is specified. A list of failing L<url>s will be written to:
4689  
4690 reports/bad/urls.txt
4691  
4692 while a corresponding list of passing L<url>s will be written to
4693  
4694 reports/good/urls.txt
4695  
4696 =head1 Description
4697  
4698 Cross reference Dita XML, match topics and ameliorate missing references.
4699  
4700  
4701 Version 20200202.
4702  
4703  
4704 The following sections describe the methods in each functional area of this
4705 module. For an alphabetic listing of all methods by name see L<Index|/Index>.
4706  
4707  
4708  
4709 =head1 Cross reference
4710  
4711 Check the cross references in a set of Dita files and report the results.
4712  
4713 =head2 xref(%attributes)
4714  
4715 Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>.
4716  
4717 Parameter Description
4718 1 %attributes Cross referencer attribute value pairs
4719  
4720 B<Example:>
4721  
4722  
4723 lll "Test 011";
4724 clearFolder(tests, 111);
4725 createSampleInputFilesForFixDitaRefsImproved3(tests);
4726  
4727 my $y = 𝘅𝗿𝗲𝗳(inputFolder => out, reports => reportFolder); # Check results without fixes
4728 ok $y->statusLine eq q(Xref: 1 ref);
4729  
4730 my $x = 𝘅𝗿𝗲𝗳
4731 (inputFolder => out,
4732 reports => reportFolder,
4733 fixBadRefs => 1,
4734 fixDitaRefs => targets,
4735 fixedFolder => outFixed);
4736  
4737 ok !$x->errors;
4738  
4739  
4740 =head1 Create test data
4741  
4742 Create files to test the various capabilities provided by Xref
4743  
4744  
4745 =head2 Data::Edit::Xml::Xref Definition
4746  
4747  
4748 Attributes used by the Xref cross referencer.
4749  
4750  
4751  
4752  
4753 =head3 Input fields
4754  
4755  
4756 B<addNavTitles> - If true, add navtitle to outgoing bookmap references to show the title of the target topic.
4757  
4758 B<changeBadXrefToPh> - Change xrefs being placed in B<M3> by L<fixBadRefs> to B<ph>.
4759  
4760 B<classificationMaps> - Create classification maps if true
4761  
4762 B<deguidize> - Set true to replace guids in dita references with file name. Given reference B<g1#g2/id> convert B<g1> to a file name by locating the topic with topicId B<g2>. This requires the guids to be genuinely unique. SDL guids are thought to be unique by language code but the same topic, translated to a different language might well have the same guid as the original topic with a different language code: =(de|en|es|fr). If the source is in just one language then the guid uniqueness is a reasonable assumption. If the conversion can be done in phases by language then the uniqueness of guids is again reasonably assured. L<Data::Edit::Xml::Lint> provides an alternative solution to deguidizing by using labels to record the dita reference in the input corpus for each id encountered, these references can then be resolved in the usual manner by L<Data::Edit::Xml::Lint::relint>.
4763  
4764 B<deleteUnusedIds> - Delete ids (except on topics) that are not referenced in any reference in the corpus regardless of the file component of any such reference.
4765  
4766 B<fixBadRefs> - Fix any remaining bad references after any all allowed attempts have been made to fix failing references by moving the failing reference to the B<xtrf> attribute i.e. placing it in B<M3> possibly renaming the tag to B<ph> if L<changeBadXrefToPh> is in effect as well.
4767  
4768 B<fixDitaRefs> - Fix references in a corpus of L<Dita|http://docs.oasis-open.org/dita/dita/v1.3/os/part2-tech-content/dita-v1.3-os-part2-tech-content.html> documents that have been converted to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard> and whose target structure has been written to the named folder.
4769  
4770 B<fixRelocatedRefs> - Fix references to topics that have been moved around in the out folder structure assuming that all file names are unique which they will be if they have been renamed to the GB Standard.
4771  
4772 B<fixXrefsByTitle> - Try to fix invalid xrefs by the Gearhart Title Method enhanced by the Monroe map method if true
4773  
4774 B<fixedFolder> - Fixed files are placed in this folder.
4775  
4776 B<fixedFolderTemp> - Fixed files are placed in this folder if we are on aws but nit the session leader - this folder is then copied back to L<fixedFolder> on the session leader.
4777  
4778 B<flattenFolder> - Files are renamed to the Gearhart standard and placed in this folder if set. References to the unflattened files are updated to references to the flattened files. This option will eventually be deprecated as the Dita::GB::Standard is now fully available allowing files to be easily flattened before being processed by Xref.
4779  
4780 B<getFileUrl> - A url to retrieve a specified file from the server running xref used in generating html reports. The complete url is obtained by appending the fully qualified file name to this value.
4781  
4782 B<html> - Generate html version of reports in this folder if supplied
4783  
4784 B<indexWords> - Index words to topics and topics to words if true.
4785  
4786 B<indexWordsFolder> - Folder into which to save words to topic and topics to word indexes if L<indexWords> is true.
4787  
4788 B<inputFolder> - A folder containing the dita and ditamap files to be cross referenced.
4789  
4790 B<matchTopics> - Match topics by title and by vocabulary to the specified confidence level between 0 and 1. This operation might take some time to complete on a large corpus.
4791  
4792 B<maxZoomIn> - Optional hash of names to regular expressions to look for in each file
4793  
4794 B<maximumNumberOfProcesses> - Maximum number of processes to run in parallel at any one time with a sensible default.
4795  
4796 B<oxygenProjects> - Create oxygen project files for each map - the project file will have an extension of .xpr and the same name and path as the map file or the name return by your implementation of: Data::Edit::Xml::Xref::xprName($map) if present.
4797  
4798 B<reports> - Reports folder: Xref will write text versions of the generated reports to files in this folder.
4799  
4800 B<requestAttributeNameAndValueCounts> - Report attribute name and value counts
4801  
4802 B<subjectSchemeMap> - Create a subject scheme map in the named file
4803  
4804 B<suppressReferenceChecks> - Suppress reference checking - which normally happens by default - but which takes time and might be irrelevant if an earlier xref has already checked all the references.
4805  
4806 B<validateUrls> - Validate urls if true by fetching their headers with L<curl|https://linux.die.net/man/1/curl>
4807  
4808  
4809  
4810 =head3 Output fields
4811  
4812  
4813 B<allowUniquePartialMatches> - Allow unique partial matches - i.e ignore the stuff to the right of the # in a reference if doing so produces a unique result. This feature has been explicitly disabled for conrefs (PS2-561) and might need to be disabled for other types of reference as well.
4814  
4815 B<attributeCount> - {file}{attribute name} == count of the different xml attributes found in the xml files.
4816  
4817 B<attributeNamesAndValuesCount> - {file}{attribute name}{value} = count
4818  
4819 B<author> - {file} = author of this file.
4820  
4821 B<badGuidHrefs> - Bad conrefs - all.
4822  
4823 B<badNavTitles> - Details of nav titles that were not resolved
4824  
4825 B<badReferencesCount> - The number of bad references at the start of the run - however depending on what options were chosen Xref might ameliorate these bad references and thereby reduce this count.
4826  
4827 B<badTables> - Array of tables that need fixing.
4828  
4829 B<badXml1> - [Files] with a bad xml encoding header on the first line.
4830  
4831 B<badXml2> - [Files] with a bad xml doc type on the second line.
4832  
4833 B<baseFiles> - {base of file name}{full file name}++ Current location of the file via uniqueness guaranteed by the GB standard
4834  
4835 B<baseTag> - Base Tag for each file
4836  
4837 B<bookMapRefs> - {bookmap full file name}{href}{navTitle}++ References from bookmaps to topics via appendix, chapter, bookmapref.
4838  
4839 B<conRefs> - {file}{href}{tag}++ : conref source detail
4840  
4841 B<createReports1> - Reports requested before references fixed
4842  
4843 B<createReports2> - Reports requested after references fixed
4844  
4845 B<currentFolder> - The current working folder used to make absolute file names from relative ones
4846  
4847 B<docType> - {file} == docType: the docType for each xml file.
4848  
4849 B<duplicateIds> - [file, id] Duplicate id definitions within each file.
4850  
4851 B<duplicateTopicIds> - [topicId, [files]] Files with duplicate topic ids - the id on the outermost tag.
4852  
4853 B<emptyTopics> - {file} : topics where the *body is empty.
4854  
4855 B<errors> - Number of significant errors as reported in L<statusLine> or 0 if no such errors found
4856  
4857 B<exteriorMaps> - {exterior map} : maps that are not referenced by another map
4858  
4859 B<fileExtensions> - Default file extensions to load
4860  
4861 B<fixRefs> - {file}{ref} where the href or conref target is not valid.
4862  
4863 B<fixedRefsBad> - [] hrefs and conrefs from L<fixRefs|/fixRefs> which were moved to the "xtrf" attribute as requested by the L<fixBadHrefs|/fixBadHrefs> attribute because the reference was invalid and could not be improved by L<deguidization|/deguidize>.
4864  
4865 B<fixedRefsGB> - [] files fixed to the Gearhart-Brenan file naming standard
4866  
4867 B<fixedRefsGood> - [] hrefs and conrefs from L<fixRefs|/fixRefs> which were invalid but have been fixed by L<deguidizing|/deguidize> them to a valid file name.
4868  
4869 B<fixedRefsNoAction> - [] hrefs and conrefs from L<fixRefs|/fixRefs> for which no action was taken.
4870  
4871 B<flattenFiles> - {old full file name} = file renamed to Gearhart-Brenan file naming standard
4872  
4873 B<goodImageFiles> - {file}++ : number of references to each good image
4874  
4875 B<goodNavTitles> - Details of nav titles that were resolved.
4876  
4877 B<guidHrefs> - {file}{href} = location where href starts with GUID- and is thus probably a guid.
4878  
4879 B<guidToFile> - {topic id which is a guid} = file defining topic id.
4880  
4881 B<hrefUrlEncoding> - Hrefs that need url encoding because they contain white space.
4882  
4883 B<idNotReferenced> - {file}{id}++ - id in a file that is not referenced
4884  
4885 B<idReferencedCount> - {file}{id}++ - the number of times this id in this file is referenced from the rest of the corpus
4886  
4887 B<idTags> - {file}{id}[tag] The tags associated with each id in a file - there might be more than one if the id is duplicated
4888  
4889 B<ids> - {file}{id} - id definitions across all files.
4890  
4891 B<idsRemoved> - {id}++ : Ids removed from all files
4892  
4893 B<images> - {file}{href} Count of image references in each file.
4894  
4895 B<imagesReferencedFromBookMaps> - {bookmap full file name}{full name of image referenced from topic referenced from bookmap}++
4896  
4897 B<imagesReferencedFromTopics> - {topic full file name}{full name of image referenced from topic}++
4898  
4899 B<imagesToRefferingBookMaps> - {image full file name}{bookmap full file name}++ : images to referring bookmaps
4900  
4901 B<indexedWords> - {word}{full file name of topic the words occurs in}.
4902  
4903 B<inputFileToTargetTopics> - {input file}{target file}++ : Tells us the topics an input file was split into
4904  
4905 B<inputFiles> - Input files from L<inputFolder|/inputFolder>.
4906  
4907 B<inputFolderImages> - {full image file name} for all files in input folder thus including any images resent
4908  
4909 B<ltgt> - {text between &lt; and &gt}{filename} = count giving the count of text items found between &lt; and &gt;
4910  
4911 B<maxZoomOut> - Results from L<maxZoomIn|/maxZoomIn> where {file name}{regular expression key name in L<maxZoomIn|/maxZoomIn>}++
4912  
4913 B<md5Sum> - MD5 sum for each input file.
4914  
4915 B<md5SumDuplicates> - {md5sum}{file}++ : md5 sums with more than one file
4916  
4917 B<missingImageFiles> - [file, href] == Missing images in each file.
4918  
4919 B<missingTopicIds> - Missing topic ids.
4920  
4921 B<noHref> - Tags that should have an href but do not have one.
4922  
4923 B<notReferenced> - {file name} Files in input area that are not referenced by a conref, image, bookmapref or xref tag and are not a bookmap.
4924  
4925 B<olBody> - The number of ol under body by file
4926  
4927 B<originalSourceFileAndIdToNewFile> - {original file}{id} = new file: Record mapping from original source file and id to the new file containing the id
4928  
4929 B<otherMeta> - {original file}{othermeta name}{othermeta content}++ : the contents of the other meta tags
4930  
4931 B<otherMetaBookMapsAfterTopicIncludes> - Bookmap othermeta after topic othermeta has been included
4932  
4933 B<otherMetaBookMapsBeforeTopicIncludes> - Bookmap othermeta before topic othermeta has been included
4934  
4935 B<otherMetaConsolidated> - {Name}{Content}++ : consolidated other meta data across entire corpus
4936  
4937 B<otherMetaDuplicatesCombined> - Duplicate othermeta in bookmaps with called topics othermeta included
4938  
4939 B<otherMetaDuplicatesSeparately> - Duplicate othermeta in bookmaps and topics considered separately
4940  
4941 B<otherMetaPushToBookMap> - Othermeta that can be pushed to the calling book map
4942  
4943 B<otherMetaRemainWithTopic> - Othermeta that must stay in the topic
4944  
4945 B<parseFailed> - {file} files that failed to parse.
4946  
4947 B<publicId> - {file} = Public id on Doctype
4948  
4949 B<references> - {file}{reference}++ - the various references encountered
4950  
4951 B<relocatedReferencesFailed> - Failing references that were not fixed by relocation
4952  
4953 B<relocatedReferencesFixed> - Relocated references fixed
4954  
4955 B<requiredCleanUp> - {full file name}{cleanup} = number of required-cleanups
4956  
4957 B<results> - Summary of results table.
4958  
4959 B<sourceTopicToTargetBookMap> - {input topic cut into multiple pieces} = output bookmap representing pieces
4960  
4961 B<statusLine> - Status line summarizing the cross reference.
4962  
4963 B<statusTable> - Status table summarizing the cross reference.
4964  
4965 B<tableDimensions> - {file}{columns}{rows} == count
4966  
4967 B<tagCount> - {file}{tags} == count of the different tag names found in the xml files.
4968  
4969 B<tags> - Number of tags encountered
4970  
4971 B<tagsTextsRatio> - Ratio of tags to text encountered
4972  
4973 B<targetFolderContent> - {file} = bookmap file name : the target folder content which shows us where an input file went
4974  
4975 B<targetTopicToInputFiles> - {current file} = the source file from which the current file was obtained
4976  
4977 B<texts> - Number of texts encountered
4978  
4979 B<timeEnded> - Time the run ended
4980  
4981 B<timeStart> - Time the run started
4982  
4983 B<title> - {full file name} = title of file.
4984  
4985 B<titleToFile> - {title}{file}++ if L<fixXrefsByTitle> is in effect
4986  
4987 B<topicFlattening> - {topic}{sources}++ : the source files for each topic that was flattened
4988  
4989 B<topicFlatteningFactor> - Topic flattening factor - higher is better
4990  
4991 B<topicIds> - {file} = topic id - the id on the outermost tag.
4992  
4993 B<topicsFlattened> - Number of topics flattened
4994  
4995 B<topicsNotReferencedFromBookMaps> - {topic file not referenced from any bookmap} = 1
4996  
4997 B<topicsReferencedFromBookMaps> - {bookmap full file name}{topic full file name}++ : bookmaps to topics
4998  
4999 B<topicsToReferringBookMaps> - {topic full file name}{bookmap full file name}++ : topics to referring bookmaps
5000  
5001 B<urls> - {topic full file name}{url}++ : urls found in each file
5002  
5003 B<urlsBad> - {url}{topic full file name}++ : failing urls found in each file
5004  
5005 B<urlsGood> - {url}{topic full file name}++ : passing urls found in each file
5006  
5007 B<validationErrors> - True means that Lint detected errors in the xml contained in the file.
5008  
5009 B<vocabulary> - The text of each topic shorn of attributes for vocabulary comparison.
5010  
5011 B<xRefs> - {file}{href}++ Xrefs references.
5012  
5013 B<xrefBadFormat> - External xrefs with no format=html.
5014  
5015 B<xrefBadScope> - External xrefs with no scope=external.
5016  
5017 B<xrefsFixedByTitle> - Xrefs fixed by locating a matching topic title from their text content.
5018  
5019  
5020  
5021 =head1 Private Methods
5022  
5023 =head2 newXref(%attributes)
5024  
5025 Create a new cross referencer
5026  
5027 Parameter Description
5028 1 %attributes Attributes
5029  
5030 =head2 xref2(%attributes)
5031  
5032 Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>
5033  
5034 Parameter Description
5035 1 %attributes Attributes of cross referencer
5036  
5037 =head2 createReportsInParallel($xref, @reports)
5038  
5039 Create reports in parallel
5040  
5041 Parameter Description
5042 1 $xref Cross referencer
5043 2 @reports Reports to be run
5044  
5045 =head2 createReportsInParallel1()
5046  
5047 Create reports in parallel that do not require fixed references
5048  
5049  
5050 =head2 createReportsInParallel2()
5051  
5052 Create reports in parallel that require fixed references
5053  
5054  
5055 =head2 countLevels($l, $h)
5056  
5057 Count has elements to the specified number of levels
5058  
5059 Parameter Description
5060 1 $l Levels
5061 2 $h Hash
5062  
5063 =head2 externalReference($reference)
5064  
5065 Check for an external reference
5066  
5067 Parameter Description
5068 1 $reference Reference to check
5069  
5070 =head2 fixingRun($xref)
5071  
5072 A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source. Returns a useful message describing this state of affairs.
5073  
5074 Parameter Description
5075 1 $xref Cross referencer
5076  
5077 =head2 loadInputFiles($xref)
5078  
5079 Load the names of the files to be processed
5080  
5081 Parameter Description
5082 1 $xref Cross referencer
5083  
5084 =head2 formatTables($xref, $data, %options)
5085  
5086 Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>.
5087  
5088 Parameter Description
5089 1 $xref Cross referencer
5090 2 $data Table to be formatted
5091 3 %options Options
5092  
5093 =head2 hashOfCountsToArray($hash)
5094  
5095 Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables>
5096  
5097 Parameter Description
5098 1 $hash Hash to be converted
5099  
5100 =head2 reportGuidsToFiles($xref)
5101  
5102 Map and report guids to files
5103  
5104 Parameter Description
5105 1 $xref Xref results
5106  
5107 =head2 editXml($in, $out, $source)
5108  
5109 Edit an xml file retaining any existing XML headers and lint trailers
5110  
5111 Parameter Description
5112 1 $in Input file
5113 2 $out Output file
5114 3 $source Source to write
5115  
5116 =head2 fixReferencesInOneFile($xref, $sourceFile)
5117  
5118 Fix one file by moving unresolved references to the xtrf attribute
5119  
5120 Parameter Description
5121 1 $xref Xref results
5122 2 $sourceFile Source file to fix
5123  
5124 =head2 fixReferencesParallel($xref, $file)
5125  
5126 Fix the references in one file
5127  
5128 Parameter Description
5129 1 $xref Cross referencer
5130 2 $file File to fix
5131  
5132 =head2 fixReferencesResults($xref, @results)
5133  
5134 Consolidate the results of fixing references.
5135  
5136 Parameter Description
5137 1 $xref Cross referencer
5138 2 @results Results from fixReferencesInParallel
5139  
5140 =head2 fixReferences($xref)
5141  
5142 Fix just the file containing references using a number of techniques and report those references that cannot be so fixed.
5143  
5144 Parameter Description
5145 1 $xref Xref results
5146  
5147 =head2 fixOneFileGB($xref, $file)
5148  
5149 Fix one file to the Gearhart-Brenan standard
5150  
5151 Parameter Description
5152 1 $xref Xref results
5153 2 $file File to fix
5154  
5155 =head2 fixFilesGB($xref)
5156  
5157 Rename files to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard>
5158  
5159 Parameter Description
5160 1 $xref Xref results
5161  
5162 =head2 analyzeOneFileParallel($Xref, $iFile)
5163  
5164 Analyze one input file
5165  
5166 Parameter Description
5167 1 $Xref Xref request
5168 2 $iFile File to analyze
5169  
5170 =head2 analyzeOneFileResults($xref, @x)
5171  
5172 Merge a list of cross reference results into the first cross referencer in the list
5173  
5174 Parameter Description
5175 1 $xref Cross referencer to merge into
5176 2 @x Other cross referencers
5177  
5178 =head2 analyzeInputFiles($xref)
5179  
5180 Analyze the input files
5181  
5182 Parameter Description
5183 1 $xref Cross referencer
5184  
5185 =head2 reportIdRefs($xref)
5186  
5187 Report the number of times each id is referenced
5188  
5189 Parameter Description
5190 1 $xref Cross referencer
5191  
5192 =head2 removeUnusedIds($xref)
5193  
5194 Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference. This is a very conservative approach which acknowledges that writers might be looking for an id if they mention it in a reference.
5195  
5196 Parameter Description
5197 1 $xref Cross referencer
5198  
5199 =head2 reportEmptyTopics($xref)
5200  
5201 Report empty topics
5202  
5203 Parameter Description
5204 1 $xref Cross referencer
5205  
5206 =head2 reportDuplicateIds($xref)
5207  
5208 Report duplicate ids
5209  
5210 Parameter Description
5211 1 $xref Cross referencer
5212  
5213 =head2 reportDuplicateTopicIds($xref)
5214  
5215 Report duplicate topic ids
5216  
5217 Parameter Description
5218 1 $xref Cross referencer
5219  
5220 =head2 reportNoHrefs($xref)
5221  
5222 Report locations where an href was expected but not found
5223  
5224 Parameter Description
5225 1 $xref Cross referencer
5226  
5227 =head2 checkReferences($xref)
5228  
5229 Check each reference, report bad references and mark them for fixing.
5230  
5231 Parameter Description
5232 1 $xref Cross referencer
5233  
5234 =head2 reportGuidHrefs($xref)
5235  
5236 Report on guid hrefs
5237  
5238 Parameter Description
5239 1 $xref Cross referencer
5240  
5241 =head2 reportImages($xref)
5242  
5243 Reports on images and references to images
5244  
5245 Parameter Description
5246 1 $xref Cross referencer
5247  
5248 =head2 reportParseFailed($xref)
5249  
5250 Report failed parses
5251  
5252 Parameter Description
5253 1 $xref Cross referencer
5254  
5255 =head2 reportXml1($xref)
5256  
5257 Report bad xml on line 1
5258  
5259 Parameter Description
5260 1 $xref Cross referencer
5261  
5262 =head2 reportXml2($xref)
5263  
5264 Report bad xml on line 2
5265  
5266 Parameter Description
5267 1 $xref Cross referencer
5268  
5269 =head2 reportDocTypeCount($xref)
5270  
5271 Report doc type count
5272  
5273 Parameter Description
5274 1 $xref Cross referencer
5275  
5276 =head2 reportTagCount($xref)
5277  
5278 Report tag counts
5279  
5280 Parameter Description
5281 1 $xref Cross referencer
5282  
5283 =head2 reportTagsAndTextsCount($xref)
5284  
5285 Report tags and texts counts
5286  
5287 Parameter Description
5288 1 $xref Cross referencer
5289  
5290 =head2 reportLtGt($xref)
5291  
5292 Report items found between &lt; and &gt;
5293  
5294 Parameter Description
5295 1 $xref Cross referencer
5296  
5297 =head2 reportAttributeCount($xref)
5298  
5299 Report attribute counts
5300  
5301 Parameter Description
5302 1 $xref Cross referencer
5303  
5304 =head2 reportAttributeNameAndValueCounts($xref)
5305  
5306 Report attribute value counts
5307  
5308 Parameter Description
5309 1 $xref Cross referencer
5310  
5311 =head2 reportValidationErrors($xref)
5312  
5313 Report the files known to have validation errors
5314  
5315 Parameter Description
5316 1 $xref Cross referencer
5317  
5318 =head2 reportTables($xref)
5319  
5320 Report on tables that have problems
5321  
5322 Parameter Description
5323 1 $xref Cross referencer
5324  
5325 =head2 reportFileExtensionCount($xref)
5326  
5327 Report file extension counts
5328  
5329 Parameter Description
5330 1 $xref Cross referencer
5331  
5332 =head2 reportFileTypes($xref)
5333  
5334 Report file type counts - takes too long in series
5335  
5336 Parameter Description
5337 1 $xref Cross referencer
5338  
5339 =head2 reportExternalXrefs($xref)
5340  
5341 Report external xrefs missing other attributes
5342  
5343 Parameter Description
5344 1 $xref Cross referencer
5345  
5346 =head2 reportMaxZoomOut($xref)
5347  
5348 Text located via Max Zoom In
5349  
5350 Parameter Description
5351 1 $xref Cross referencer
5352  
5353 =head2 reportTopicDetails($xref)
5354  
5355 Things that occur once in each file
5356  
5357 Parameter Description
5358 1 $xref Cross referencer
5359  
5360 =head2 reportTopicReuse($xref)
5361  
5362 Count how frequently each topic is reused
5363  
5364 Parameter Description
5365 1 $xref Cross referencer
5366  
5367 =head2 reportFixRefs($xref)
5368  
5369 Report of hrefs that need to be fixed
5370  
5371 Parameter Description
5372 1 $xref Cross referencer
5373  
5374 =head2 reportSourceFiles($xref)
5375  
5376 Source file for each topic
5377  
5378 Parameter Description
5379 1 $xref Cross referencer
5380  
5381 =head2 reportReferencesFromBookMaps($xref)
5382  
5383 Topics and images referenced from bookmaps
5384  
5385 Parameter Description
5386 1 $xref Cross referencer
5387  
5388 =head2 reportExteriorMaps($xref)
5389  
5390 Maps that are not referenced by any other map
5391  
5392 Parameter Description
5393 1 $xref Cross referencer
5394  
5395 =head2 reportTopicsNotReferencedFromBookMaps($xref)
5396  
5397 Topics not referenced from bookmaps
5398  
5399 Parameter Description
5400 1 $xref Cross referencer
5401  
5402 =head2 reportTableDimensions($xref)
5403  
5404 Report table dimensions
5405  
5406 Parameter Description
5407 1 $xref Cross referencer
5408  
5409 =head2 reportOtherMeta($xref)
5410  
5411 Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map.
5412  
5413 Parameter Description
5414 1 $xref Cross referencer
5415  
5416 =head2 createSubjectSchemeMap($xref)
5417  
5418 Create a subject scheme map from othermeta
5419  
5420 Parameter Description
5421 1 $xref Cross referencer
5422  
5423 =head2 writeClassificationHtml($xref, $classification)
5424  
5425 Write classification tree as html
5426  
5427 Parameter Description
5428 1 $xref Cross referencer
5429 2 $classification {title=>{subject=>{file=>++}}}
5430  
5431 =head2 createClassificationMap($xref, $bookMap, $classification)
5432  
5433 Create a classification map for each bookmap
5434  
5435 Parameter Description
5436 1 $xref Cross referencer
5437 2 $bookMap Bookmap to classify
5438 3 $classification Classification scheme
5439  
5440 =head2 createClassificationMaps($xref)
5441  
5442 Create classification maps for each bookmap
5443  
5444 Parameter Description
5445 1 $xref Cross referencer
5446  
5447 =head2 reportSimilarTopicsByTitle($xref)
5448  
5449 Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names
5450  
5451 Parameter Description
5452 1 $xref Cross referencer
5453  
5454 =head2 reportSimilarTopicsByVocabulary($xref)
5455  
5456 Report topics likely to be similar on the basis of their vocabulary
5457  
5458 Parameter Description
5459 1 $xref Cross referencer
5460  
5461 =head2 reportWordsByFile($xref)
5462  
5463 Index words to the files they occur in
5464  
5465 Parameter Description
5466 1 $xref Cross referencer
5467  
5468 =head2 reportMd5Sum($xref)
5469  
5470 Report files with identical md5 sums
5471  
5472 Parameter Description
5473 1 $xref Cross referencer
5474  
5475 =head2 reportOlBody($xref)
5476  
5477 ol under body - indicative of a task
5478  
5479 Parameter Description
5480 1 $xref Cross referencer
5481  
5482 =head2 reportHrefUrlEncoding($xref)
5483  
5484 href needs url encoding
5485  
5486 Parameter Description
5487 1 $xref Cross referencer
5488  
5489 =head2 reportConRefMatching($xref)
5490  
5491 Report conref matching
5492  
5493 Parameter Description
5494 1 $xref Cross referencer
5495  
5496 =head2 reportPublicIds($xref)
5497  
5498 Report public ids in use
5499  
5500 Parameter Description
5501 1 $xref Cross referencer
5502  
5503 =head2 reportRequiredCleanUps($xref)
5504  
5505 Report required clean ups
5506  
5507 Parameter Description
5508 1 $xref Cross referencer
5509  
5510 =head2 reportUrls($xref)
5511  
5512 Report urls that fail to resolve
5513  
5514 Parameter Description
5515 1 $xref Cross referencer
5516  
5517 =head2 addNavTitlesToOneMap($xref, $file)
5518  
5519 Fix navtitles in one map
5520  
5521 Parameter Description
5522 1 $xref Xref results
5523 2 $file File to fix
5524  
5525 =head2 addNavTitlesToMaps($xref)
5526  
5527 Add nav titles to files containing maps.
5528  
5529 Parameter Description
5530 1 $xref Xref results
5531  
5532 =head2 oxygenProjectFileMetaData()
5533  
5534 Meta data for the oxygen project files
5535  
5536  
5537 =head2 createOxygenProjectFile($xref, $bm, $xprName)
5538  
5539 Create an Oxygen project file for the specified bookmap
5540  
5541 Parameter Description
5542 1 $xref Xref
5543 2 $bm Bookmap
5544 3 $xprName Xpr name from bookmap
5545  
5546 =head2 createOxygenProjectMapFiles($xref)
5547  
5548 Create Oxygen project files from Xref results
5549  
5550 Parameter Description
5551 1 $xref Cross referencer
5552  
5553 =head2 oneBadRef($xref, $file, $href)
5554  
5555 Check one reference and return the first error encountered or B<undef> if no errors encountered. Relies on L<topicIds> to test files present and test the B<topicId> is valid, relies on L<ids> to check that the referenced B<id> is valid.
5556  
5557 Parameter Description
5558 1 $xref Cross referencer
5559 2 $file File containing reference
5560 3 $href Reference
5561  
5562 =head2 createSampleInputFilesBaseCase($in, $N)
5563  
5564 Create sample input files for testing. The attribute B<inputFolder> supplies the name of the folder in which to create the sample files.
5565  
5566 Parameter Description
5567 1 $in Input folder
5568 2 $N Number of sample files
5569  
5570 =head2 createSampleInputFilesFixFolder($in)
5571  
5572 Create sample input files for testing fixFolder
5573  
5574 Parameter Description
5575 1 $in Folder to create the files in
5576  
5577 =head2 createSampleInputFilesLtGt($in)
5578  
5579 Create sample input files for testing items between &lt; and &gt;
5580  
5581 Parameter Description
5582 1 $in Folder to create the files in
5583  
5584 =head2 createSampleInputFilesForFixDitaRefs($in, $targets)
5585  
5586 Create sample input files for fixing renamed topic refs
5587  
5588 Parameter Description
5589 1 $in Folder to create the files in
5590 2 $targets Targets folder
5591  
5592 =head2 createSampleInputFilesForFixDitaRefsXref($in)
5593  
5594 Create sample input files for fixing references into renamed topics by xref
5595  
5596 Parameter Description
5597 1 $in Folder to create the files in
5598  
5599 =head2 createSampleConRefs($in)
5600  
5601 Create sample input files for fixing a conref
5602  
5603 Parameter Description
5604 1 $in Folder to create the files in
5605  
5606 =head2 createSampleConRefMatching($in)
5607  
5608 Create sample input files for matching conref source and targets
5609  
5610 Parameter Description
5611 1 $in Folder to create the files in
5612  
5613 =head2 createSampleDuplicateMd5Sum($in)
5614  
5615 Create sample input files with duplicate md5 sums
5616  
5617 Parameter Description
5618 1 $in Folder to create the files in
5619  
5620 =head2 createSampleUnreferencedIds($in)
5621  
5622 Create sample input files with unreferenced ids
5623  
5624 Parameter Description
5625 1 $in Folder to create the files in
5626  
5627 =head2 createEmptyBody($in)
5628  
5629 Create sample input files for empty body detection
5630  
5631 Parameter Description
5632 1 $in Folder to create the files in
5633  
5634 =head2 createClassificationMapsTest($in)
5635  
5636 Create sample input files for a classification map
5637  
5638 Parameter Description
5639 1 $in Folder to create the files in
5640  
5641 =head2 createWordsToFilesTest($in)
5642  
5643 Index words to file
5644  
5645 Parameter Description
5646 1 $in Folder to create the files in
5647  
5648 =head2 createUrlTests($in)
5649  
5650 Check urls
5651  
5652 Parameter Description
5653 1 $in Folder to create the files in
5654  
5655 =head2 changeFolderAndWriteFiles($f, $D)
5656  
5657 Change file structure to the current folder and write
5658  
5659 Parameter Description
5660 1 $f Data structure as a string
5661 2 $D Target folder
5662  
5663 =head2 createSampleInputFilesForFixDitaRefsImproved1($folder)
5664  
5665 Create sample input files for fixing references via the targets/ folder
5666  
5667 Parameter Description
5668 1 $folder Folder to switch to
5669  
5670 =head2 createSampleInputFilesForFixDitaRefsImproved2($folder)
5671  
5672 Create sample input files for fixing conref references via the targets/ folder
5673  
5674 Parameter Description
5675 1 $folder Folder to switch to
5676  
5677 =head2 createSampleInputFilesForFixDitaRefsImproved3($folder)
5678  
5679 Create sample input files for fixing bookmap references to topics that get cut into multiple pieces
5680  
5681 Parameter Description
5682 1 $folder Folder to switch to
5683  
5684 =head2 createSampleInputFilesForFixDitaRefsImproved4($folder)
5685  
5686 Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces
5687  
5688 Parameter Description
5689 1 $folder Folder to switch to
5690  
5691 =head2 createSampleImageTest($folder)
5692  
5693 Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces
5694  
5695 Parameter Description
5696 1 $folder Folder to switch to
5697  
5698 =head2 createTestTopicFlattening($folder)
5699  
5700 Create sample input files for testing topic flattening ratio reporting
5701  
5702 Parameter Description
5703 1 $folder Folder to switch to
5704  
5705 =head2 createTestReferencedToFlattenedTopic($folder)
5706  
5707 Full reference to a topic that has been flattened
5708  
5709 Parameter Description
5710 1 $folder Folder to switch to
5711  
5712 =head2 createTestReferenceToCutOutTopic($folder)
5713  
5714 References from a topic that has been cut out to a topic that has been cut out
5715  
5716 Parameter Description
5717 1 $folder Folder to switch to
5718  
5719 =head2 createSampleOtherMeta($out)
5720  
5721 Create sample data for othermeta reports
5722  
5723 Parameter Description
5724 1 $out Folder
5725  
5726 =head2 createTestOneNotRef($folder)
5727  
5728 One topic refernced and the other not
5729  
5730 Parameter Description
5731 1 $folder Folder to switch to
5732  
5733 =head2 createSampleTopicsReferencedFromBookMaps($in)
5734  
5735 The number of times a topic is referenced from a bookmap
5736  
5737 Parameter Description
5738 1 $in Folder to create the files in
5739  
5740 =head2 createSampleImageReferences($in)
5741  
5742 Good and bad image references
5743  
5744 Parameter Description
5745 1 $in Folder to create the files in
5746  
5747 =head2 createRequiredCleanUps($in)
5748  
5749 Required clean ups report
5750  
5751 Parameter Description
5752 1 $in Folder to create the files in
5753  
5754 =head2 createSoftConrefs($in)
5755  
5756 Fix file part of conref even if the rest is invalid
5757  
5758 Parameter Description
5759 1 $in Folder to create the files in
5760  
5761 =head2 checkXrefStructure($x, $field, @folders)
5762  
5763 Check an output structure produced by Xrf
5764  
5765 Parameter Description
5766 1 $x Cross references
5767 2 $field Field to check
5768 3 @folders Folders to suppress
5769  
5770 =head2 writeXrefStructure($x, $field, @folders)
5771  
5772 Write the test for an Xref structure
5773  
5774 Parameter Description
5775 1 $x Cross referencer
5776 2 $field Field
5777 3 @folders Names of the folders to suppress
5778  
5779 =head2 deleteVariableFields($x)
5780  
5781 Remove time and other fields that do not affect the end results
5782  
5783 Parameter Description
5784 1 $x Cross referencer
5785  
5786 =head2 testReferenceChecking()
5787  
5788 Test reference checking
5789  
5790  
5791  
5792 =head1 Index
5793  
5794  
5795 1 L<addNavTitlesToMaps|/addNavTitlesToMaps> - Add nav titles to files containing maps.
5796  
5797 2 L<addNavTitlesToOneMap|/addNavTitlesToOneMap> - Fix navtitles in one map
5798  
5799 3 L<analyzeInputFiles|/analyzeInputFiles> - Analyze the input files
5800  
5801 4 L<analyzeOneFileParallel|/analyzeOneFileParallel> - Analyze one input file
5802  
5803 5 L<analyzeOneFileResults|/analyzeOneFileResults> - Merge a list of cross reference results into the first cross referencer in the list
5804  
5805 6 L<changeFolderAndWriteFiles|/changeFolderAndWriteFiles> - Change file structure to the current folder and write
5806  
5807 7 L<checkReferences|/checkReferences> - Check each reference, report bad references and mark them for fixing.
5808  
5809 8 L<checkXrefStructure|/checkXrefStructure> - Check an output structure produced by Xrf
5810  
5811 9 L<countLevels|/countLevels> - Count has elements to the specified number of levels
5812  
5813 10 L<createClassificationMap|/createClassificationMap> - Create a classification map for each bookmap
5814  
5815 11 L<createClassificationMaps|/createClassificationMaps> - Create classification maps for each bookmap
5816  
5817 12 L<createClassificationMapsTest|/createClassificationMapsTest> - Create sample input files for a classification map
5818  
5819 13 L<createEmptyBody|/createEmptyBody> - Create sample input files for empty body detection
5820  
5821 14 L<createOxygenProjectFile|/createOxygenProjectFile> - Create an Oxygen project file for the specified bookmap
5822  
5823 15 L<createOxygenProjectMapFiles|/createOxygenProjectMapFiles> - Create Oxygen project files from Xref results
5824  
5825 16 L<createReportsInParallel|/createReportsInParallel> - Create reports in parallel
5826  
5827 17 L<createReportsInParallel1|/createReportsInParallel1> - Create reports in parallel that do not require fixed references
5828  
5829 18 L<createReportsInParallel2|/createReportsInParallel2> - Create reports in parallel that require fixed references
5830  
5831 19 L<createRequiredCleanUps|/createRequiredCleanUps> - Required clean ups report
5832  
5833 20 L<createSampleConRefMatching|/createSampleConRefMatching> - Create sample input files for matching conref source and targets
5834  
5835 21 L<createSampleConRefs|/createSampleConRefs> - Create sample input files for fixing a conref
5836  
5837 22 L<createSampleDuplicateMd5Sum|/createSampleDuplicateMd5Sum> - Create sample input files with duplicate md5 sums
5838  
5839 23 L<createSampleImageReferences|/createSampleImageReferences> - Good and bad image references
5840  
5841 24 L<createSampleImageTest|/createSampleImageTest> - Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces
5842  
5843 25 L<createSampleInputFilesBaseCase|/createSampleInputFilesBaseCase> - Create sample input files for testing.
5844  
5845 26 L<createSampleInputFilesFixFolder|/createSampleInputFilesFixFolder> - Create sample input files for testing fixFolder
5846  
5847 27 L<createSampleInputFilesForFixDitaRefs|/createSampleInputFilesForFixDitaRefs> - Create sample input files for fixing renamed topic refs
5848  
5849 28 L<createSampleInputFilesForFixDitaRefsImproved1|/createSampleInputFilesForFixDitaRefsImproved1> - Create sample input files for fixing references via the targets/ folder
5850  
5851 29 L<createSampleInputFilesForFixDitaRefsImproved2|/createSampleInputFilesForFixDitaRefsImproved2> - Create sample input files for fixing conref references via the targets/ folder
5852  
5853 30 L<createSampleInputFilesForFixDitaRefsImproved3|/createSampleInputFilesForFixDitaRefsImproved3> - Create sample input files for fixing bookmap references to topics that get cut into multiple pieces
5854  
5855 31 L<createSampleInputFilesForFixDitaRefsImproved4|/createSampleInputFilesForFixDitaRefsImproved4> - Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces
5856  
5857 32 L<createSampleInputFilesForFixDitaRefsXref|/createSampleInputFilesForFixDitaRefsXref> - Create sample input files for fixing references into renamed topics by xref
5858  
5859 33 L<createSampleInputFilesLtGt|/createSampleInputFilesLtGt> - Create sample input files for testing items between &lt; and &gt;
5860  
5861 34 L<createSampleOtherMeta|/createSampleOtherMeta> - Create sample data for othermeta reports
5862  
5863 35 L<createSampleTopicsReferencedFromBookMaps|/createSampleTopicsReferencedFromBookMaps> - The number of times a topic is referenced from a bookmap
5864  
5865 36 L<createSampleUnreferencedIds|/createSampleUnreferencedIds> - Create sample input files with unreferenced ids
5866  
5867 37 L<createSoftConrefs|/createSoftConrefs> - Fix file part of conref even if the rest is invalid
5868  
5869 38 L<createSubjectSchemeMap|/createSubjectSchemeMap> - Create a subject scheme map from othermeta
5870  
5871 39 L<createTestOneNotRef|/createTestOneNotRef> - One topic refernced and the other not
5872  
5873 40 L<createTestReferencedToFlattenedTopic|/createTestReferencedToFlattenedTopic> - Full reference to a topic that has been flattened
5874  
5875 41 L<createTestReferenceToCutOutTopic|/createTestReferenceToCutOutTopic> - References from a topic that has been cut out to a topic that has been cut out
5876  
5877 42 L<createTestTopicFlattening|/createTestTopicFlattening> - Create sample input files for testing topic flattening ratio reporting
5878  
5879 43 L<createUrlTests|/createUrlTests> - Check urls
5880  
5881 44 L<createWordsToFilesTest|/createWordsToFilesTest> - Index words to file
5882  
5883 45 L<deleteVariableFields|/deleteVariableFields> - Remove time and other fields that do not affect the end results
5884  
5885 46 L<editXml|/editXml> - Edit an xml file retaining any existing XML headers and lint trailers
5886  
5887 47 L<externalReference|/externalReference> - Check for an external reference
5888  
5889 48 L<fixFilesGB|/fixFilesGB> - Rename files to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard>
5890  
5891 49 L<fixingRun|/fixingRun> - A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source.
5892  
5893 50 L<fixOneFileGB|/fixOneFileGB> - Fix one file to the Gearhart-Brenan standard
5894  
5895 51 L<fixReferences|/fixReferences> - Fix just the file containing references using a number of techniques and report those references that cannot be so fixed.
5896  
5897 52 L<fixReferencesInOneFile|/fixReferencesInOneFile> - Fix one file by moving unresolved references to the xtrf attribute
5898  
5899 53 L<fixReferencesParallel|/fixReferencesParallel> - Fix the references in one file
5900  
5901 54 L<fixReferencesResults|/fixReferencesResults> - Consolidate the results of fixing references.
5902  
5903 55 L<formatTables|/formatTables> - Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>.
5904  
5905 56 L<hashOfCountsToArray|/hashOfCountsToArray> - Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables>
5906  
5907 57 L<loadInputFiles|/loadInputFiles> - Load the names of the files to be processed
5908  
5909 58 L<newXref|/newXref> - Create a new cross referencer
5910  
5911 59 L<oneBadRef|/oneBadRef> - Check one reference and return the first error encountered or B<undef> if no errors encountered.
5912  
5913 60 L<oxygenProjectFileMetaData|/oxygenProjectFileMetaData> - Meta data for the oxygen project files
5914  
5915 61 L<removeUnusedIds|/removeUnusedIds> - Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference.
5916  
5917 62 L<reportAttributeCount|/reportAttributeCount> - Report attribute counts
5918  
5919 63 L<reportAttributeNameAndValueCounts|/reportAttributeNameAndValueCounts> - Report attribute value counts
5920  
5921 64 L<reportConRefMatching|/reportConRefMatching> - Report conref matching
5922  
5923 65 L<reportDocTypeCount|/reportDocTypeCount> - Report doc type count
5924  
5925 66 L<reportDuplicateIds|/reportDuplicateIds> - Report duplicate ids
5926  
5927 67 L<reportDuplicateTopicIds|/reportDuplicateTopicIds> - Report duplicate topic ids
5928  
5929 68 L<reportEmptyTopics|/reportEmptyTopics> - Report empty topics
5930  
5931 69 L<reportExteriorMaps|/reportExteriorMaps> - Maps that are not referenced by any other map
5932  
5933 70 L<reportExternalXrefs|/reportExternalXrefs> - Report external xrefs missing other attributes
5934  
5935 71 L<reportFileExtensionCount|/reportFileExtensionCount> - Report file extension counts
5936  
5937 72 L<reportFileTypes|/reportFileTypes> - Report file type counts - takes too long in series
5938  
5939 73 L<reportFixRefs|/reportFixRefs> - Report of hrefs that need to be fixed
5940  
5941 74 L<reportGuidHrefs|/reportGuidHrefs> - Report on guid hrefs
5942  
5943 75 L<reportGuidsToFiles|/reportGuidsToFiles> - Map and report guids to files
5944  
5945 76 L<reportHrefUrlEncoding|/reportHrefUrlEncoding> - href needs url encoding
5946  
5947 77 L<reportIdRefs|/reportIdRefs> - Report the number of times each id is referenced
5948  
5949 78 L<reportImages|/reportImages> - Reports on images and references to images
5950  
5951 79 L<reportLtGt|/reportLtGt> - Report items found between &lt; and &gt;
5952  
5953 80 L<reportMaxZoomOut|/reportMaxZoomOut> - Text located via Max Zoom In
5954  
5955 81 L<reportMd5Sum|/reportMd5Sum> - Report files with identical md5 sums
5956  
5957 82 L<reportNoHrefs|/reportNoHrefs> - Report locations where an href was expected but not found
5958  
5959 83 L<reportOlBody|/reportOlBody> - ol under body - indicative of a task
5960  
5961 84 L<reportOtherMeta|/reportOtherMeta> - Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map.
5962  
5963 85 L<reportParseFailed|/reportParseFailed> - Report failed parses
5964  
5965 86 L<reportPublicIds|/reportPublicIds> - Report public ids in use
5966  
5967 87 L<reportReferencesFromBookMaps|/reportReferencesFromBookMaps> - Topics and images referenced from bookmaps
5968  
5969 88 L<reportRequiredCleanUps|/reportRequiredCleanUps> - Report required clean ups
5970  
5971 89 L<reportSimilarTopicsByTitle|/reportSimilarTopicsByTitle> - Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names
5972  
5973 90 L<reportSimilarTopicsByVocabulary|/reportSimilarTopicsByVocabulary> - Report topics likely to be similar on the basis of their vocabulary
5974  
5975 91 L<reportSourceFiles|/reportSourceFiles> - Source file for each topic
5976  
5977 92 L<reportTableDimensions|/reportTableDimensions> - Report table dimensions
5978  
5979 93 L<reportTables|/reportTables> - Report on tables that have problems
5980  
5981 94 L<reportTagCount|/reportTagCount> - Report tag counts
5982  
5983 95 L<reportTagsAndTextsCount|/reportTagsAndTextsCount> - Report tags and texts counts
5984  
5985 96 L<reportTopicDetails|/reportTopicDetails> - Things that occur once in each file
5986  
5987 97 L<reportTopicReuse|/reportTopicReuse> - Count how frequently each topic is reused
5988  
5989 98 L<reportTopicsNotReferencedFromBookMaps|/reportTopicsNotReferencedFromBookMaps> - Topics not referenced from bookmaps
5990  
5991 99 L<reportUrls|/reportUrls> - Report urls that fail to resolve
5992  
5993 100 L<reportValidationErrors|/reportValidationErrors> - Report the files known to have validation errors
5994  
5995 101 L<reportWordsByFile|/reportWordsByFile> - Index words to the files they occur in
5996  
5997 102 L<reportXml1|/reportXml1> - Report bad xml on line 1
5998  
5999 103 L<reportXml2|/reportXml2> - Report bad xml on line 2
6000  
6001 104 L<testReferenceChecking|/testReferenceChecking> - Test reference checking
6002  
6003 105 L<writeClassificationHtml|/writeClassificationHtml> - Write classification tree as html
6004  
6005 106 L<writeXrefStructure|/writeXrefStructure> - Write the test for an Xref structure
6006  
6007 107 L<xref|/xref> - Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder.
6008  
6009 108 L<xref2|/xref2> - Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder.
6010  
6011 =head1 Installation
6012  
6013 This module is written in 100% Pure Perl and, thus, it is easy to read,
6014 comprehend, use, modify and install via B<cpan>:
6015  
6016 sudo cpan install Data::Edit::Xml::Xref
6017  
6018 =head1 Author
6019  
6020 L<philiprbrenan@gmail.com|mailto:philiprbrenan@gmail.com>
6021  
6022 L<http://www.appaapps.com|http://www.appaapps.com>
6023  
6024 =head1 Copyright
6025  
6026 Copyright (c) 2016-2019 Philip R Brenan.
6027  
6028 This module is free software. It may be used, redistributed and/or modified
6029 under the same terms as Perl itself.
6030  
6031 =cut
6032  
6033  
6034  
6035 # Tests and documentation
6036  
6037 sub test
6038 {my $p = __PACKAGE__;
6039 binmode($_, ":utf8") for *STDOUT, *STDERR;
6040 return if eval "eof(${p}::DATA)";
6041 my $s = eval "join('', <${p}::DATA>)";
6042 $@ and die $@;
6043 eval $s;
6044 $@ and die $@;
6045 1
6046 }
6047  
6048 test unless caller;
6049  
6050 1;
6051 # podDocumentation
6052 __DATA__
↑ Summary