| 1 | #!/usr/bin/perl -I/home/phil/perl/cpan/DataEditXml/lib/ -I/home/phil/perl/cpan/DataTableText/lib/ -I/home/phil/perl/cpan/DitaGBStandard/lib/ | ||
| 2 | #------------------------------------------------------------------------------- | ||
| 3 | # Cross reference Dita XML, match topics and ameliorate missing references. | ||
| 4 | # Philip R Brenan at gmail dot com, Appa Apps Ltd Inc, 2016-2019 | ||
| 5 | # Improvements and maxzoomin | ||
| 6 | #------------------------------------------------------------------------------- | ||
| 7 | # Check for image formats that will not display in a browser | ||
| 8 | # Do not consider companion files! | ||
| 9 | # Images that are referenced by topics which are not referenced by bookmaps showup as referenced | ||
| 10 | # It should be possible to remove reportImages by using generic references instead | ||
| 11 | # Conref processing in reportReferencesFromBookmaps | ||
| 12 | # Fix xref external/scope and eliminate error count if fixbadrefs in operation. | ||
| 13 | # Add labels to ditaRefs processing so that references to labels are also fixed | ||
| 14 | # Add xref expansion from id in file as it is a pain to code up the full details by hand | ||
| 15 | # Find topics that have no text in them per: PS2-617 | ||
| 16 | # Need test for changeBadXrefToPh | ||
| 17 | # Unique target needs tests | ||
| 18 | # Create list of images found in input folder | ||
| 19 | # Conrefs report should use targets/ to update the conref file so conrefs fixed by fixDitaRefs are considered | ||
| 20 | |||
| 21 | package Data::Edit::Xml::Xref; | ||
| 22 | our $VERSION = 20200424; | ||
| 23 | 1 | ▶ | use v5.26; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 24 | 1 | ▶ | use warnings FATAL => qw(all); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 26 | 1 | ▶ | use Carp qw(confess cluck); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 27 | 1 | ▶ | use Data::Dump qw(dump); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 28 | 0 | ▶ | use Data::Edit::Xml; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 29 | use Data::Table::Text qw(:all); | ||
| 30 | use Dita::GB::Standard; | ||
| 31 | use Storable qw(store retrieve); | ||
| 32 | use Time::HiRes qw(time); | ||
| 33 | use utf8; | ||
| 34 | |||
| 35 | #sub improvementLength {80} #P Maximum length of the test of an improvement suggestion | ||
| 36 | sub classificationMapSuffix{q(_classification.ditamap)} #P Suffix to add to map files to create corresponding classification map file | ||
| 37 | |||
| 38 | #D1 Cross reference # Check the cross references in a set of Dita files and report the results. | ||
| 39 | |||
| 40 | sub newXref(%) #P Create a new cross referencer | ||
| 41 | {my (%attributes) = @_; # Attributes | ||
| 42 | |||
| 43 | my $xref = genHash(__PACKAGE__, # Attributes used by the Xref cross referencer. | ||
| 44 | addNavTitles => undef, #I If true, add navtitle to outgoing bookmap references to show the title of the target topic. | ||
| 45 | allowUniquePartialMatches => undef, # Allow unique partial matches - i.e ignore the stuff to the right of the # in a reference if doing so produces a unique result. This feature has been explicitly disabled for conrefs (PS2-561) and might need to be disabled for other types of reference as well. | ||
| 46 | attributeCount => {}, # {file}{attribute name} == count of the different xml attributes found in the xml files. | ||
| 47 | attributeNamesAndValuesCount => {}, # {file}{attribute name}{value} = count | ||
| 48 | author => {}, # {file} = author of this file. | ||
| 49 | badGuidHrefs => {}, # Bad conrefs - all. | ||
| 50 | badNavTitles => {}, # Details of nav titles that were not resolved | ||
| 51 | badReferencesCount => 0, # The number of bad references at the start of the run - however depending on what options were chosen Xref might ameliorate these bad references and thereby reduce this count. | ||
| 52 | badTables => [], # Array of tables that need fixing. | ||
| 53 | badXml1 => {}, # [Files] with a bad xml encoding header on the first line. | ||
| 54 | badXml2 => {}, # [Files] with a bad xml doc type on the second line. | ||
| 55 | baseFiles => {}, # {base of file name}{full file name}++ Current location of the file via uniqueness guaranteed by the GB standard | ||
| 56 | baseTag => {}, # Base Tag for each file | ||
| 57 | bookMapRefs => {}, # {bookmap full file name}{href}{navTitle}++ References from bookmaps to topics via appendix, chapter, bookmapref. | ||
| 58 | changeBadXrefToPh => undef, #I Change xrefs being placed in B<M3> by L<fixBadRefs> to B<ph>. | ||
| 59 | classificationMaps => undef, #I Create classification maps if true | ||
| 60 | conRefs => {}, # {file}{href}{tag}++ : conref source detail | ||
| 61 | createReports1 => [], # Reports requested before references fixed | ||
| 62 | createReports2 => [], # Reports requested after references fixed | ||
| 63 | currentFolder => currentDirectory, # The current working folder used to make absolute file names from relative ones | ||
| 64 | deleteUnusedIds => 0, #I Delete ids (except on topics) that are not referenced in any reference in the corpus regardless of the file component of any such reference. | ||
| 65 | deguidize => undef, #I Set true to replace guids in dita references with file name. Given reference B<g1#g2/id> convert B<g1> to a file name by locating the topic with topicId B<g2>. This requires the guids to be genuinely unique. SDL guids are thought to be unique by language code but the same topic, translated to a different language might well have the same guid as the original topic with a different language code: =(de|en|es|fr). If the source is in just one language then the guid uniqueness is a reasonable assumption. If the conversion can be done in phases by language then the uniqueness of guids is again reasonably assured. L<Data::Edit::Xml::Lint> provides an alternative solution to deguidizing by using labels to record the dita reference in the input corpus for each id encountered, these references can then be resolved in the usual manner by L<Data::Edit::Xml::Lint::relint>. | ||
| 66 | docType => {}, # {file} == docType: the docType for each xml file. | ||
| 67 | duplicateIds => {}, # [file, id] Duplicate id definitions within each file. | ||
| 68 | duplicateTopicIds => {}, # Duplicate topic ids | ||
| 69 | duplicateTopicIds => {}, # [topicId, [files]] Files with duplicate topic ids - the id on the outermost tag. | ||
| 70 | emptyTopics => {}, # {file} : topics where the *body is empty. | ||
| 71 | errors => 0, # Number of significant errors as reported in L<statusLine> or 0 if no such errors found | ||
| 72 | exteriorMaps => {}, # {exterior map} : maps that are not referenced by another map | ||
| 73 | fileExtensions => [qw(.dita .ditamap .xml .fodt)], # Default file extensions to load | ||
| 74 | fixBadRefs => undef, #I Fix any remaining bad references after any all allowed attempts have been made to fix failing references by moving the failing reference to the B<xtrf> attribute i.e. placing it in B<M3> possibly renaming the tag to B<ph> if L<changeBadXrefToPh> is in effect as well. | ||
| 75 | fixDitaRefs => undef, #I Fix references in a corpus of L<Dita> documents that have been converted to the L<GBStandard> and whose target structure has been written to the named folder. | ||
| 76 | fixedFolder => undef, #I Fixed files are placed in this folder. | ||
| 77 | fixedFolderTemp => undef, #I Fixed files are placed in this folder if we are on aws but nit the session leader - this folder is then copied back to L<fixedFolder> on the session leader. | ||
| 78 | fixedRefsBad => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> which were moved to the "xtrf" attribute as requested by the L<fixBadHrefs|/fixBadHrefs> attribute because the reference was invalid and could not be improved by L<deguidization|/deguidize>. | ||
| 79 | fixedRefsGB => [], # [] files fixed to the Gearhart-Brenan file naming standard | ||
| 80 | fixedRefsGood => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> which were invalid but have been fixed by L<deguidizing|/deguidize> them to a valid file name. | ||
| 81 | fixedRefsNoAction => [], # [] hrefs and conrefs from L<fixRefs|/fixRefs> for which no action was taken. | ||
| 82 | fixRefs => {}, # {file}{ref} where the href or conref target is not valid. | ||
| 83 | fixRelocatedRefs => undef, #I Fix references to topics that have been moved around in the out folder structure assuming that all file names are unique which they will be if they have been renamed to the GB Standard. | ||
| 84 | fixXrefsByTitle => undef, #I Try to fix invalid xrefs by the Gearhart Title Method enhanced by the Monroe map method if true | ||
| 85 | flattenFiles => {}, # {old full file name} = file renamed to Gearhart-Brenan file naming standard | ||
| 86 | flattenFolder => undef, #I Files are renamed to the Gearhart standard and placed in this folder if set. References to the unflattened files are updated to references to the flattened files. This option will eventually be deprecated as the Dita::GB::Standard is now fully available allowing files to be easily flattened before being processed by Xref. | ||
| 87 | getFileUrl => qq(/cgi-bin/uiSelfServiceXref/client.pl?getFile=), #I A url to retrieve a specified file from the server running xref used in generating html reports. The complete url is obtained by appending the fully qualified file name to this value. | ||
| 88 | goodImageFiles => {}, # {file}++ : number of references to each good image | ||
| 89 | goodNavTitles => {}, # Details of nav titles that were resolved. | ||
| 90 | guidHrefs => {}, # {file}{href} = location where href starts with GUID- and is thus probably a guid. | ||
| 91 | guidToFile => {}, # {topic id which is a guid} = file defining topic id. | ||
| 92 | hrefUrlEncoding => {}, # Hrefs that need url encoding because they contain white space. | ||
| 93 | html => undef, #I Generate html version of reports in this folder if supplied | ||
| 94 | idNotReferenced => {}, # {file}{id}++ - id in a file that is not referenced | ||
| 95 | idReferencedCount => {}, # {file}{id}++ - the number of times this id in this file is referenced from the rest of the corpus | ||
| 96 | ids => {}, # {file}{id} - id definitions across all files. | ||
| 97 | idsRemoved => {}, # {id}++ : Ids removed from all files | ||
| 98 | idTags => {}, # {file}{id}[tag] The tags associated with each id in a file - there might be more than one if the id is duplicated | ||
| 99 | images => {}, # {file}{href} Count of image references in each file. | ||
| 100 | imagesReferencedFromBookMaps => {}, # {bookmap full file name}{full name of image referenced from topic referenced from bookmap}++ | ||
| 101 | imagesReferencedFromTopics => {}, # {topic full file name}{full name of image referenced from topic}++ | ||
| 102 | imagesToRefferingBookMaps => {}, # {image full file name}{bookmap full file name}++ : images to referring bookmaps | ||
| 103 | indexWords => undef, #I Index words to topics and topics to words if true. | ||
| 104 | indexWordsFolder => undef, #I Folder into which to save words to topic and topics to word indexes if L<indexWords> is true. | ||
| 105 | indexedWords => {}, # {word}{full file name of topic the words occurs in}. | ||
| 106 | inputFiles => [], # Input files from L<inputFolder|/inputFolder>. | ||
| 107 | inputFileToTargetTopics => {}, # {input file}{target file}++ : Tells us the topics an input file was split into | ||
| 108 | inputFolderImages => {}, # {full image file name} for all files in input folder thus including any images resent | ||
| 109 | inputFolder => undef, #I A folder containing the dita and ditamap files to be cross referenced. | ||
| 110 | ltgt => {}, # {text between < and >}{filename} = count giving the count of text items found between < and > | ||
| 111 | matchTopics => undef, #I Match topics by title and by vocabulary to the specified confidence level between 0 and 1. This operation might take some time to complete on a large corpus. | ||
| 112 | maximumNumberOfProcesses => numberOfCpus(8), #I Maximum number of processes to run in parallel at any one time with a sensible default. | ||
| 113 | maxZoomIn => undef, #I Optional hash of names to regular expressions to look for in each file | ||
| 114 | maxZoomOut => {}, # Results from L<maxZoomIn|/maxZoomIn> where {file name}{regular expression key name in L<maxZoomIn|/maxZoomIn>}++ | ||
| 115 | md5Sum => {}, # MD5 sum for each input file. | ||
| 116 | md5SumDuplicates => {}, # {md5sum}{file}++ : md5 sums with more than one file | ||
| 117 | missingImageFiles => {}, # [file, href] == Missing images in each file. | ||
| 118 | missingTopicIds => {}, # Missing topic ids. | ||
| 119 | noHref => {}, # Tags that should have an href but do not have one. | ||
| 120 | notReferenced => {}, # {file name} Files in input area that are not referenced by a conref, image, bookmapref or xref tag and are not a bookmap. | ||
| 121 | olBody => {}, # The number of ol under body by file | ||
| 122 | originalSourceFileAndIdToNewFile => {}, # {original file}{id} = new file: Record mapping from original source file and id to the new file containing the id | ||
| 123 | otherMeta => {}, # {original file}{othermeta name}{othermeta content}++ : the contents of the other meta tags | ||
| 124 | otherMetaDuplicatesSeparately => [], # Duplicate othermeta in bookmaps and topics considered separately | ||
| 125 | otherMetaDuplicatesCombined => [], # Duplicate othermeta in bookmaps with called topics othermeta included | ||
| 126 | otherMetaRemainWithTopic => [], # Othermeta that must stay in the topic | ||
| 127 | otherMetaPushToBookMap => [], # Othermeta that can be pushed to the calling book map | ||
| 128 | otherMetaBookMapsBeforeTopicIncludes=> [], # Bookmap othermeta before topic othermeta has been included | ||
| 129 | otherMetaBookMapsAfterTopicIncludes => [], # Bookmap othermeta after topic othermeta has been included | ||
| 130 | otherMetaConsolidated => {}, # {Name}{Content}++ : consolidated other meta data across entire corpus | ||
| 131 | oxygenProjects => undef, #I Create oxygen project files for each map - the project file will have an extension of .xpr and the same name and path as the map file or the name return by your implementation of: Data::Edit::Xml::Xref::xprName($map) if present. | ||
| 132 | parseFailed => {}, # {file} files that failed to parse. | ||
| 133 | publicId => {}, # {file} = Public id on Doctype | ||
| 134 | references => {}, # {file}{reference}++ - the various references encountered | ||
| 135 | relocatedReferencesFailed => [], # Failing references that were not fixed by relocation | ||
| 136 | relocatedReferencesFixed => [], # Relocated references fixed | ||
| 137 | requestAttributeNameAndValueCounts => undef, #I Report attribute name and value counts | ||
| 138 | requiredCleanUp => undef, # {full file name}{cleanup} = number of required-cleanups | ||
| 139 | reports => undef, #I Reports folder: Xref will write text versions of the generated reports to files in this folder. | ||
| 140 | results => [], # Summary of results table. | ||
| 141 | # sourceFile => undef, # The source file from whic#h this structure was generated. | ||
| 142 | sourceTopicToTargetBookMap => {}, # {input topic cut into multiple pieces} = output bookmap representing pieces | ||
| 143 | statusLine => undef, # Status line summarizing the cross reference. | ||
| 144 | statusTable => undef, # Status table summarizing the cross reference. | ||
| 145 | subjectSchemeMap => undef, #I Create a subject scheme map in the named file | ||
| 146 | suppressReferenceChecks => undef, #I Suppress reference checking - which normally happens by default - but which takes time and might be irrelevant if an earlier xref has already checked all the references. | ||
| 147 | tableDimensions => {}, # {file}{columns}{rows} == count | ||
| 148 | tagCount => {}, # {file}{tags} == count of the different tag names found in the xml files. | ||
| 149 | tagsTextsRatio => undef, # Ratio of tags to text encountered | ||
| 150 | tags => undef, # Number of tags encountered | ||
| 151 | targetFolderContent => {}, # {file} = bookmap file name : the target folder content which shows us where an input file went | ||
| 152 | targetTopicToInputFiles => {}, # {current file} = the source file from which the current file was obtained | ||
| 153 | texts => undef, # Number of texts encountered | ||
| 154 | timeEnded => undef, # Time the run ended | ||
| 155 | timeStart => undef, # Time the run started | ||
| 156 | title => {}, # {full file name} = title of file. | ||
| 157 | titleToFile => {}, # {title}{file}++ if L<fixXrefsByTitle> is in effect | ||
| 158 | topicFlatteningFactor => {}, # Topic flattening factor - higher is better | ||
| 159 | topicFlattening => {}, # {topic}{sources}++ : the source files for each topic that was flattened | ||
| 160 | topicIds => {}, # {file} = topic id - the id on the outermost tag. | ||
| 161 | topicsFlattened => undef, # Number of topics flattened | ||
| 162 | topicsNotReferencedFromBookMaps => {}, # {topic file not referenced from any bookmap} = 1 | ||
| 163 | topicsReferencedFromBookMaps => {}, # {bookmap full file name}{topic full file name}++ : bookmaps to topics | ||
| 164 | topicsToReferringBookMaps => {}, # {topic full file name}{bookmap full file name}++ : topics to referring bookmaps | ||
| 165 | urls => {}, # {topic full file name}{url}++ : urls found in each file | ||
| 166 | urlsBad => {}, # {url}{topic full file name}++ : failing urls found in each file | ||
| 167 | urlsGood => {}, # {url}{topic full file name}++ : passing urls found in each file | ||
| 168 | validateUrls => undef, #I Validate urls if true by fetching their headers with L<curl> | ||
| 169 | validationErrors => {}, # True means that Lint detected errors in the xml contained in the file. | ||
| 170 | vocabulary => {}, # The text of each topic shorn of attributes for vocabulary comparison. | ||
| 171 | xrefBadFormat => {}, # External xrefs with no format=html. | ||
| 172 | xrefBadScope => {}, # External xrefs with no scope=external. | ||
| 173 | xRefs => {}, # {file}{href}++ Xrefs references. | ||
| 174 | xrefsFixedByTitle => [], # Xrefs fixed by locating a matching topic title from their text content. | ||
| 175 | ); | ||
| 176 | |||
| 177 | loadHash($xref, @_); # Load attributes complaining about any invalid ones | ||
| 178 | } # newXref | ||
| 179 | |||
| 180 | sub xref2(%) #P Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref> | ||
| 181 | {my (%attributes) = @_; # Attributes of cross referencer | ||
| 182 | my ($xref) = newXref(@_); # Cross referencer | ||
| 183 | $xref->timeStart = time; # Start time | ||
| 184 | |||
| 185 | $xref->inputFolder or confess "Please supply a value for: inputFolder"; | ||
| 186 | $xref->inputFolder =~ s(\/+\Z) (\/)gs; # Cleanup path names | ||
| 187 | $xref->inputFolder = # Make input folder absolute | ||
| 188 | absFromAbsPlusRel($xref->currentFolder, $xref->inputFolder) | ||
| 189 | if $xref->inputFolder !~ m(\A/); | ||
| 190 | |||
| 191 | $xref->reports or confess "Please supply a value for: reports"; | ||
| 192 | |||
| 193 | if (1) # Write title and some of the parameters | ||
| 194 | {my $r = $xref->reports; | ||
| 195 | owf(fpe($r, qw(xref_parameter_settings txt)), dump($xref)) if $r; # Print all parameters | ||
| 196 | |||
| 197 | my $i = $xref->inputFolder; | ||
| 198 | lll "Xref started on folder: $i, reports: $r" # Show that we are starting unless in development | ||
| 199 | unless $i =~ m(/tmp/); | ||
| 200 | } | ||
| 201 | |||
| 202 | if (my $d = $xref->fixDitaRefs) # Fully qualify and validate targets folder | ||
| 203 | {$xref->fixDitaRefs = fullyQualifiedFile($d) ? $d : # Fully qualified target folder name | ||
| 204 | absFromAbsPlusRel($xref->currentFolder, $d); # Get fully qualified target folder if necessary | ||
| 205 | if (!-d $d) # Check targets folder is available | ||
| 206 | {confess "Targets folder does not exist: fixDitaRefs=>$d"; | ||
| 207 | } | ||
| 208 | my @d = searchDirectoryTreesForMatchingFiles($d); | ||
| 209 | @d or confess "Targets folder is empty: fixDitaRefs=>$d"; | ||
| 210 | } | ||
| 211 | |||
| 212 | if (!$xref->fixedFolder and $xref->fixDitaRefs || $xref->fixRelocatedRefs) # Fixing references | ||
| 213 | {my $s = join ' and ', | ||
| 214 | ($xref->fixDitaRefs ? "fixDitaRefs" : (), | ||
| 215 | $xref->fixRelocatedRefs ? "fixRelocatedRefs" : ()); | ||
| 216 | warn "No fixedFolder attribute specified yet $s specified.\n". | ||
| 217 | "Assuming inputFolder for fixedFolder.\n"; | ||
| 218 | $xref->fixedFolder = $xref->inputFolder; | ||
| 219 | } | ||
| 220 | |||
| 221 | if (my $f = $xref->fixedFolder) # Fixing references in a folder other than the input folder requires us to copy the files across so we can make differential changes | ||
| 222 | {if (my $i = $xref->inputFolder) | ||
| 223 | {if ($f ne $i) | ||
| 224 | {copyFolder($i, $f); | ||
| 225 | } | ||
| 226 | } | ||
| 227 | } | ||
| 228 | |||
| 229 | $xref->fixedFolderTemp //= fpd(temporaryFolder); # Preserve this temporary value across session instances | ||
| 230 | |||
| 231 | my @series = # Must be done in series at the start | ||
| 232 | (q(loadInputFiles), | ||
| 233 | q(analyzeInputFiles), | ||
| 234 | q(reportReferencesFromBookMaps), # Used by fixReferences to get bookmap references | ||
| 235 | $xref->deguidize ? q(reportGuidsToFiles) : (), # Used by addNavTitleToMaps | ||
| 236 | q(checkReferences), # Check all the references | ||
| 237 | q(createReportsInParallel1), # Create reports that do not rely on fixed references | ||
| 238 | q(fixReferences), # Fix any failing references | ||
| 239 | q(createReportsInParallel2), # Create reports that rely on fixed references | ||
| 240 | ); | ||
| 241 | |||
| 242 | my @parallel1 = # Create reports in parallel that do not rely on fixed references | ||
| 243 | (q(reportXml1), | ||
| 244 | q(reportXml2), | ||
| 245 | q(reportDuplicateIds), | ||
| 246 | q(reportDuplicateTopicIds), | ||
| 247 | q(reportNoHrefs), | ||
| 248 | q(reportTables), | ||
| 249 | q(reportParseFailed), | ||
| 250 | q(reportAttributeCount), | ||
| 251 | q(reportLtGt), | ||
| 252 | q(reportTagCount), | ||
| 253 | q(reportTagsAndTextsCount), | ||
| 254 | q(reportDocTypeCount), | ||
| 255 | q(reportFileExtensionCount), | ||
| 256 | q(reportFileTypes), | ||
| 257 | q(reportValidationErrors), | ||
| 258 | q(reportGuidHrefs), | ||
| 259 | q(reportExternalXrefs), | ||
| 260 | q(reportTopicDetails), | ||
| 261 | q(reportTopicReuse), | ||
| 262 | q(reportMd5Sum), | ||
| 263 | q(reportOlBody), | ||
| 264 | q(reportHrefUrlEncoding), | ||
| 265 | q(reportFixRefs), | ||
| 266 | q(reportSourceFiles), | ||
| 267 | q(reportOtherMeta), | ||
| 268 | q(createSubjectSchemeMap), | ||
| 269 | q(reportTopicsNotReferencedFromBookMaps), | ||
| 270 | q(reportTableDimensions), | ||
| 271 | q(reportExteriorMaps), | ||
| 272 | q(createClassificationMaps), | ||
| 273 | q(reportIdRefs), | ||
| 274 | q(reportEmptyTopics), | ||
| 275 | q(reportConRefMatching), | ||
| 276 | q(reportPublicIds), | ||
| 277 | q(reportRequiredCleanUps), | ||
| 278 | ); | ||
| 279 | |||
| 280 | if ($xref->addNavTitles) # Add nav titles to bookmaps if requested | ||
| 281 | {push @parallel1, q(addNavTitlesToMaps); | ||
| 282 | } | ||
| 283 | |||
| 284 | if ($xref->requestAttributeNameAndValueCounts) # Report attribute name and value counts | ||
| 285 | {push @parallel1, q(reportAttributeNameAndValueCounts); | ||
| 286 | } | ||
| 287 | |||
| 288 | if ($xref->flattenFolder) # Fix file names to the Gearhart-Brenan file naming standard | ||
| 289 | {push @parallel1, q(fixFilesGB) | ||
| 290 | } | ||
| 291 | |||
| 292 | if ($xref->matchTopics) # Topic matching reports | ||
| 293 | {push @parallel1, q(reportSimilarTopicsByTitle), | ||
| 294 | q(reportSimilarTopicsByVocabulary); | ||
| 295 | } | ||
| 296 | |||
| 297 | if ($xref->indexWords) # Word indexing required | ||
| 298 | {push @parallel1, q(reportWordsByFile); | ||
| 299 | $xref->indexWordsFolder or confess "Please set indexWordsFolder"; | ||
| 300 | } | ||
| 301 | |||
| 302 | if ($xref->validateUrls) # Validate urls | ||
| 303 | {push @parallel1, q(reportUrls); | ||
| 304 | } | ||
| 305 | |||
| 306 | push $xref->createReports1->@*, @parallel1; # Create reports that do not rely on references being fixed | ||
| 307 | |||
| 308 | my @parallel2 = # Create reports in parallel after references fixed | ||
| 309 | (q(removeUnusedIds), # Count and/or remove unused ids | ||
| 310 | q(reportImages), # Images relies on fixed references to locate images | ||
| 311 | ); | ||
| 312 | |||
| 313 | if ($xref->oxygenProjects) # Create oxygen project files if requested | ||
| 314 | {push @parallel2, q(createOxygenProjectMapFiles); | ||
| 315 | } | ||
| 316 | |||
| 317 | push $xref->createReports2->@*, @parallel2; # Create reports that rely on references being fixed | ||
| 318 | |||
| 319 | if (1) # Perform phases in series that must be run in series | ||
| 320 | {my @times; | ||
| 321 | |||
| 322 | for my $phase(@series) # Each phase in series | ||
| 323 | {my $startTime = time; | ||
| 324 | #lll "Xref phase $phase"; | ||
| 325 | |||
| 326 | $xref->$phase; # Execute phase | ||
| 327 | |||
| 328 | #mmm "Xref phase $phase"; | ||
| 329 | push @times, [$phase, sprintf("%12.4f", time - $startTime)]; # Phase time | ||
| 330 | } | ||
| 331 | |||
| 332 | my $delta = sprintf("%.3f seconds", time - $xref->timeStart); # Time so far | ||
| 333 | |||
| 334 | formatTables($xref, [sort {$$b[1] <=> $$a[1]} @times], # Update after each phase so we can see progress on long running jobs | ||
| 335 | columns => <<END, | ||
| 336 | Phase Xref processing phase | ||
| 337 | Time_Seconds Time in seconds taken by this processing phase | ||
| 338 | END | ||
| 339 | title => qq(Processing phases elapsed times in descending order), | ||
| 340 | head => <<END, | ||
| 341 | Xref phases took the following times on DDDD | ||
| 342 | |||
| 343 | Total run time: $delta | ||
| 344 | END | ||
| 345 | file => fpe(q(timing), qw(xref_phases txt))); # Write phase times | ||
| 346 | } | ||
| 347 | |||
| 348 | formattedTablesReport # Needs update for reports in parallel | ||
| 349 | (title=>q(Reports available), | ||
| 350 | head=><<END, | ||
| 351 | NNNN reports available on DDDD | ||
| 352 | |||
| 353 | Sorted by title | ||
| 354 | END | ||
| 355 | file=>fpe($xref->reports, qw(reports txt))) if $xref->reports; | ||
| 356 | |||
| 357 | if (1) # Summarize | ||
| 358 | {my @o; | ||
| 359 | my $save = sub | ||
| 360 | {my ($levels, $field, $plural, $single) = @_; | ||
| 361 | my $n = &countLevels($levels, $xref->{$field}); | ||
| 362 | push @o, [$n, $plural] if $n > 1; | ||
| 363 | push @o, [$n, $single // ($plural =~ s(s\Z) ()gsr)] if $n == 1; | ||
| 364 | }; | ||
| 365 | |||
| 366 | # $save->(1, "badConRefsList", q(conrefs)); | ||
| 367 | # $save->(1, "badConRefs", q(files with bad conrefs), q(file with bad conrefs)); | ||
| 368 | $save->(1, "badGuidHrefs", q(invalid guid hrefs)); | ||
| 369 | $save->(1, "badTables", q(tables)); | ||
| 370 | $save->(1, "badXml1", q(first lines)); | ||
| 371 | $save->(1, "badXml2", q(second lines)); | ||
| 372 | # $save->(1, "badXRefsList", q(xrefs)); | ||
| 373 | # $save->(1, "badXRefs", q(files with bad xrefs), q(file with bad xrefs)); | ||
| 374 | $save->(2, "duplicateIds", q(duplicate ids)); | ||
| 375 | $save->(1, "fixedRefsBad", q(refs)); # Unable to resolve these references - L<fixBadRefs> can be used to ameliorate them. | ||
| 376 | $save->(1, "hrefUrlEncoding", q(href url encoding), q(href url encoding)); | ||
| 377 | $save->(2, "md5SumDuplicates", q(duplicate files)); | ||
| 378 | $save->(1, "missingImageFiles", q(image refs)); | ||
| 379 | $save->(1, "missingTopicIds", q(missing topic ids)); | ||
| 380 | $save->(1, "notReferenced", q(files not referenced), q(file not referenced)); | ||
| 381 | $save->(1, "parseFailed", q(files failed to parse), q(file failed to parse)); | ||
| 382 | $save->(1, "duplicateTopicIds", q(duplicate topic ids)); | ||
| 383 | $save->(2, "noHref", q(hrefs missing), q(href missing)); | ||
| 384 | $save->(2, "validationErrors", q(validation errors)); # Needs testing | ||
| 385 | $save->(2, "xrefBadFormat", q(External xrefs with no format=html)); | ||
| 386 | $save->(2, "xrefBadScope", q(External xrefs with no scope=external)); | ||
| 387 | $save->(1, "urlsBad", q(urls)); | ||
| 388 | |||
| 389 | my $files = $xref->inputFiles->@*; | ||
| 390 | |||
| 391 | my $statusLine = $xref->statusLine = @o ? join " ", # Status line | ||
| 392 | "Xref:", join ", ", | ||
| 393 | map {join " ", @$_} | ||
| 394 | sort | ||
| 395 | {return $$a[1] cmp $$b[1] if $$b[0] == $$a[0]; | ||
| 396 | $$b[0] <=> $$a[0] | ||
| 397 | } | ||
| 398 | @o : qq(Xref: processed $files files, found no errors); | ||
| 399 | |||
| 400 | lll $statusLine; # Print Xref results summary with date and time | ||
| 401 | |||
| 402 | $xref->errors = @o; | ||
| 403 | |||
| 404 | $xref->statusTable = formatTable | ||
| 405 | ([sort {$$b[0] <=> $$a[0]} @o], [qw(Count Condition)]); # Summary in status form | ||
| 406 | $xref->results = \@o; # Save status line components | ||
| 407 | } | ||
| 408 | |||
| 409 | $xref->timeEnded = time; # Run ended time | ||
| 410 | |||
| 411 | formatTables($xref, [[$xref->timeStart, $xref->timeEnded, # Write run times | ||
| 412 | $xref->timeEnded - $xref->timeStart]], | ||
| 413 | columns => <<END, | ||
| 414 | Start_Time Start time of the run | ||
| 415 | End_Time End time of the run | ||
| 416 | Elapsed_Time Xref took this many seconds to run | ||
| 417 | END | ||
| 418 | title => qq(Run times in seconds), | ||
| 419 | head => qq(Xref took the following time to run on DDDD), | ||
| 420 | file => fpe(q(timing), qw(run txt))); | ||
| 421 | |||
| 422 | formatHtmlTablesIndex($xref->reports, q(), $xref->getFileUrl, 2); # Create an index of html files for use as an initial page of Xref results - this is done in Dita.pm as well but we need it here too so it can be tested. | ||
| 423 | |||
| 424 | lll "Xref finished on folder: ", $xref->inputFolder unless # Show that we have finished unless in development | ||
| 425 | $xref->inputFolder =~ m(/tmp/); | ||
| 426 | |||
| 427 | $xref # Return Xref results | ||
| 428 | } # xref2 | ||
| 429 | |||
| 430 | sub xref(%) # Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>. | ||
| 431 | {my (%attributes) = @_; # Cross referencer attribute value pairs | ||
| 432 | my $x = callSubInParallel {xref2(%attributes)}; # Process in alternate process to avoid memory fragmentation | ||
| 433 | newXref %$x # Return blessed results after creating in child process | ||
| 434 | } | ||
| 435 | |||
| 436 | sub createReportsInParallel($@) #P Create reports in parallel | ||
| 437 | {my ($xref, @reports) = @_; # Cross referencer, reports to be run | ||
| 438 | |||
| 439 | runInParallel($xref->maximumNumberOfProcesses, | ||
| 440 | sub # Execute each report in parallel | ||
| 441 | {my ($report) = @_; | ||
| 442 | my ($result) = my @result = ($xref->$report); # Check that the value returned by the report is a single hash reference | ||
| 443 | |||
| 444 | formatHtmlAndTextTablesWaitPids; # Wait for report tables to be formatted in parallel | ||
| 445 | |||
| 446 | if (scalar(@result) != 1) # Check return from multiverse to universe | ||
| 447 | {confess "Phase $report does not return one result"; | ||
| 448 | } | ||
| 449 | if ($result =~ m(hash)s) | ||
| 450 | {confess "Phase $report does not return a hash reference"; | ||
| 451 | } | ||
| 452 | |||
| 453 | $result # Return results from report | ||
| 454 | }, | ||
| 455 | sub # Decode results | ||
| 456 | {for my $r(@_) | ||
| 457 | {for my $k(sort keys %$r) | ||
| 458 | {$xref->{$k} = $$r{$k}; | ||
| 459 | } | ||
| 460 | } | ||
| 461 | }, | ||
| 462 | @reports); # Each report to be run parallel | ||
| 463 | } | ||
| 464 | |||
| 465 | sub createReportsInParallel1() #P Create reports in parallel that do not require fixed references | ||
| 466 | {my ($xref) = @_; # Cross referencer | ||
| 467 | createReportsInParallel($xref, $xref->createReports1->@*) | ||
| 468 | } | ||
| 469 | |||
| 470 | sub createReportsInParallel2() #P Create reports in parallel that require fixed references | ||
| 471 | {my ($xref) = @_; # Cross referencer | ||
| 472 | createReportsInParallel($xref, $xref->createReports2->@*) | ||
| 473 | } | ||
| 474 | |||
| 475 | sub countLevels($$) #P Count has elements to the specified number of levels | ||
| 476 | {my ($l, $h) = @_; # Levels, hash | ||
| 477 | if ($l == 0) | ||
| 478 | {return $h; | ||
| 479 | } | ||
| 480 | if ($l == 1) | ||
| 481 | {return scalar keys @$h if ref($h) =~ m(array)i; | ||
| 482 | return scalar keys %$h if ref($h) =~ m(hash)i; | ||
| 483 | } | ||
| 484 | my $n = 0; | ||
| 485 | if (ref($h) =~ m(hash)i) | ||
| 486 | {$n += &countLevels($l-1, $_) for values %$h; | ||
| 487 | } | ||
| 488 | elsif (ref($h) =~ m(array)i) | ||
| 489 | {$n += &countLevels($l-1, $_) for values @$h; | ||
| 490 | } | ||
| 491 | $n | ||
| 492 | } | ||
| 493 | |||
| 494 | sub externalReference($) #P Check for an external reference | ||
| 495 | {my ($reference) = @_; # Reference to check | ||
| 496 | $reference =~ m(\A(https?:|mailto:|www))is # Check reference | ||
| 497 | } | ||
| 498 | |||
| 499 | sub fixingRun($) #P A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source. Returns a useful message describing this state of affairs. | ||
| 500 | {my ($xref) = @_; # Cross referencer | ||
| 501 | my @i; | ||
| 502 | for my $i(qw(fixBadRefs fixDitaRefs fixRelocatedRefs fixXrefsByTitle)) | ||
| 503 | {push @i, $i if $xref->{$i}; | ||
| 504 | } | ||
| 505 | |||
| 506 | if (@i) # Fixing run | ||
| 507 | {my $i = join ', ', @i; | ||
| 508 | return <<END; | ||
| 509 | |||
| 510 | Caution: this is a fixing run with options: $i set to on. | ||
| 511 | |||
| 512 | This report represents the status quo ante - i.e. what things look like at the | ||
| 513 | start of the run. This is useful information for driving development of Xref | ||
| 514 | but confusing if you only want to improve your document corpus. | ||
| 515 | |||
| 516 | To get the state of play, a posteriori, request a copy of this report as | ||
| 517 | produced after an Xref run with all the options above set to off. | ||
| 518 | |||
| 519 | END | ||
| 520 | } | ||
| 521 | else # Reporting run | ||
| 522 | {return ''; | ||
| 523 | } | ||
| 524 | } | ||
| 525 | |||
| 526 | sub loadInputFiles($) #P Load the names of the files to be processed | ||
| 527 | {my ($xref) = @_; # Cross referencer | ||
| 528 | |||
| 529 | my $in = $xref->inputFiles = | ||
| 530 | [searchDirectoryTreesForMatchingFiles | ||
| 531 | $xref->inputFolder, @{$xref->fileExtensions}]; | ||
| 532 | |||
| 533 | if (!$in or @$in == 0) # Complain if there are no input files to analyze | ||
| 534 | {my $i = $xref->inputFolder; | ||
| 535 | my $e = join " ", @{$xref->fileExtensions}; | ||
| 536 | my $x = -d $i ? "The input folder does exist." : | ||
| 537 | "The input folder does NOT exist!"; | ||
| 538 | confess join '', | ||
| 539 | "No files with the specified file extensions ", | ||
| 540 | "in the specified input folder:\n", | ||
| 541 | "$e\n$i\n$x\n"; | ||
| 542 | } | ||
| 543 | |||
| 544 | my @images = searchDirectoryTreesForMatchingFiles($xref->inputFolder); # Input files | ||
| 545 | $xref->inputFolderImages = {map {fn($_), $_} @images}; # Image file name which works well for images because the md5 sum in their name is probably unique | ||
| 546 | } | ||
| 547 | |||
| 548 | sub formatTables($$%) #P Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>. | ||
| 549 | {my ($xref, $data, %options) = @_; # Cross referencer, table to be formatted, options | ||
| 550 | |||
| 551 | $xref && ref($xref) =~ m(xref)i or cluck "No cross referencer"; # Check parameters | ||
| 552 | $data && ref($data) =~ m(array)i or cluck "No data for table"; | ||
| 553 | |||
| 554 | cluck "No file for table" unless $options{file}; # Check for required options | ||
| 555 | cluck "No columns for table" unless $options{columns}; | ||
| 556 | cluck "No title for table" unless $options{title}; | ||
| 557 | $options{zero} = 1 if $options{facet} and !$options{zero}; | ||
| 558 | |||
| 559 | formatHtmlAndTextTables | ||
| 560 | ($xref->reports, $xref->html, $xref->getFileUrl, | ||
| 561 | $xref->inputFolder, $data, %options); | ||
| 562 | } | ||
| 563 | |||
| 564 | sub hashOfCountsToArray($) #P Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables> | ||
| 565 | {my ($hash) = @_; # Hash to be converted | ||
| 566 | my $array = []; | ||
| 567 | for my $key(sort keys %$hash) | ||
| 568 | {push @$array, [$$hash{$key}, $key]; | ||
| 569 | } | ||
| 570 | $array | ||
| 571 | } | ||
| 572 | |||
| 573 | sub reportGuidsToFiles($) #P Map and report guids to files | ||
| 574 | {my ($xref) = @_; # Xref results | ||
| 575 | my @r; | ||
| 576 | if (my $xrefTopicIds = $xref->topicIds) | ||
| 577 | {for my $file(sort keys %{$xrefTopicIds}) # Each input file which will be absolute | ||
| 578 | {if (my $topicId = $xrefTopicIds->{$file}) # Topic Id for file - we report missing topicIds in: reportDuplicateTopicIds | ||
| 579 | {next unless $topicId =~ m(\AGUID-)is; | ||
| 580 | $xref->guidToFile->{$topicId} = $file; # Guid Topic Id to file | ||
| 581 | push @r, [$topicId, $file]; | ||
| 582 | } | ||
| 583 | } | ||
| 584 | } | ||
| 585 | |||
| 586 | formatTables($xref, \@r, | ||
| 587 | columns => <<END, | ||
| 588 | Guid The guid being defined | ||
| 589 | File The file that defines the guid | ||
| 590 | END | ||
| 591 | title =>qq(Guid topic definitions), | ||
| 592 | head =>qq(Xref found NNNN guid topic definitions on DDDD), | ||
| 593 | summarize=>1, | ||
| 594 | file =>fpe(q(lists), qw(guidsToFiles txt))); | ||
| 595 | } | ||
| 596 | |||
| 597 | sub editXml($$$) #P Edit an xml file retaining any existing XML headers and lint trailers | ||
| 598 | {my ($in, $out, $source) = @_; # Input file, output file, source to write | ||
| 599 | |||
| 600 | my @s = readFile($in); # Read existing source | ||
| 601 | |||
| 602 | my @h; # Headers if any present | ||
| 603 | if (@s > 0) # Remove header lines using a very basic parse that is not a general solution | ||
| 604 | {if ($s[0] =~ m(\A\<\?xml)is) # First line | ||
| 605 | {push @h, shift @s; | ||
| 606 | if (@s > 0 and $s[0] =~ m(\A<!DOCTYPE)s) # Second line start | ||
| 607 | {push @h, shift @s; | ||
| 608 | while(@s > 0 and $s[0] !~ m(\A\s*<[a-z])i) # Parse to root tag | ||
| 609 | {push @h, shift @s; | ||
| 610 | } | ||
| 611 | } | ||
| 612 | } | ||
| 613 | } | ||
| 614 | |||
| 615 | my @l; # Lint data if any | ||
| 616 | if (1) | ||
| 617 | {my $state; | ||
| 618 | for my $s(@s) | ||
| 619 | {if (!$state && $s =~ m(\A\<\!\-\-linted\:)s or $state) | ||
| 620 | {push @l, $s; | ||
| 621 | $state++; | ||
| 622 | } | ||
| 623 | } | ||
| 624 | } | ||
| 625 | |||
| 626 | owf($out, join '', @h, $source, @l) # Insert new source between old headers and trailers | ||
| 627 | } | ||
| 628 | |||
| 629 | # Fix a file by moving its hrefs and conrefs to the xtrf attribute unless | ||
| 630 | # deguidization is in effect and the guid can be converted into a valid Dita | ||
| 631 | # reference accessing a file in the input corpus. | ||
| 632 | # | ||
| 633 | # If fixRelocatedRefs is in effect: such references are fixed by assuming that | ||
| 634 | # the files mentioned in broken links have been relocated else where in the | ||
| 635 | # folder structure and can be located by base file name alone. | ||
| 636 | # | ||
| 637 | # If fixXrefsByTitle is in effect apply the Gearhart Title Method: fix broken | ||
| 638 | # xrefs by looking for a unique topic with the same title text as the content of | ||
| 639 | # the xref. | ||
| 640 | # | ||
| 641 | # If fixDitaRefs is in effect we are converting Dita to Dita: relink Dita | ||
| 642 | # references that were valid in the input corpus to make them valid again in the | ||
| 643 | # output corpus even after files have been cut out and renamed to the GB Standard. | ||
| 644 | # The targets/ folder provides the mapping between the input and output corpii. | ||
| 645 | |||
| 646 | sub fixReferencesInOneFile($$) #P Fix one file by moving unresolved references to the xtrf attribute | ||
| 647 | {my ($xref, $sourceFile) = @_; # Xref results, source file to fix | ||
| 648 | my $fixed = newXref(); # Fix results | ||
| 649 | my $node; # The current node we are working with | ||
| 650 | my $attr; # The current attribute we are working with | ||
| 651 | my $ref; # The current reference we are working with | ||
| 652 | my @bad; $fixed->fixedRefsBad = \@bad; # Hrefs that could not be fixed and so were ameliorated by moving them to @xtrf | ||
| 653 | my @good; $fixed->fixedRefsGood = \@good; # Hrefs that were fixed by resolving a Guid | ||
| 654 | |||
| 655 | my $refDetails = sub # Save details of a reference | ||
| 656 | {my ($r) = @_; | ||
| 657 | my $s = $xref->targetTopicToInputFiles->{$sourceFile}; # The source file(s) from which each target was obtained | ||
| 658 | [$r, $node->tag, $attr, $ref, $sourceFile, sort keys %$s] # Construct reference details | ||
| 659 | }; | ||
| 660 | |||
| 661 | my $bad = sub # Save details of a bad reference | ||
| 662 | {my ($r) = @_; | ||
| 663 | push @bad, my $R = &$refDetails($r); | ||
| 664 | $R | ||
| 665 | }; | ||
| 666 | |||
| 667 | my $good = sub # Save details of a good reference | ||
| 668 | {my ($target, $r) = @_; # Target file, reason | ||
| 669 | my $R = &$refDetails($r); | ||
| 670 | push @good, [@$R[0..3], $target, @$R[4..$#$R]]; # Insert target at correct location | ||
| 671 | $R | ||
| 672 | }; | ||
| 673 | |||
| 674 | my $fixXrefByTitle = sub # Attempt to fix an xref by using its text content to search for a matching title | ||
| 675 | {return undef unless -t $node eq q(xref); # Only works for xrefs | ||
| 676 | |||
| 677 | my $xTitle = nws($node->stringContent); # Normalized title from xref node | ||
| 678 | if (my $topics = $xref->titleToFile->{$xTitle}) # Find the topics that match the title text content | ||
| 679 | {my $N = keys %$topics; # Matching topics | ||
| 680 | |||
| 681 | if ($N == 1) # Unique matching topic - the original Gearhart Title Method | ||
| 682 | {my ($path) = keys %$topics; | ||
| 683 | my $rel = relFromAbsAgainstAbs($path, $sourceFile); # Relative file name | ||
| 684 | $node->href = $rel; # Update xref | ||
| 685 | return &$good($path, q(Fixed by Gearhart Title Method)); # Report the fix made | ||
| 686 | } | ||
| 687 | elsif ($N > 1) # Multiple matches | ||
| 688 | {if (my $l = fileLargestSize(sort keys %$topics)) # Boldly choose the topic with the largest size to resolve the ambiguity on the basis that it is probably the most interesting | ||
| 689 | {my $rel = relFromAbsAgainstAbs($l, $sourceFile); # File name of target topic relative to source file | ||
| 690 | $node->href = $rel; # Update reference | ||
| 691 | return &$good($l, q(Fixed by Gearhart Bold Title Method)); # Report the fix made | ||
| 692 | } | ||
| 693 | } | ||
| 694 | } | ||
| 695 | undef # Failed | ||
| 696 | }; | ||
| 697 | |||
| 698 | my $locateUniqueTopicSourceForTargetFile = sub # Unique source file in the input corpus corresponding to the specified target file else undef. There could be multiple such files because of flattening: if there are then they are all supposed to be identical and so any one of them should do. | ||
| 699 | {my ($targetFile) = @_; # The target file we want to locate a unique source file from | ||
| 700 | my $inputFiles = $xref->targetTopicToInputFiles->{$targetFile}; # Input files corresponding to target file | ||
| 701 | return undef unless $inputFiles; # Only if we have input files corresponding to this target file | ||
| 702 | my ($inputFile) = sort keys %$inputFiles; # The unique source file | ||
| 703 | $inputFile | ||
| 704 | }; | ||
| 705 | |||
| 706 | # Given a bookmap and a href to a topic in in in/ find that topic in out/ | ||
| 707 | # - Find source of bookmap | ||
| 708 | # - Find source of topic | ||
| 709 | # - Find target of topic | ||
| 710 | # - If target is a bookmap, substitute it | ||
| 711 | # - If target is a topic, replace it | ||
| 712 | # The original source topic was split into several sub topics as described by a bookmap. | ||
| 713 | # A Dita Book Map reference to such a topic should be replaced by the bookmap content. | ||
| 714 | # Conversely, existing map to map references should be left alone | ||
| 715 | |||
| 716 | my $fixBookMapDitaRef = sub # Fix a partial dita reference in a bookmap. If the reference is to a single topic then replace the href with the renamed topic. If the reference is to a topic that was cut into multiple sub topics then replace the reference with the bookmap that represents the cut out topic. | ||
| 717 | {return undef unless $xref->fixDitaRefs; # Only works if we have the targets folder information | ||
| 718 | |||
| 719 | if (my $docType = $xref->docType->{$sourceFile}) # Must be a bookmap | ||
| 720 | {if ($docType !~ m(map\Z)s) | ||
| 721 | {lll "Not a bookmap: $sourceFile"; | ||
| 722 | return undef; | ||
| 723 | } | ||
| 724 | } | ||
| 725 | else | ||
| 726 | {lll "Not a file known to xref: $sourceFile"; | ||
| 727 | return undef; | ||
| 728 | } | ||
| 729 | |||
| 730 | if (my $bookMapSource = &$locateUniqueTopicSourceForTargetFile($sourceFile))# Source of the book map | ||
| 731 | {my $sourceTopic = absFromAbsPlusRel($bookMapSource, $ref); # Source topic relative to source bookmap | ||
| 732 | |||
| 733 | if (my $sourceTarget = $xref->sourceTopicToTargetBookMap->{$sourceTopic}) # Target of source topic via targets/ folder | ||
| 734 | {my $sourceDocType = $sourceTarget->{sourceDocType} // q(); # Source document type | ||
| 735 | my $sourceTargetType = $sourceTarget->{targetType} // q(); # Target document type | ||
| 736 | if ($sourceDocType !~ m(map\Z)s and $sourceTargetType =~ m(\Abookmap\Z))# Replace this chapter or topic with the content of the book map generated to represent a non bookmap topic that was split into several sub topics described by a bookmap | ||
| 737 | {if (my $generatedBookMap = $sourceTarget->{target}) | ||
| 738 | {if (my $x = Data::Edit::Xml::new($generatedBookMap)) # Parse the generated bookmap for chapters | ||
| 739 | {if ($x->at_bookmap) | ||
| 740 | {if (my @c = $x->c_chapter) # Chapters | ||
| 741 | {for my $c(reverse @c) # Copy chapters | ||
| 742 | {$node->putFirstCut($c); | ||
| 743 | } | ||
| 744 | $node->unwrap; # Unwrap the referencing topic | ||
| 745 | } | ||
| 746 | } | ||
| 747 | else | ||
| 748 | {lll "Expected bookmap, got:", $x->tag, | ||
| 749 | "sourceDocType=$sourceDocType from file:\n$b"; | ||
| 750 | } | ||
| 751 | } | ||
| 752 | return &$good(q(), q(Expanded representative bookmap)); # Record the fix made | ||
| 753 | } | ||
| 754 | else | ||
| 755 | {lll "Generated bookmap does not exist $generatedBookMap"; | ||
| 756 | } | ||
| 757 | } | ||
| 758 | else # Not a bookmap so just upgrade href | ||
| 759 | {my $path = $sourceTarget->{target}; | ||
| 760 | $node->href = relFromAbsAgainstAbs($path, $sourceFile); | ||
| 761 | return &$good($path, q(Unique target from bookmap)); # Record the fix made | ||
| 762 | } | ||
| 763 | } | ||
| 764 | } | ||
| 765 | else | ||
| 766 | {lll "No source for $sourceFile\n"; | ||
| 767 | } | ||
| 768 | |||
| 769 | undef # Failed | ||
| 770 | }; | ||
| 771 | |||
| 772 | my $checkImageRef = sub # Check whether an image exists or not | ||
| 773 | {my $i = absFromAbsPlusRel($sourceFile, $ref); # Local file name | ||
| 774 | return 1 if -e $i; # Local file exists | ||
| 775 | return 2 if -e wwwDecode($i); # Local file exists | ||
| 776 | undef # Local file exists after decoding % signs | ||
| 777 | }; | ||
| 778 | |||
| 779 | my $fixOnePartialDitaRef = sub # Fix a partial dita reference to an externally cut out topic renamed to the GB Standard where such a reference is just a file name as used in a bookmapref. | ||
| 780 | {my ($ref) = @_; # Partial reference | ||
| 781 | return undef unless $xref->fixDitaRefs; # Fixing dita references not requested | ||
| 782 | |||
| 783 | my $topicSource = &$locateUniqueTopicSourceForTargetFile($sourceFile); # Unique source file corresponding to the target file else undef | ||
| 784 | return undef unless $topicSource; # The references can not be resolved without a unique source file. | ||
| 785 | |||
| 786 | my $refIn = absFromAbsPlusRel($topicSource, $ref); # The referenced input file that was present in the input being transformed because we assume that (most of) the input Dita refs were valid | ||
| 787 | |||
| 788 | if (my $new = $xref->inputFileToTargetTopics->{$refIn}) # The target files new files that were cut out of the referenced input file - there might several such | ||
| 789 | {if (my $referencedTarget = fileLargestSize(sort keys %$new)) # Boldly assume that the largest possible target is the one we want | ||
| 790 | {my $link = relFromAbsAgainstAbs($referencedTarget, $sourceFile); # Create relative link from book map | ||
| 791 | $node->set($attr=>$link);# if $xref->fixBadRefs; # Reset reference - we know fixDitaRefs is true. | ||
| 792 | &$good($link, q(unique target)); # Record successful fix | ||
| 793 | return 1; # Success | ||
| 794 | } | ||
| 795 | } | ||
| 796 | undef # Failed | ||
| 797 | }; | ||
| 798 | |||
| 799 | my $fixOneFullDitaRef = sub # Fix a full dita reference to an externally cut out topic renamed to the GB Standard where such a reference is: file#topicId/label | ||
| 800 | {return undef unless $xref->fixDitaRefs; # Fixing dita references not requested | ||
| 801 | return &$fixOnePartialDitaRef($ref) unless $ref =~ m(#); # Confirm it is a full reference else fix it as a partial reference | ||
| 802 | |||
| 803 | my $topicSource = &$locateUniqueTopicSourceForTargetFile($sourceFile); # Unique source file corresponding to the target file else undef | ||
| 804 | return undef unless $topicSource; # The references can not be resolved without a unique source file. | ||
| 805 | my ($rf, $rt, $ri) = parseDitaRef($ref, $topicSource); # Parse the dita ref | ||
| 806 | |||
| 807 | if (my $new = $xref->originalSourceFileAndIdToNewFile->{$rf}{$ri}) # The new files cut out of the original topic source file | ||
| 808 | {my $targetFile = relFromAbsAgainstAbs($new, $sourceFile); # Create relative link from current file | ||
| 809 | if (my $topicId = $xref->topicIds->{$new}) # Topic id for target file | ||
| 810 | {my $href = qq($targetFile#$topicId/$ri); # New href | ||
| 811 | $node->set($attr=>$href);# if $xref->fixBadRefs; # Reset href - we know fixDitaRefs is true. | ||
| 812 | &$good($new, q(Unique target for file ref)); # Record the fix made | ||
| 813 | return 1; # Record the fix made | ||
| 814 | } | ||
| 815 | } | ||
| 816 | |||
| 817 | if ($xref->allowUniquePartialMatches && $attr !~ m(\Aconref)s) # Partial matching - i.e ignoring the stuff to the right of the # in the reference sometimes produces a unique result. | ||
| 818 | {return &$fixOnePartialDitaRef($ref =~ s(#.*\Z) ()rs); # Try to resolve reference as a partial re | ||
| 819 | } | ||
| 820 | |||
| 821 | undef # Failed | ||
| 822 | }; | ||
| 823 | |||
| 824 | my $fixRelRef = sub # Attempt to fix a reference broken by relocation | ||
| 825 | {my ($R, $rest) = split m(#)s, $ref, 2; # Get referenced file name | ||
| 826 | if ($R) | ||
| 827 | {my $r = fne($R); # Href file base name | ||
| 828 | if (my $F = $xref->baseFiles->{$r}) # Relocated else where | ||
| 829 | {my @targets = sort keys(%$F); # Relocation targets | ||
| 830 | if (@targets == 1) # Just one such relocation | ||
| 831 | {my $f = relFromAbsAgainstAbs($targets[0], $sourceFile); # Link to it | ||
| 832 | if ($f ne $R) | ||
| 833 | {my $newLink; # Fix if the target is else where | ||
| 834 | if ($rest) # Link has more than one component | ||
| 835 | {$node->set($attr=>($newLink = $f.q(#).$rest)); # Reset link | ||
| 836 | } | ||
| 837 | else # Link has just one component | ||
| 838 | {$node->set($attr=>($newLink = $f)); # Reset link | ||
| 839 | } | ||
| 840 | my $saveRef = $ref; $ref = $newLink; # Try fixing the relocated reference as a dita reference. | ||
| 841 | my $r = &$fixOneFullDitaRef; | ||
| 842 | $ref = $saveRef; | ||
| 843 | return $r; | ||
| 844 | } | ||
| 845 | } | ||
| 846 | } | ||
| 847 | } | ||
| 848 | undef # Failed | ||
| 849 | }; | ||
| 850 | |||
| 851 | my $fixOneRef = sub # Fix one unresolved reference either by ameliorating it or by moving it to the xtrf attribute thereby putting it in M3. | ||
| 852 | {return unless $xref->fixRefs->{$sourceFile}{$ref}; # Fix not requested for this reference | ||
| 853 | |||
| 854 | if ($xref->deguidize and $ref =~ m(GUID-)is) # On a guid and deguidization allowed so given g1#g2/id convert g1 to a file name by locating the topic with topicId g2. | ||
| 855 | {my @refs = split /\s+/, $ref; # There might be multiple references in the href | ||
| 856 | my @unresolved; # Unresolved targets | ||
| 857 | my @resolved; # Resolved targets | ||
| 858 | |||
| 859 | for my $subRef(@refs) # Each reference in the reference | ||
| 860 | {my ($guid, $rest) = split /#/, $subRef; | ||
| 861 | if (my $target = $xref->guidToFile->{$guid}) # Target file associated with guid | ||
| 862 | {my $link = relFromAbsAgainstAbs($target, $sourceFile); # Relative link | ||
| 863 | $link .= q(#).$rest if $rest; # Remainder of reference which does not change as it is not file related | ||
| 864 | if (!@resolved) # First resolution | ||
| 865 | {$node->set($attr=>$link); # New href or conref | ||
| 866 | &$good($target, q(Deguidized reference)); # Report fix made | ||
| 867 | } | ||
| 868 | push @resolved, $subRef; | ||
| 869 | } | ||
| 870 | else | ||
| 871 | {push @unresolved, $subRef; | ||
| 872 | } | ||
| 873 | } | ||
| 874 | |||
| 875 | if (@unresolved and $xref->fixBadRefs) # Unresolved - transfer all references to xtrf so some-one else can try | ||
| 876 | {$node->renameAttr($attr, q(xtrf)); # No target file for guid | ||
| 877 | &$bad(q(No file for guid)); # Report failure | ||
| 878 | } | ||
| 879 | } | ||
| 880 | elsif ($xref->fixRelocatedRefs and &$fixRelRef) # Try to fix as a relocated ref if possible | ||
| 881 | { | ||
| 882 | } | ||
| 883 | elsif ($xref->fixXrefsByTitle and &$fixXrefByTitle) # Try to fix a missing xref by title | ||
| 884 | { | ||
| 885 | } | ||
| 886 | elsif ($xref->fixBadRefs) # Move href to xtrf as no other fix seems possible given that we have already tried to fix it as a guid and it was reportedly not working as a standard dita reference. | ||
| 887 | {$node->renameAttr($attr, q(xtrf)); # No target file for guid | ||
| 888 | |||
| 889 | if ($xref->changeBadXrefToPh) # Change bad xref to ph if requested | ||
| 890 | {if ($node->at_xref) | ||
| 891 | {$node->change_ph; | ||
| 892 | } | ||
| 893 | } | ||
| 894 | &$bad(q(No such target)); # Report failure | ||
| 895 | } | ||
| 896 | else # ffff - Fix not requested so href left alone | ||
| 897 | {&$bad(q(Not fixable)); # Unable to fix the reference using any known method | ||
| 898 | } | ||
| 899 | }; | ||
| 900 | |||
| 901 | my $x = Data::Edit::Xml::new($sourceFile); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed | ||
| 902 | my $s = -p $x; # Source before any changes | ||
| 903 | |||
| 904 | $x->by(sub # Check any references encountered on each node, Ameliorate some specific cases. If the reference is still invalid report the discrepancy. | ||
| 905 | {my ($o) = @_; # Current node | ||
| 906 | $node = $o; # Make current node available globally | ||
| 907 | my $t = $node->tag; # Tag | ||
| 908 | if ($t =~ m(\A(appendix|chapter|image|link|mapref|topicref|xref)\Z)is) # Hrefs that need to be fixed | ||
| 909 | {if ($ref = $node->attr($attr = q(href))) # The attribute and reference to ameliorate or fix | ||
| 910 | #if ($t =~ m(\A(appendix|chapter|topicref)\Z)is) # Fix bookmap hrefs | ||
| 911 | {if ($t =~ m(\A(appendix|chapter|mapref|topicref)\Z)is) # Fix bookmap hrefs | ||
| 912 | {&$fixBookMapDitaRef or &$fixOneRef; # Fix references to topics cut into multiple pieces and now represented by a bookmap | ||
| 913 | } | ||
| 914 | elsif ($t =~ m(\Aimage\Z)is) # Check image references | ||
| 915 | {&$checkImageRef or &$fixOneRef; # No additional fixes available yet for images, as so far, the resolution of images is done in thee calling frame work. Hence we only need to check whether the reference is valid and if it is not then the standard techniques can be applied and the results reported as usual. | ||
| 916 | } | ||
| 917 | else # Fix hrefs without the benefit of the targets/ folder | ||
| 918 | {&$fixOneFullDitaRef or &$fixOneRef; # Fix references not in a bookmap | ||
| 919 | } | ||
| 920 | } | ||
| 921 | elsif ($t =~ m(\Axref\Z)s and $xref->fixXrefsByTitle and &$fixXrefByTitle)# Try to fix a missing xref by title | ||
| 922 | { | ||
| 923 | } | ||
| 924 | } | ||
| 925 | if ($ref = $node->attr($attr = q(conref))) # Fix a conref | ||
| 926 | {&$fixOneFullDitaRef or &$fixOneRef; | ||
| 927 | } | ||
| 928 | if ($ref = $node->attr($attr = q(conrefend))) # Fix a conrefend | ||
| 929 | {&$fixOneFullDitaRef or &$fixOneRef; | ||
| 930 | } | ||
| 931 | }); | ||
| 932 | |||
| 933 | if (my $S = -p $x) # Source after any changes | ||
| 934 | {if ($S ne $s) # Write any changes - seems to be slightly faster than not checking | ||
| 935 | {if (onAwsSecondary) # Write output to temporary folder regardless so it can be copied enmasse back to the session leader | ||
| 936 | {my $f = swapFolderPrefix($sourceFile, # Output file name | ||
| 937 | $xref->inputFolder, $xref->fixedFolderTemp); | ||
| 938 | editXml($sourceFile, $f, $S); # Write the fixed file to the fixedFolder retaining headers and trailers | ||
| 939 | } | ||
| 940 | elsif (my $fixedFolder = $xref->fixedFolder) # New output file in fixedFolder | ||
| 941 | {my $f = swapFolderPrefix($sourceFile, $xref->inputFolder, $fixedFolder);# Output file name | ||
| 942 | editXml($sourceFile, $f, $S); # Write the fixed file to the fixedFolder retaining headers and trailers | ||
| 943 | } | ||
| 944 | else | ||
| 945 | {editXml($sourceFile, $sourceFile, $S); # Edit existing xml retaining headers and trailers | ||
| 946 | } | ||
| 947 | } | ||
| 948 | } | ||
| 949 | |||
| 950 | $fixed # Results of fixing this file | ||
| 951 | } # fixReferencesInOneFile | ||
| 952 | |||
| 953 | sub fixReferencesParallel($$) #P Fix the references in one file | ||
| 954 | {my ($xref, $file) = @_; # Cross referencer, file to fix | ||
| 955 | |||
| 956 | newXref; # Recreate Xref LVALUE methods | ||
| 957 | |||
| 958 | if (my $d = $xref->fixedFolderTemp) # Create the folder to be used for fixed files | ||
| 959 | {makePath($d); | ||
| 960 | } | ||
| 961 | |||
| 962 | my $x = $xref->fixReferencesInOneFile($file); | ||
| 963 | newXref(fixedRefsGood => $x->fixedRefsGood, | ||
| 964 | fixedRefsBad => $x->fixedRefsBad); | ||
| 965 | }; | ||
| 966 | |||
| 967 | sub fixReferencesResults($@) #P Consolidate the results of fixing references. | ||
| 968 | {my ($xref, @results) = @_; # Cross referencer, results from fixReferencesInParallel | ||
| 969 | newXref; # Recreate LVALUE methods | ||
| 970 | |||
| 971 | $xref->fixedRefsBad = []; | ||
| 972 | $xref->fixedRefsGood = []; | ||
| 973 | for my $x(@results) | ||
| 974 | {push $xref->fixedRefsBad ->@*, $x->fixedRefsBad ->@*; | ||
| 975 | push $xref->fixedRefsGood->@*, $x->fixedRefsGood->@*; | ||
| 976 | } | ||
| 977 | |||
| 978 | if (onAwsPrimary) # Recover fixed files from secondary instances | ||
| 979 | {my $s = $xref->fixedFolderTemp; | ||
| 980 | makePath($s); | ||
| 981 | my $t = $xref->fixedFolder // $xref->inputFolder; | ||
| 982 | awsParallelGatherFolder($s); # Recover fixed files from secondaries | ||
| 983 | mergeFolder($s, $t); # Merge fixed files into the target area | ||
| 984 | } | ||
| 985 | |||
| 986 | $xref | ||
| 987 | } | ||
| 988 | |||
| 989 | sub fixReferences($) #P Fix just the file containing references using a number of techniques and report those references that cannot be so fixed. | ||
| 990 | {my ($xref) = @_; # Xref results | ||
| 991 | my $startTime = time; # Time each block | ||
| 992 | |||
| 993 | if (1) # Map titles to files for the Gearhart Title Method | ||
| 994 | {my %titleToFile; # Titles to file | ||
| 995 | if (my $xrefTitle = $xref->title) | ||
| 996 | {for my $file(keys %{$xrefTitle}) # Title for each file | ||
| 997 | {if (my $tag = $xref->docType->{$file}) # Document type for file | ||
| 998 | {if ($tag !~ m(map\Z)s) # Ignore maps as we want the topic in the map not the map. | ||
| 999 | {$titleToFile{nws($xrefTitle->{$file})}{$file}++; # Record title to topic | ||
| 1000 | } | ||
| 1001 | } | ||
| 1002 | } | ||
| 1003 | } | ||
| 1004 | |||
| 1005 | $xref->titleToFile = \%titleToFile; # Record titles to files | ||
| 1006 | |||
| 1007 | if (1) # Report titles with duplicated titles | ||
| 1008 | {my @r; | ||
| 1009 | |||
| 1010 | for my $t(sort keys %titleToFile) | ||
| 1011 | {my %f = %{$titleToFile{$t}}; | ||
| 1012 | if (my @f = sort keys %f) | ||
| 1013 | {if (@f > 1) | ||
| 1014 | {push @r, map {[$t, $_]} @f; | ||
| 1015 | } | ||
| 1016 | } | ||
| 1017 | } | ||
| 1018 | |||
| 1019 | formatTables($xref, \@r, | ||
| 1020 | columns => <<END, | ||
| 1021 | Title Topic title | ||
| 1022 | File Topic file | ||
| 1023 | END | ||
| 1024 | title => qq(Topics with duplicate titles), | ||
| 1025 | head => <<END, | ||
| 1026 | Xref noted NNNN topics have duplicated titles on DDDD | ||
| 1027 | END | ||
| 1028 | clearUpLeft=>1, | ||
| 1029 | file=>(fpe(qw(bad topics_with_duplicated_titles txt)))); | ||
| 1030 | } | ||
| 1031 | } | ||
| 1032 | |||
| 1033 | if (my $d = $xref->fixDitaRefs) # Map where the input files went and where the target files came from | ||
| 1034 | {my $bookmaps = $xref->targetFolderContent = readFiles($d); # The targets produced from each input file | ||
| 1035 | |||
| 1036 | my @r; # Source to target report | ||
| 1037 | my %sourceToTarget; # Maps a source file to its resulting output topics | ||
| 1038 | my %sourceTopicToTargetBookMap; # Source topic cut into multiple parts produces a bookmap | ||
| 1039 | |||
| 1040 | for my $source(sort keys %$bookmaps) # Each input file represented in the targets folder | ||
| 1041 | {my $sourceToTarget = eval $$bookmaps{$source}; # Mapping for input file | ||
| 1042 | $@ and confess $@; # Check eval was successful | ||
| 1043 | |||
| 1044 | my $bookMap = genHash(qq(Bookmap), %$sourceToTarget); # Details of bookmap representing source file after possible cutting out | ||
| 1045 | my $bookMapSource = $bookMap->source; # Input file name from input folder rather than target folder | ||
| 1046 | my $bookMapTarget = $bookMap->{target} // q(); # The target bookmap | ||
| 1047 | my $sourceDocType = $bookMap->{sourceDocType} // q(); # The docType of the source input file if known | ||
| 1048 | my $targetType = $bookMap->{targetType} // q(); # The target type, initially just a bookmap, now extended to include topics and images | ||
| 1049 | |||
| 1050 | push @r, [$targetType, $sourceDocType, $bookMapSource, $bookMapTarget]; # Report source to targets | ||
| 1051 | $sourceTopicToTargetBookMap{$bookMapSource} = $bookMap; # Source to target details of topic | ||
| 1052 | |||
| 1053 | if ($sourceDocType =~ m(map\Z)s) # If the input file was a map then its target is a map | ||
| 1054 | {$sourceToTarget{$bookMapSource} = {$bookMapTarget=>1}; | ||
| 1055 | } | ||
| 1056 | elsif ($targetType =~ m(\Atopic\Z)is) # If the input file produced a single dita topic then the target is that topic | ||
| 1057 | {$sourceToTarget{$bookMapSource} = {$bookMapTarget=>1}; | ||
| 1058 | } | ||
| 1059 | elsif ($targetType =~ m(\Abookmap\Z)i) # If the input file was a topic that was cut into multiple topics then the target is a bookmap | ||
| 1060 | {if (my $refs = $xref->topicsReferencedFromBookMaps->{$bookMapTarget}) # The references from the bookmap the source file became | ||
| 1061 | {$sourceToTarget{$bookMapSource} = $refs; # The output files files this source input file was split into | ||
| 1062 | } | ||
| 1063 | else | ||
| 1064 | {#lll "No references from bookmap $bookMapTarget"; | ||
| 1065 | } | ||
| 1066 | } | ||
| 1067 | } | ||
| 1068 | |||
| 1069 | my %targetToSource; # The source files for each output topic - the reverse of sourceTotarget - output topics that have been flattened will have multiple sources | ||
| 1070 | for my $source(sort keys %sourceToTarget) # Each source input file | ||
| 1071 | {for my $target(sort keys %{$sourceToTarget{$source}}) # Each of the target topics that were derived from this source file | ||
| 1072 | {$targetToSource{$target}{$source}++; # Maps an output topic back to the input files that gave rise to it | ||
| 1073 | } | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | my %targetToSourceDuplicated; # The target files for which there is more than one source file | ||
| 1077 | for my $target(sort keys %targetToSource) # Each of the target topics that were derived from this source file | ||
| 1078 | {my %s = %{$targetToSource{$target}}; # Each source input file | ||
| 1079 | if (keys %s > 1) # Mapping is not injective | ||
| 1080 | {push @{$targetToSourceDuplicated{$target}}, sort keys %s; | ||
| 1081 | } | ||
| 1082 | } | ||
| 1083 | |||
| 1084 | my %si; # Source ids: {original source input file}{id} = target file | ||
| 1085 | my $xrefIds = $xref->ids; | ||
| 1086 | for my $targetFile(keys %$xrefIds) # Each target file with an id in it | ||
| 1087 | {if (my $sourceFiles = $targetToSource{$targetFile}) # Originating source files for this target file | ||
| 1088 | {for my $sourceFile(keys %$sourceFiles) # Each originating source files for this target file | ||
| 1089 | {for my $id(keys $xrefIds->{$targetFile}->%*) # Each id in the target file | ||
| 1090 | {$si{$sourceFile}{$id} = $targetFile; # The new file containing the id defined in the source file | ||
| 1091 | } | ||
| 1092 | } | ||
| 1093 | } | ||
| 1094 | } | ||
| 1095 | |||
| 1096 | $xref->inputFileToTargetTopics = \%sourceToTarget; # The targets for each input file | ||
| 1097 | $xref->targetTopicToInputFiles = \%targetToSource; # The source file from which each target was obtained | ||
| 1098 | $xref->sourceTopicToTargetBookMap = \%sourceTopicToTargetBookMap; # The bookmap representing a cut up topic | ||
| 1099 | $xref->topicFlattening = \%targetToSourceDuplicated; # Topics that arose from flattening several source files | ||
| 1100 | $xref->originalSourceFileAndIdToNewFile = \%si; # Record mapping from original source file and id to the new file containing the id | ||
| 1101 | |||
| 1102 | formatTables($xref, \@r, | ||
| 1103 | columns => <<END, | ||
| 1104 | Type The type of reference | ||
| 1105 | DocType Document type of the source file | ||
| 1106 | Source Source file | ||
| 1107 | Target Cut out file | ||
| 1108 | END | ||
| 1109 | summarize=>1, | ||
| 1110 | title=>qq(The target topics cut out of the source documents), | ||
| 1111 | head=><<END, | ||
| 1112 | Xref noted NNNN cut out topics on DDDD | ||
| 1113 | END | ||
| 1114 | file=>(fpe(qw(lists source_to_targets txt)))); | ||
| 1115 | |||
| 1116 | if (1) # Report topic flattening | ||
| 1117 | {my @r; | ||
| 1118 | my $s = 0; my $t = 0; | ||
| 1119 | for my $target(sort keys %targetToSourceDuplicated) # Each of the target topics that were derived from this source file | ||
| 1120 | {my @s = @{$targetToSourceDuplicated{$target}}; # Each source input file | ||
| 1121 | push @r, [scalar(@s), $target]; | ||
| 1122 | push @r, [q(), q( ).$_] for @s; | ||
| 1123 | push @r, [q()]; | ||
| 1124 | ++$t; $s += @s; | ||
| 1125 | } | ||
| 1126 | $xref->topicsFlattened = $s; # Record the number of topics flattened | ||
| 1127 | my $F = $xref->topicFlatteningFactor = $t ? $s / $t : 0; # Topic flattening factor - higher is better | ||
| 1128 | my $f = sprintf("%7.4f", $F); | ||
| 1129 | my $n = @{$xref->inputFiles}; # Number of topics | ||
| 1130 | my $p = sprintf("%7.4f", $n ? 100*$t/$n : 0); # Percentage topics flattened versus total number of topics | ||
| 1131 | |||
| 1132 | formatTables($xref, \@r, | ||
| 1133 | columns => <<END, | ||
| 1134 | Count Number of sources that created this target | ||
| 1135 | Target The target file flattened out from multiple source files | ||
| 1136 | END | ||
| 1137 | summarize => 1, | ||
| 1138 | title => qq(Topic files flattened from multiple sources), | ||
| 1139 | head => <<END, | ||
| 1140 | Xref noted that $s source topics were reduced to $t target topics on DDDD | ||
| 1141 | |||
| 1142 | This represents a flattening factor of: $f (higher is better) in the topics that got flattened | ||
| 1143 | |||
| 1144 | Total number of topics : $n | ||
| 1145 | Number of topics flattened: $t | ||
| 1146 | Percent topics flattened: $p | ||
| 1147 | END | ||
| 1148 | file => fpe(qw(lists topic_flattening txt))); | ||
| 1149 | } | ||
| 1150 | } | ||
| 1151 | |||
| 1152 | if ($xref->fixRelocatedRefs) # Load base file name to full name but if needed to do relocation fixes | ||
| 1153 | {my %baseFiles; # Map base files back to full files. The base file is the file name shorn of the path - the reason the GB Standard is so important | ||
| 1154 | for my $file(searchDirectoryTreesForMatchingFiles($xref->inputFolder)) # All input files | ||
| 1155 | {my $base = fne $file; # Base file name - the GB Standard name for the file | ||
| 1156 | $baseFiles{$base}{$file}++; # Current location of the file | ||
| 1157 | } | ||
| 1158 | $xref->baseFiles = \%baseFiles; | ||
| 1159 | } | ||
| 1160 | |||
| 1161 | my @bad; # Hrefs that could not be fixed and so were ameliorated by moving them to @xtrf | ||
| 1162 | my @good; # Hrefs that were fixed by resolving a Guid | ||
| 1163 | if (my @files = sort keys %{$xref->fixRefs}) # Fix files if requested | ||
| 1164 | {awsParallelProcessFiles $xref, # Fix files in parallel | ||
| 1165 | \&fixReferencesParallel, # Fix one file | ||
| 1166 | \&fixReferencesResults, # Consolidate results | ||
| 1167 | [@files]; | ||
| 1168 | |||
| 1169 | @good = $xref->fixedRefsGood->@*; # Results from fixReferencesResults | ||
| 1170 | @bad = $xref->fixedRefsBad ->@*; | ||
| 1171 | } | ||
| 1172 | |||
| 1173 | @good = sort {join(' ', @$a) cmp join(' ', @$b)} @good; | ||
| 1174 | @bad = sort {join(' ', @$a) cmp join(' ', @$b)} @bad; | ||
| 1175 | |||
| 1176 | my $fbr = $xref->fixBadRefs; # Are we fixing bad refs? | ||
| 1177 | my $facet = q(Dita references); | ||
| 1178 | |||
| 1179 | formatTables($xref, $xref->fixedRefsBad = \@bad, # Report references we cannot fix | ||
| 1180 | columns => <<END, | ||
| 1181 | Reason The reason the reference was not fixed | ||
| 1182 | Tag The tag of the node in which the reference failure occurs | ||
| 1183 | Attr The attribute of the node in which the reference failure occurs | ||
| 1184 | Reference The reference not being fixed | ||
| 1185 | File The file in which the reference appears | ||
| 1186 | Source_Files One or more source files that from which this file was derived | ||
| 1187 | END | ||
| 1188 | summarize => 1, | ||
| 1189 | title => q(Invalid references), | ||
| 1190 | facet => $facet, aspectColor => q(red), | ||
| 1191 | head => $fbr ? <<END : <<END2, | ||
| 1192 | Xref moved NNNN invalid references to M3 on DDDD as fixBadRefs=>$fbr was specified | ||
| 1193 | END | ||
| 1194 | Xref was unable to resolve NNNN failing references on DDDD, fixBadRefs=> was not specified | ||
| 1195 | END2 | ||
| 1196 | zero => 1, | ||
| 1197 | file => fpe(qw(bad failing_references txt))); | ||
| 1198 | |||
| 1199 | formatTables($xref, $xref->fixedRefsGood = \@good, # Report hrefs which were failing but were successfully resolved by ingenuity. | ||
| 1200 | columns => <<END, | ||
| 1201 | Method The way that the reference was fixed | ||
| 1202 | Tag The tag of the node on which the reference was fixed | ||
| 1203 | Attr The attribute being fixed - normally href | ||
| 1204 | Ref The reference that is being resolved | ||
| 1205 | Target_File The file the reference resolves to. | ||
| 1206 | File The file in which the reference appears | ||
| 1207 | Source_Files The source files that gave rise to the file containing the reference after file flattening | ||
| 1208 | END | ||
| 1209 | summarize => 1, | ||
| 1210 | title => qq(These failing references were successfully resolved), | ||
| 1211 | facet => $facet, aspectColor => q(green), | ||
| 1212 | head => <<END, | ||
| 1213 | Xref successfully resolved NNNN previously failing references on DDDD | ||
| 1214 | END | ||
| 1215 | zero => 1, | ||
| 1216 | file => fpe(qw(good fixed_references txt))); | ||
| 1217 | |||
| 1218 | } # fixReferences | ||
| 1219 | |||
| 1220 | sub fixOneFileGB($$) #P Fix one file to the Gearhart-Brenan standard | ||
| 1221 | {my ($xref, $file) = @_; # Xref results, file to fix | ||
| 1222 | my @r; # Hrefs changed | ||
| 1223 | |||
| 1224 | my $x = Data::Edit::Xml::new($file); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed | ||
| 1225 | |||
| 1226 | $x->by(sub # Each node | ||
| 1227 | {my ($o) = @_; | ||
| 1228 | if (my $h = $o->href) # Href encountered | ||
| 1229 | {my ($localFile, $rest) = split /#/, $h, 2; # Split reference | ||
| 1230 | my $fullFile = absFromAbsPlusRel($file, $localFile); # Full name of referenced file | ||
| 1231 | if (my $target = $xref->flattenFiles->{$fullFile}) # Target file name | ||
| 1232 | {$o->href = $target.($rest ? qq(#$rest) : qq()); # Reassemble href | ||
| 1233 | } | ||
| 1234 | else | ||
| 1235 | {push @r, [$h, $file]; | ||
| 1236 | } | ||
| 1237 | } | ||
| 1238 | }); | ||
| 1239 | |||
| 1240 | my $target = fpf($xref->flattenFolder, $xref->flattenFiles->{$file}); # Previously assigned GB name. We cannot use the very latest name because other files have to be told about it and in changing them to reflect the latest name we would change their name as well. So close has to be good enough. | ||
| 1241 | editXml($file, $target, -p $x); # Edit xml | ||
| 1242 | |||
| 1243 | \@r # Return report of items fixed | ||
| 1244 | } | ||
| 1245 | |||
| 1246 | sub fixFilesGB($) #P Rename files to the L<GBStandard> | ||
| 1247 | {my ($xref) = @_; # Xref results | ||
| 1248 | my @files = grep {!$xref->parseFailed->{$_}} sort @{$xref->inputFiles}; # Fix files that parsed if requested | ||
| 1249 | |||
| 1250 | my @r; # Fixes made | ||
| 1251 | processFilesInParallel | ||
| 1252 | sub # Each file | ||
| 1253 | {my ($file) = @_; # File to analyze | ||
| 1254 | $xref->fixOneFileGB($file); # Analyze one input file | ||
| 1255 | }, | ||
| 1256 | sub {push @r, deSquareArray @_}, @files; # Flatten results | ||
| 1257 | |||
| 1258 | formatTables($xref, $xref->fixedRefsGB = \@r, # Report results | ||
| 1259 | columns => <<END, | ||
| 1260 | Href The href being fixed | ||
| 1261 | Source The source file containing the href | ||
| 1262 | END | ||
| 1263 | summarize=>1, | ||
| 1264 | title=>qq(Hrefs that can not be renamed to the Gearhart-Brenan file naming standard), | ||
| 1265 | head=><<END, | ||
| 1266 | Xref failed to fix NNNN hrefs to the Gearhart-Brenan file naming standard | ||
| 1267 | END | ||
| 1268 | file=>(my $f = fpe(qw(bad fixedRefsGB txt)))); | ||
| 1269 | |||
| 1270 | {fixedRefsGB => $xref->fixedRefsGB, # From multiverse to universe | ||
| 1271 | } | ||
| 1272 | } # fixFilesGB | ||
| 1273 | |||
| 1274 | sub analyzeOneFileParallel($$) #P Analyze one input file | ||
| 1275 | {my ($Xref, $iFile) = @_; # Xref request, File to analyze | ||
| 1276 | |||
| 1277 | my $xref = newXref(); # Cross referencer for this file | ||
| 1278 | # $xref->sourceFile = $iFile; # File analyzed | ||
| 1279 | my %maxZoomIn = $Xref->maxZoomIn ? %{$Xref->maxZoomIn} : (); # Regular expressions from maxZoomIn to look for text | ||
| 1280 | my %maxZoomOut; # Text elements that match a maxZoomIn regular expression | ||
| 1281 | my %countAttrNames; # Attribute names | ||
| 1282 | my %countAttrNamesAndValues; # Attribute names and values | ||
| 1283 | my %countTagNames; # Tag names | ||
| 1284 | my $changes; # Changes made to the file | ||
| 1285 | my $tags; my $texts; # Number of tags and text elements | ||
| 1286 | |||
| 1287 | my $source = readFile($iFile); # Source of file so we can gets its GB Standard name | ||
| 1288 | |||
| 1289 | my $x = eval {Data::Edit::Xml::new($iFile)}; # Parse xml - at this point if the caller is interested in line numbers they should have added them. | ||
| 1290 | |||
| 1291 | if ($@) # Check we were able to parse the xml | ||
| 1292 | {$xref->parseFailed->{$iFile}++; | ||
| 1293 | return $xref; | ||
| 1294 | } | ||
| 1295 | |||
| 1296 | my $md5 = $xref->md5Sum->{$iFile} = -M $x; # Md5 sum for parse tree | ||
| 1297 | |||
| 1298 | if ($Xref->flattenFolder) | ||
| 1299 | {$xref->flattenFiles->{$iFile} = # Record correspondence between existing file and its GB Standard file name | ||
| 1300 | Dita::GB::Standard::gbStandardFileName($source, fe($iFile), md5=>$md5); | ||
| 1301 | } | ||
| 1302 | |||
| 1303 | my $saveReference = sub # Save a reference so it can be integrity checked later | ||
| 1304 | {my ($ref) = @_; # Reference | ||
| 1305 | return if externalReference($ref); # Looks like an external reference | ||
| 1306 | $xref->references->{$iFile}{$ref}++; # Save reference | ||
| 1307 | }; | ||
| 1308 | |||
| 1309 | my $isADitaMap = $x->isADitaMap; # Map | ||
| 1310 | |||
| 1311 | $x->by(sub # Each node | ||
| 1312 | {my ($o) = @_; | ||
| 1313 | |||
| 1314 | # my $content = sub #P First few characters of content on one line to avoid triggering multi table layouts | ||
| 1315 | # {my ($o) = @_; # String | ||
| 1316 | # nws($o->stringContent, improvementLength); # Length of improvement | ||
| 1317 | # }; | ||
| 1318 | |||
| 1319 | my $tag = -t $o; # Element tag | ||
| 1320 | if ($tag eq q(CDATA)) {++$texts} else {++$tags} # Count texts and tags | ||
| 1321 | |||
| 1322 | if (my $h = $o->href) # Check href | ||
| 1323 | {if ($h =~ m(\s)s and externalReference($h)) # Check href for url encoding needed | ||
| 1324 | {$xref->{hrefUrlEncoding}{$iFile}{$o->lineLocation} = $h; | ||
| 1325 | } | ||
| 1326 | if ($xref->{deguidize} and $h =~ m(\bguid-)is) # Deguidizing a href that looks as if it might have a guid in it | ||
| 1327 | {$xref->{fixRefs}{$iFile}{$h}++ | ||
| 1328 | } | ||
| 1329 | &$saveReference($h); | ||
| 1330 | } | ||
| 1331 | |||
| 1332 | if (my $conref = $o->attr(q(conref))) # Conref | ||
| 1333 | {my $saveConRef = sub # Save a conref | ||
| 1334 | {my ($conRef) = @_; # Conref | ||
| 1335 | $xref->{conRefs}{$iFile}{$conRef}{$tag}++; | ||
| 1336 | }; | ||
| 1337 | |||
| 1338 | &$saveConRef($conref); | ||
| 1339 | &$saveReference($conref); | ||
| 1340 | |||
| 1341 | if (my $conref = $o->attr(q(conrefend))) # Conref end | ||
| 1342 | {&$saveConRef($conref); | ||
| 1343 | &$saveReference($conref); | ||
| 1344 | } | ||
| 1345 | } | ||
| 1346 | |||
| 1347 | if (my $i = $o->id) # Id definitions | ||
| 1348 | {$xref->{ids}{$iFile}{$i}++; | ||
| 1349 | push $xref->{idTags}{$iFile}{$i}->@*, $tag; # Tags for each id in the file | ||
| 1350 | } | ||
| 1351 | |||
| 1352 | if ($tag eq q(xref)) # Xrefs but not to the web | ||
| 1353 | {if (my $h = $o->href) | ||
| 1354 | {if (externalReference($h)) # Check attributes on external links | ||
| 1355 | {if ($o->attrX(q(scope)) !~ m(\Aexternal\Z)s) | ||
| 1356 | {$xref->{xrefBadScope}{$iFile}{$h} = -A $o; | ||
| 1357 | } | ||
| 1358 | if ($o->attrX(q(format)) !~ m(\Ahtml\Z)s) | ||
| 1359 | {$xref->{xrefBadFormat}{$iFile}{$h} = -A $o; | ||
| 1360 | } | ||
| 1361 | } | ||
| 1362 | elsif ($h =~ m(\Aguid-)is) # Href is a guid | ||
| 1363 | {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation]; | ||
| 1364 | } | ||
| 1365 | else #if ($o->attrX_format =~ m(\Adita)i) # Check xref has format=dita | ||
| 1366 | {$xref->{xRefs}{$iFile}{$h}{$o->stringText}++; | ||
| 1367 | } | ||
| 1368 | } | ||
| 1369 | else | ||
| 1370 | {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href | ||
| 1371 | $xref->{fixRefs}{$iFile}{q()}++ # Try and fix by the Gearhart Title method augmented by the Monroe Map Method | ||
| 1372 | } | ||
| 1373 | } | ||
| 1374 | elsif ($isADitaMap and $tag =~ m(\A(appendix|chapter|link|mapref|notices|topicref)\Z)is) # References from bookmaps at 2019.11.10 22:58:24 as mapref can be used in a topic | ||
| 1375 | {if (my $h = $o->href) | ||
| 1376 | {if ($h =~ m(\Aguid-)is) # Href is a guid | ||
| 1377 | {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation]; | ||
| 1378 | } | ||
| 1379 | else | ||
| 1380 | {$xref->{bookMapRefs}{$iFile}{$h}{$o->attr_navtitle//$o->stringText}++; | ||
| 1381 | } | ||
| 1382 | } | ||
| 1383 | elsif ($tag ne q(notices)) # Notices is often positioned in a bookmap and left empty for author convenience | ||
| 1384 | {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href | ||
| 1385 | } | ||
| 1386 | } | ||
| 1387 | elsif ($tag eq q(image)) # Images | ||
| 1388 | {if (my $h = $o->href) | ||
| 1389 | {if ($h =~ m(\Aguid-)is) # Href is a guid | ||
| 1390 | {$xref->{guidHrefs}{$iFile}{$h} = [$tag, $o->lineLocation]; # Resolve image later | ||
| 1391 | } | ||
| 1392 | else | ||
| 1393 | {$xref->{images}{$iFile}{$h}++; | ||
| 1394 | } | ||
| 1395 | $xref->{imagesReferencedFromTopics}{$iFile}{$h}++; # Image referenced from a topic | ||
| 1396 | } | ||
| 1397 | else | ||
| 1398 | {push @{$xref->{noHref}{$iFile}}, [$tag, $o->lineLocation, $iFile]; # No href | ||
| 1399 | } | ||
| 1400 | } | ||
| 1401 | elsif ($tag eq q(required-cleanup)) # Required cleanup | ||
| 1402 | {$xref->{requiredCleanUp}{$iFile}{nws($o->stringContent)}++; | ||
| 1403 | } | ||
| 1404 | elsif ($tag eq q(title) and $o->parent == $x) # Title | ||
| 1405 | {$xref->{title}{$iFile} = $o->stringContent; # Topic Id | ||
| 1406 | } | ||
| 1407 | elsif ($tag eq q(mainbooktitle)) # Title for bookmaps | ||
| 1408 | {$xref->{title}{$iFile} //= $o->stringText; | ||
| 1409 | } | ||
| 1410 | elsif ($tag eq q(author)) # Author | ||
| 1411 | {$xref->{author}{$iFile} = $o->stringContent; | ||
| 1412 | } | ||
| 1413 | elsif ($tag eq q(ol)) # Ol | ||
| 1414 | {if (my $p = $o->parent) | ||
| 1415 | {if ($p->tag =~ m(body\Z)s) | ||
| 1416 | {$xref->{olBody}{$iFile}++; | ||
| 1417 | } | ||
| 1418 | } | ||
| 1419 | } | ||
| 1420 | elsif ($tag eq q(tgroup)) # Tgroup cols | ||
| 1421 | {my $error = sub # Table error message | ||
| 1422 | {push @{$xref->{badTables}}, | ||
| 1423 | [join('', @_), $tag, $o->lineLocation, $iFile]; | ||
| 1424 | }; | ||
| 1425 | |||
| 1426 | my $stats = $o->ditaTGroupStatistics; # Statistics for table | ||
| 1427 | my $cols = $stats->colsAttribute; | ||
| 1428 | my $maxCols = max($stats->maxHead//0, $stats->maxBody//0); | ||
| 1429 | my $maxColsMP = max($stats->maxHeadMinusPadding//0, | ||
| 1430 | $stats->maxBodyMinusPadding//0); | ||
| 1431 | if (($stats->maxHead//0) == $maxCols && # The right combination of body and header | ||
| 1432 | ($stats->minHead//0) == $maxCols && | ||
| 1433 | ($stats->maxBody//0) == $maxCols && | ||
| 1434 | ($stats->minBody//0) == $maxCols && | ||
| 1435 | $stats->colSpec == $maxCols | ||
| 1436 | or !defined($stats->maxHead) && # No headers but everything else looks good | ||
| 1437 | ($stats->maxBody//0) == $maxCols && | ||
| 1438 | ($stats->minBody//0) == $maxCols && | ||
| 1439 | $stats->colSpec == $maxCols) | ||
| 1440 | {if (!$cols) # Check for cols attribute | ||
| 1441 | {$error->(qq(No cols attribute, should be $maxCols)); | ||
| 1442 | } | ||
| 1443 | elsif ($cols ne $maxCols) # Cols present but wrong | ||
| 1444 | {$error->(qq(Cols attribute is $cols but should be $maxCols)); | ||
| 1445 | } | ||
| 1446 | } | ||
| 1447 | elsif ($maxColsMP > (my $actual = $stats->maxHead//0)) # Not enough headers | ||
| 1448 | {$error->(qq(Not enough headers, $actual vs $maxColsMP)); | ||
| 1449 | } | ||
| 1450 | else | ||
| 1451 | {$error->(qq(Column padding required)); | ||
| 1452 | } | ||
| 1453 | if ($maxCols and my $rows = $stats->rows) # Count table sizes | ||
| 1454 | {$xref->{tableDimensions}{$iFile}{$maxCols}{$rows}++; | ||
| 1455 | } | ||
| 1456 | } | ||
| 1457 | elsif ($tag =~ m(\Aothermeta\Z)) # Other meta tags | ||
| 1458 | {my $c = $o->attrX(q(content)); | ||
| 1459 | my $n = $o->attrX(q(name)); | ||
| 1460 | $xref->{otherMeta}{$iFile}{$n}{$c}++; | ||
| 1461 | } | ||
| 1462 | elsif ($tag =~ m(body\Z) and $o->isAllBlankText) # Empty body | ||
| 1463 | {$xref->{emptyTopics}{$iFile}++; | ||
| 1464 | } | ||
| 1465 | elsif ($o->isText) # Not much interest - obscured at the start of this 'if' statement - Check text for interesting constructs | ||
| 1466 | {my $t = $o->text; | ||
| 1467 | my @l = $t =~ m(<(.*?)>)g; | ||
| 1468 | for my $l(@l) | ||
| 1469 | {$xref->{ltgt}{$iFile}{$l}++; | ||
| 1470 | } | ||
| 1471 | if (keys %maxZoomIn) # Search for text using Micaela's Max Zoom In Method | ||
| 1472 | {for my $name(sort keys %maxZoomIn) # Each regular expression to check | ||
| 1473 | {my $re = $maxZoomIn{$name}; | ||
| 1474 | if ($t =~ m($re)is) | ||
| 1475 | {$maxZoomOut{$name}++ | ||
| 1476 | } | ||
| 1477 | } | ||
| 1478 | } | ||
| 1479 | } | ||
| 1480 | |||
| 1481 | if (my $a = $o->{attributes}) # Count tags, attributes, attribute values | ||
| 1482 | {for my $k(sort keys %$a) | ||
| 1483 | {$countAttrNames{$k}++; | ||
| 1484 | if (my $v = $$a{$k}) | ||
| 1485 | {$countAttrNamesAndValues{$k}{$v}++; | ||
| 1486 | } | ||
| 1487 | } | ||
| 1488 | $countTagNames{$tag}++; | ||
| 1489 | } | ||
| 1490 | }); | ||
| 1491 | |||
| 1492 | $xref->maxZoomOut->{$iFile} = \%maxZoomOut; # Save max zoom | ||
| 1493 | |||
| 1494 | my $docType = parseXmlDocType($source); # Get DocType details | ||
| 1495 | |||
| 1496 | $xref->attributeCount ->{$iFile} = \%countAttrNames; # Attribute names | ||
| 1497 | $xref->attributeNamesAndValuesCount->{$iFile} = \%countAttrNamesAndValues; # Attribute names and values | ||
| 1498 | $xref->tagCount ->{$iFile} = \%countTagNames; # Tag names | ||
| 1499 | $xref->baseTag ->{$iFile} = $x->tag; # Tag on base node | ||
| 1500 | $xref->docType ->{$iFile} = $x->tag; # Document type | ||
| 1501 | $xref->publicId ->{$iFile} = $docType->{publicId}; # Public id on Doctype | ||
| 1502 | $xref->tags ->{$iFile} = $tags; # Number of tags | ||
| 1503 | $xref->texts ->{$iFile} = $texts; # Number of texts | ||
| 1504 | $xref->topicIds ->{$iFile} = $x->id; # Topic Id | ||
| 1505 | $xref->vocabulary ->{$iFile} = $x->stringTagsAndText # Text of topic minus attributes | ||
| 1506 | if !$isADitaMap and $Xref->indexWords||$Xref->matchTopics; # Maps tend not to have any matchable text in them. | ||
| 1507 | |||
| 1508 | if (my @urls = $source =~ m(["'](https?://[^"']*?)["'])g) # Urls found in file | ||
| 1509 | {$xref->urls->{$iFile} = {map {$_=>1} @urls}; | ||
| 1510 | } | ||
| 1511 | |||
| 1512 | if (1) # Check xml headers and lint errors | ||
| 1513 | {my @h = split /\n/, my $s = readFile($iFile); | ||
| 1514 | if (!$h[0] or $h[0] !~ m(\A<\?xml version=\"1.0\" encoding=\"UTF-8\"\?>\Z)) | ||
| 1515 | {$xref->badXml1->{$iFile}++; | ||
| 1516 | } | ||
| 1517 | my $tag = $x->tag; | ||
| 1518 | if (!$h[1] or $h[1] !~ m(\A<!DOCTYPE $tag PUBLIC "-//)) | ||
| 1519 | {$xref->badXml2->{$iFile}++; | ||
| 1520 | } | ||
| 1521 | |||
| 1522 | $xref->validationErrors->{$iFile}++ if $s =~ m(<!--compressedErrors:)s; # File has validation errors | ||
| 1523 | } | ||
| 1524 | |||
| 1525 | $xref | ||
| 1526 | } # analyzeOneFileParallel | ||
| 1527 | |||
| 1528 | sub analyzeOneFileResults($@) #P Merge a list of cross reference results into the first cross referencer in the list | ||
| 1529 | {my ($xref, @x) = @_; # Cross referencer to merge into, other cross referencers | ||
| 1530 | |||
| 1531 | newXref; # Create LVALUE methods | ||
| 1532 | |||
| 1533 | my @fields = ( # Fields to be merged | ||
| 1534 | q(attributeCount), | ||
| 1535 | q(attributeNamesAndValuesCount), | ||
| 1536 | q(author), | ||
| 1537 | q(badXml1), | ||
| 1538 | q(badXml2), | ||
| 1539 | q(baseTag), | ||
| 1540 | q(conRefs), | ||
| 1541 | q(docType), | ||
| 1542 | q(emptyTopics), | ||
| 1543 | q(fixRefs), | ||
| 1544 | q(flattenFiles), | ||
| 1545 | q(guidHrefs), | ||
| 1546 | q(hrefUrlEncoding), | ||
| 1547 | q(ids), | ||
| 1548 | q(idTags), | ||
| 1549 | q(images), | ||
| 1550 | q(imagesReferencedFromTopics), | ||
| 1551 | q(ltgt), | ||
| 1552 | q(maxZoomOut), | ||
| 1553 | q(md5Sum), | ||
| 1554 | q(noHref), | ||
| 1555 | q(olBody), | ||
| 1556 | q(otherMeta), | ||
| 1557 | q(parseFailed), | ||
| 1558 | q(publicId), | ||
| 1559 | q(references), | ||
| 1560 | q(requiredCleanUp), | ||
| 1561 | q(tableDimensions), | ||
| 1562 | q(tagCount), | ||
| 1563 | q(tags), | ||
| 1564 | q(texts), | ||
| 1565 | q(title), | ||
| 1566 | q(topicIds), | ||
| 1567 | q(bookMapRefs), | ||
| 1568 | q(targetTopicToInputFiles), | ||
| 1569 | q(urls), | ||
| 1570 | q(validationErrors), | ||
| 1571 | q(vocabulary), | ||
| 1572 | q(xrefBadFormat), | ||
| 1573 | q(xrefBadScope), | ||
| 1574 | q(xRefs), | ||
| 1575 | ); | ||
| 1576 | |||
| 1577 | my $ip = awsCurrentIp; | ||
| 1578 | my @times; # Time for each merge | ||
| 1579 | |||
| 1580 | if (@x > 10) # Merge in parallel if there are enough items to merge - 10 tests Xref but allows final merge to occur in series | ||
| 1581 | {my $fields = @fields; | ||
| 1582 | my $q = newProcessStarter($xref->maximumNumberOfProcesses); # Process starter | ||
| 1583 | $q->processingTitle = q(Xref Analyze Merge on ip: $ip); | ||
| 1584 | $q->totalToBeStarted = $fields; | ||
| 1585 | |||
| 1586 | if (my $reports = $xref->reports) # Log file for merge | ||
| 1587 | {$q->processingLogFile = fpe($reports, qw(log xref analyzeMerge txt)); | ||
| 1588 | } | ||
| 1589 | |||
| 1590 | for my $field(@fields) # Merge hashes by file names which are unique - ffff | ||
| 1591 | {$q->start(sub | ||
| 1592 | {my $startTime = time; | ||
| 1593 | my $target = $xref->{$field} //= {}; # Field to be merged | ||
| 1594 | for my $x(@x) # Merge results from each file analyzed | ||
| 1595 | {if (my $xf = $x->{$field}) | ||
| 1596 | {for my $f(keys %$xf) # Each file analyzed | ||
| 1597 | {$target->{$f} = $xf->{$f} # Merge | ||
| 1598 | } | ||
| 1599 | } | ||
| 1600 | } | ||
| 1601 | [$field, $xref, time - $startTime] # Return results as a reference | ||
| 1602 | }); | ||
| 1603 | } | ||
| 1604 | my @merge = $q->finish; # Load results | ||
| 1605 | |||
| 1606 | for my $m(@merge) | ||
| 1607 | {my ($f, $x, $t) = @$m; | ||
| 1608 | $xref->{$f} = $x->{$f}; | ||
| 1609 | push @times, [$f, $t]; | ||
| 1610 | } | ||
| 1611 | } | ||
| 1612 | else # Merge in series | ||
| 1613 | {for my $field(@fields) # Merge hashes by file names which are unique - ffff | ||
| 1614 | {my $startTime = time; | ||
| 1615 | my $target = $xref->{$field} //= {}; # Field to be merged | ||
| 1616 | for my $x(@x) # Merge results from each file analyzed | ||
| 1617 | {if (my $xf = $x->{$field}) | ||
| 1618 | {for my $f(keys %$xf) # Each file analyzed | ||
| 1619 | {$target->{$f} = $xf->{$f} # Merge | ||
| 1620 | } | ||
| 1621 | } | ||
| 1622 | } | ||
| 1623 | push @times, [$field, time - $startTime] # Save merge times | ||
| 1624 | } | ||
| 1625 | } | ||
| 1626 | |||
| 1627 | formatTables($xref, [sort {$$b[1] <=> $$a[1]} @times], | ||
| 1628 | columns => <<END, | ||
| 1629 | Field Xref field merge | ||
| 1630 | Time Time in seconds to merge this field | ||
| 1631 | END | ||
| 1632 | title=>qq(Field merging elapsed times in descending order), | ||
| 1633 | head =>qq(Xref field merging took the following times on DDDD), | ||
| 1634 | file =>fpe(q(timing), qw(merges txt))); | ||
| 1635 | |||
| 1636 | for my $field( # Merge arrays | ||
| 1637 | qw(badTables)) | ||
| 1638 | {for my $x(@x) # mmmm Merge results from each file analyzed | ||
| 1639 | {next unless my $xf = $x->{$field}; | ||
| 1640 | push @{$xref->{$field}}, @$xf; | ||
| 1641 | } | ||
| 1642 | } | ||
| 1643 | |||
| 1644 | $xref | ||
| 1645 | } # analyzeOneFileResults | ||
| 1646 | |||
| 1647 | sub analyzeInputFiles($) #P Analyze the input files | ||
| 1648 | {my ($xref) = @_; # Cross referencer | ||
| 1649 | |||
| 1650 | awsParallelProcessFiles | ||
| 1651 | ($xref, | ||
| 1652 | \&analyzeOneFileParallel, | ||
| 1653 | \&analyzeOneFileResults, | ||
| 1654 | $xref->inputFiles); | ||
| 1655 | |||
| 1656 | } # analyzeInputFiles | ||
| 1657 | |||
| 1658 | # 63.128s Salesforce restructure on one instance | ||
| 1659 | |||
| 1660 | sub reportIdRefs($) #P Report the number of times each id is referenced | ||
| 1661 | {my ($xref) = @_; # Cross referencer | ||
| 1662 | |||
| 1663 | my %n; # Ids that might not be referenced | ||
| 1664 | if (my $xrefIds = $xref->ids) | ||
| 1665 | {for my $f(sort keys %{$xrefIds}) # Each input file | ||
| 1666 | {for my $i(sort keys %{$xrefIds->{$f}}) # Each id in each input file | ||
| 1667 | {$n{$f}{$i}++; # Ids that might not be referenced | ||
| 1668 | } | ||
| 1669 | } | ||
| 1670 | } | ||
| 1671 | |||
| 1672 | my %c; # Count of how often each id in each file was referenced | ||
| 1673 | if (my $xrefReferences = $xref->references) | ||
| 1674 | {for my $f(sort keys %{$xrefReferences}) # Each input file | ||
| 1675 | {for my $r(sort keys %{$xrefReferences->{$f}}) # Each reference from each input file | ||
| 1676 | {my ($rf, $rt, $ri) = parseDitaRef($r, $f); # Parse reference | ||
| 1677 | if ($ri) # If this is an id not a topic id | ||
| 1678 | { $c{$rf}{$ri} += $xrefReferences->{$f}{$r}; # Count id references | ||
| 1679 | delete $n{$rf}{$ri}; # Remove ids that get referenced | ||
| 1680 | } | ||
| 1681 | } | ||
| 1682 | } | ||
| 1683 | } | ||
| 1684 | |||
| 1685 | my @n; my $notTopicIds = 0; # Ids not referenced | ||
| 1686 | for my $f(sort keys %n) # Each input file | ||
| 1687 | {my $topicId = $xref->topicIds->{$f} // q(); | ||
| 1688 | for my $i(sort keys %{$n{$f}}) # Each unreferenced id in each input file | ||
| 1689 | {my $t = $i eq $topicId ? "**" : q(); # Show ids which are topic ids | ||
| 1690 | ++$notTopicIds unless $t; # Count non topic ids that are not referenced | ||
| 1691 | push @n, [$t, $i, $f]; # Report as topicId or not, id, file | ||
| 1692 | } | ||
| 1693 | } | ||
| 1694 | |||
| 1695 | my @c; | ||
| 1696 | for my $f(sort keys %c) # Each input file | ||
| 1697 | {for my $i(sort keys %{$c{$f}}) # Each referenced id in each input file | ||
| 1698 | {push @c, [$c{$f}{$i}, $i, $f]; # Report as count, id, file | ||
| 1699 | } | ||
| 1700 | } | ||
| 1701 | |||
| 1702 | my $facet = q(Ids referenced); | ||
| 1703 | |||
| 1704 | formatTables($xref, \@n, | ||
| 1705 | columns => <<END, | ||
| 1706 | TopicId If this id is a topic id then this column will have ** in it. | ||
| 1707 | Id The id that has not been referenced | ||
| 1708 | File The file containing the id that has not been referenced | ||
| 1709 | END | ||
| 1710 | title => qq(Ids that have not been referenced), | ||
| 1711 | facet => $facet, aspectColor=>q(red), | ||
| 1712 | head => <<END, | ||
| 1713 | Xref found $notTopicIds unreferenced ids on DDDD | ||
| 1714 | |||
| 1715 | The $notTopicIds ids below which are not marked with ** are not referenced by | ||
| 1716 | any other file in this corpus and thus should be considered for removal. | ||
| 1717 | |||
| 1718 | END | ||
| 1719 | summarize => 1, csv => 1, | ||
| 1720 | file => fpe(qw(bad ids_not_referenced txt))); | ||
| 1721 | |||
| 1722 | formatTables($xref, \@c, | ||
| 1723 | columns => <<END, | ||
| 1724 | Count The number of times this id is referenced | ||
| 1725 | Id The id in question | ||
| 1726 | File The file in which the id appears | ||
| 1727 | END | ||
| 1728 | title => qq(Ids referenced), | ||
| 1729 | facet => $facet, aspectColor=>q(green), | ||
| 1730 | head => <<END, | ||
| 1731 | Xref found NNNN ids are being referenced on DDDD | ||
| 1732 | |||
| 1733 | END | ||
| 1734 | summarize => 1, csv => 1, | ||
| 1735 | file => fpe(qw(good id_reference_counts txt))); | ||
| 1736 | |||
| 1737 | {idReferencedCount => \%c, # From multiverse to universe | ||
| 1738 | idNotReferenced => \%n, | ||
| 1739 | } | ||
| 1740 | } # reportIdRefs | ||
| 1741 | |||
| 1742 | sub removeUnusedIds($) #P Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference. This is a very conservative approach which acknowledges that writers might be looking for an id if they mention it in a reference. | ||
| 1743 | {my ($xref) = @_; # Cross referencer | ||
| 1744 | |||
| 1745 | my %keep; # {id}++ : id to be kept as it occurs in a reference some where | ||
| 1746 | if (my $xrefReferences = $xref->references) | ||
| 1747 | {for my $f(sort keys $xrefReferences->%*) # Each input file with references | ||
| 1748 | {for my $r(sort keys $xrefReferences->{$f}->%*) # Each id in each input file with references | ||
| 1749 | {my (undef, undef, $i) = parseDitaRef($r); # Ids to keep as they might be referenced now or soon | ||
| 1750 | $keep{$i}++ | ||
| 1751 | } | ||
| 1752 | } | ||
| 1753 | } | ||
| 1754 | |||
| 1755 | my %remove; # {file}{id}++ : id to be removed from a file | ||
| 1756 | my %ids; # {id}++ : id removed from all files as a consequence of removing it from each file | ||
| 1757 | if (my $xrefIds = $xref->ids) | ||
| 1758 | {for my $f(sort keys $xrefIds->%*) # Each input file | ||
| 1759 | {for my $i(sort keys $xrefIds->{$f}->%*) # Each id in each input file | ||
| 1760 | {next if $keep{$i}; # Id that can be deleted as it is never referenced in any form | ||
| 1761 | $remove{$f}{$i}++; | ||
| 1762 | $ids {$i}++ | ||
| 1763 | } | ||
| 1764 | } | ||
| 1765 | } | ||
| 1766 | |||
| 1767 | my $disposition = $xref->deleteUnusedIds ? # Disposition of deletable ids | ||
| 1768 | q(The deleteUnusedIds option was set on so these ids have been removed.) : | ||
| 1769 | q(The deleteUnusedIds option was set off so these ids have been retained.); | ||
| 1770 | |||
| 1771 | formatTables($xref, [map {[$_]} sort keys %ids], | ||
| 1772 | columns => <<END, | ||
| 1773 | Id An unreferenced id | ||
| 1774 | END | ||
| 1775 | title => qq(Ids removed), | ||
| 1776 | head => <<END, | ||
| 1777 | Xref found NNNN unreferenced ids on DDDD | ||
| 1778 | |||
| 1779 | $disposition | ||
| 1780 | END | ||
| 1781 | summarize => 1, | ||
| 1782 | file => fpe(qw(lists ids_removed txt))); | ||
| 1783 | |||
| 1784 | processFilesInParallel | ||
| 1785 | (sub | ||
| 1786 | {my ($f) = @_; # File with ids to be removed | ||
| 1787 | my %i = %{$remove{$f}}; # Ids to remove | ||
| 1788 | my @remove; # [id, file] : removed this id in this file | ||
| 1789 | if (my $x = Data::Edit::Xml::new($f)) # Parse file with ids to be removed | ||
| 1790 | {$x->by(sub | ||
| 1791 | {my ($o) = @_; | ||
| 1792 | if ($o != $x and my $i = $o->id) # Keep topic ids | ||
| 1793 | {if ($i{$i}) | ||
| 1794 | {$o->id = undef; | ||
| 1795 | push @remove, [$i, $o->tag, $f]; # Report id removed | ||
| 1796 | } | ||
| 1797 | } | ||
| 1798 | }); | ||
| 1799 | editXml($f, $f, -p $x) if @remove; | ||
| 1800 | } | ||
| 1801 | [@remove] | ||
| 1802 | }, | ||
| 1803 | |||
| 1804 | sub # Consolidate removal report | ||
| 1805 | {my (@removed) = @_; | ||
| 1806 | |||
| 1807 | my @r; | ||
| 1808 | for my $removed(@removed) # Consolidate each batch of removals | ||
| 1809 | {push @r, @$removed if $removed; | ||
| 1810 | } | ||
| 1811 | |||
| 1812 | formatTables($xref, \@r, | ||
| 1813 | columns => <<END, | ||
| 1814 | Id The id removed | ||
| 1815 | Tag The tag the removed id was on | ||
| 1816 | File The file in which the id appeared | ||
| 1817 | END | ||
| 1818 | title => qq(Ids removed from files), | ||
| 1819 | head => <<END, | ||
| 1820 | Xref removed NNNN unreferenced ids from files on DDDD | ||
| 1821 | |||
| 1822 | Ids are only removed if the id is not mentioned in any reference in the entire | ||
| 1823 | corpus regardless of what file the reference is to. This is to allow for the | ||
| 1824 | possibility of fixing possibly broken references in the future. | ||
| 1825 | |||
| 1826 | END | ||
| 1827 | summarize => 1, | ||
| 1828 | file => fpe(qw(lists ids_removed_from_files txt))); | ||
| 1829 | }, sort keys %remove) if $xref->deleteUnusedIds; # Remove unused ids unless except on the root element | ||
| 1830 | |||
| 1831 | {idsRemoved => \%ids, # Multiverse to universe | ||
| 1832 | } | ||
| 1833 | } # removeUnusedIds | ||
| 1834 | |||
| 1835 | sub reportEmptyTopics($) #P Report empty topics | ||
| 1836 | {my ($xref) = @_; # Cross referencer | ||
| 1837 | |||
| 1838 | my @E; # Ids that might not be referenced | ||
| 1839 | for my $f(sort keys %{$xref->emptyTopics}) # Each empty topic | ||
| 1840 | {my $d = $xref->docType->{$f}; | ||
| 1841 | my $t = $xref->title->{$f} // q(); | ||
| 1842 | push @E, [$d, $t, $f]; | ||
| 1843 | } | ||
| 1844 | |||
| 1845 | my @e = sort {$$a[1] cmp $$b[1]} @E; | ||
| 1846 | |||
| 1847 | formatTables($xref, \@e, | ||
| 1848 | columns => <<END, | ||
| 1849 | TopicType The type of topic thatis empty | ||
| 1850 | Title The title of the empty topic | ||
| 1851 | File The source file containing the empty topic | ||
| 1852 | END | ||
| 1853 | title => <<END, | ||
| 1854 | Empty Topics | ||
| 1855 | END | ||
| 1856 | head => <<END, | ||
| 1857 | Xref found NNNN empty topics on DDDD | ||
| 1858 | |||
| 1859 | END | ||
| 1860 | summarize => 1, | ||
| 1861 | csv => 1, | ||
| 1862 | file => fpe(qw(lists empty_topics txt))); | ||
| 1863 | |||
| 1864 | { # From multiverse to universe | ||
| 1865 | } | ||
| 1866 | } # reportEmptyTopics | ||
| 1867 | |||
| 1868 | sub reportDuplicateIds($) #P Report duplicate ids | ||
| 1869 | {my ($xref) = @_; # Cross referencer | ||
| 1870 | |||
| 1871 | my @dups; # Duplicate ids definitions | ||
| 1872 | my %dups; | ||
| 1873 | my %active; my %removed; # Active duplicate ids, removable ids | ||
| 1874 | if (my $xrefIds = $xref->ids) | ||
| 1875 | {for my $f(sort keys %$xrefIds) # Each input file | ||
| 1876 | {for my $i(sort keys $xrefIds->{$f}->%*) # Each id in the file | ||
| 1877 | {my $count = $xrefIds->{$f}{$i}; # Number of definitions of this id in the file | ||
| 1878 | if ($count > 1) # Duplicate definition | ||
| 1879 | {if ($xref->idsRemoved->{$i}) # Duplicated and being referred to | ||
| 1880 | {$removed{$i}++; | ||
| 1881 | } | ||
| 1882 | else | ||
| 1883 | {$active{$i}++; | ||
| 1884 | } | ||
| 1885 | push @dups, [$i, $count, $active{$i} ? q(**) : q(), $f]; # Save details of duplicate definition | ||
| 1886 | $dups{$f}{$i} = $count; | ||
| 1887 | } | ||
| 1888 | } | ||
| 1889 | } | ||
| 1890 | } | ||
| 1891 | |||
| 1892 | my $A = keys %active; # Number of possibly active duplicate ids | ||
| 1893 | my $R = keys %removed; # Number of ids removed | ||
| 1894 | |||
| 1895 | my $r = $xref->deleteUnusedIds ? <<END : # Explain active vs inactive ids | ||
| 1896 | $R unused ids are being deleted as they are not active (used in a reference | ||
| 1897 | regardless of the file component of the reference) and Xref was invoked with: | ||
| 1898 | |||
| 1899 | deleteUnusedIds => 1 | ||
| 1900 | |||
| 1901 | The $A active ids that cannot be removed are marked in this report with **. | ||
| 1902 | END | ||
| 1903 | <<END; | ||
| 1904 | $R unused ids could be deleted as they are not active (used in a reference | ||
| 1905 | regardless of the file component of the reference). To remove these ids from | ||
| 1906 | the entire corpus invoke Xref with: | ||
| 1907 | |||
| 1908 | deleteUnusedIds => 1 | ||
| 1909 | |||
| 1910 | The $A active ids that cannot be so removed are marked in this report with **. | ||
| 1911 | END | ||
| 1912 | |||
| 1913 | formatTables($xref, \@dups, | ||
| 1914 | columns => <<END, | ||
| 1915 | Id The id that has been duplicated | ||
| 1916 | Count The number of times the id was duplicated | ||
| 1917 | Active ** if this id is possibly being referred to from elsewhere in the corpus. | ||
| 1918 | File The file in which the duplication occurs | ||
| 1919 | END | ||
| 1920 | title=>qq(Duplicate id definitions within files), | ||
| 1921 | head=><<END, | ||
| 1922 | Xref found NNNN duplicate id definitions within files on DDDD | ||
| 1923 | |||
| 1924 | $A of these ids are possibly being referred to - see rows containing **. | ||
| 1925 | |||
| 1926 | Duplicate topic ids are reported in ../bad/topicIds.txt. | ||
| 1927 | |||
| 1928 | $r | ||
| 1929 | END | ||
| 1930 | summarize=>1, csv=>1, | ||
| 1931 | file=>(my $f = fpe(qw(bad duplicateIds txt)))); | ||
| 1932 | |||
| 1933 | {duplicateIds => \%dups, | ||
| 1934 | } # From multiverse to universe | ||
| 1935 | } # reportDuplicateIds | ||
| 1936 | |||
| 1937 | sub reportDuplicateTopicIds($) #P Report duplicate topic ids | ||
| 1938 | {my ($xref) = @_; # Cross referencer | ||
| 1939 | |||
| 1940 | my %dups; # Duplicate topic ids definitions | ||
| 1941 | my @dups; # Duplicate topic ids definitions report | ||
| 1942 | my @miss; # Missing topic id definitions report | ||
| 1943 | if (my $xrefTopicIds = $xref->topicIds) # Each input file | ||
| 1944 | {for my $file(sort keys %{$xrefTopicIds}) # Each input file | ||
| 1945 | {if (my $i = $xrefTopicIds->{$file}) # Topic Id | ||
| 1946 | {if (my $d = $dups{$i}) # Duplicate topic id | ||
| 1947 | {push @dups, [$i, $file, $d]; # Save details of duplicate definition | ||
| 1948 | } | ||
| 1949 | else | ||
| 1950 | {$dups{$i} = $file; # Save topic id | ||
| 1951 | } | ||
| 1952 | } | ||
| 1953 | elsif ($xref->docType->{$file} !~ m(map\Z)s) # Maps are not required to have topics ids | ||
| 1954 | {push @miss, [$file]; # Missing topic id | ||
| 1955 | } | ||
| 1956 | } | ||
| 1957 | } | ||
| 1958 | |||
| 1959 | my $dups = $xref->duplicateTopicIds = {map {$$_[0]=>$_} @dups}; # All duplicates | ||
| 1960 | my $miss = $xref->missingTopicIds = {map {$$_[0]=>$_} @miss}; # All missing | ||
| 1961 | |||
| 1962 | formatTables($xref, \@dups, columns => <<END, # Duplicate topic ids report | ||
| 1963 | TopicId The topic id that has been duplicated in File1 and File2 | ||
| 1964 | File1 The first file in which the duplicated topic id appears | ||
| 1965 | File2 The second file in which the duplicated topic id appears | ||
| 1966 | END | ||
| 1967 | title => qq(Duplicate topic id definitions), | ||
| 1968 | head => <<END, | ||
| 1969 | Xref found NNNN duplicate topic id definitions on DDDD | ||
| 1970 | END | ||
| 1971 | file=>(fpe(qw(bad duplicate_topics_ids txt)))); | ||
| 1972 | |||
| 1973 | formatTables($xref, \@miss, columns => <<END, # Missing topic ids report | ||
| 1974 | File A file containing a topic with no topic id | ||
| 1975 | END | ||
| 1976 | title=>qq(Topics without ids), | ||
| 1977 | head=><<END, | ||
| 1978 | Xref found NNNN topics that have no topic id on DDDD | ||
| 1979 | |||
| 1980 | END | ||
| 1981 | file=>(fpe(qw(bad topic_id_missing txt)))); | ||
| 1982 | |||
| 1983 | {duplicateTopicIds => $dups, | ||
| 1984 | missingTopicIds => $miss, | ||
| 1985 | } | ||
| 1986 | } # reportDuplicateTopicIds | ||
| 1987 | |||
| 1988 | sub reportNoHrefs($) #P Report locations where an href was expected but not found | ||
| 1989 | {my ($xref) = @_; # Cross referencer | ||
| 1990 | my @t; | ||
| 1991 | if (my $xrefNoHref = $xref->noHref) | ||
| 1992 | {for my $file(sort keys %{$xrefNoHref}) # Each input file | ||
| 1993 | {push @t, @{$xrefNoHref->{$file}}; # Missing href details | ||
| 1994 | } | ||
| 1995 | } | ||
| 1996 | |||
| 1997 | formatTables($xref, \@t, | ||
| 1998 | columns => <<END, | ||
| 1999 | Tag A tag that should have an xref. | ||
| 2000 | Location The location of the tag that should have an xref. | ||
| 2001 | File The source file containing the tag | ||
| 2002 | END | ||
| 2003 | title=>qq(Missing hrefs), | ||
| 2004 | head=><<END, | ||
| 2005 | Xref found NNNN tags that should have href attributes but did not on DDDD | ||
| 2006 | END | ||
| 2007 | file=>(fpe(qw(bad missing_href_attributes txt)))); | ||
| 2008 | {} # From multiverse to universe | ||
| 2009 | } # reportNoHrefs | ||
| 2010 | |||
| 2011 | sub checkReferences($) #P Check each reference, report bad references and mark them for fixing. | ||
| 2012 | {my ($xref) = @_; # Cross referencer | ||
| 2013 | |||
| 2014 | my @bad; # Bad references | ||
| 2015 | my @good; # Good references | ||
| 2016 | |||
| 2017 | if (my $xrefReferences = $xref->references) | ||
| 2018 | {for my $file(sort keys %$xrefReferences) # Each input file which will be absolute | ||
| 2019 | {for my $ref (sort keys $xrefReferences->{$file}->%*) # Each href in the file which will be relative | ||
| 2020 | {if (my $r = &oneBadRef($xref, $file, $ref)) # Check reference | ||
| 2021 | {push @bad, $r; | ||
| 2022 | $xref->fixRefs->{$file}{$ref}++; # Request fix attempt for this reference | ||
| 2023 | $xref->badReferencesCount++; # Number of bad references encountered | ||
| 2024 | } | ||
| 2025 | else # Good references | ||
| 2026 | {push @good, [$ref, $file] | ||
| 2027 | } | ||
| 2028 | } | ||
| 2029 | } | ||
| 2030 | } | ||
| 2031 | |||
| 2032 | my $facet = q(References between topics at start); | ||
| 2033 | |||
| 2034 | formatTables($xref, \@bad, # Report the failing references | ||
| 2035 | columns => <<END, | ||
| 2036 | Reason The reason why the reference failed to resolve | ||
| 2037 | Reference The href in the source file | ||
| 2038 | Ref_File The file containing the referenced topic relative to the referencing file | ||
| 2039 | Ref_TopicId The topic id of the referenced topic | ||
| 2040 | Ref_Id The id of the statement in the referenced topic | ||
| 2041 | Source_TopicId The topic id of the referencing file | ||
| 2042 | Target_TopicId The topic id of the referenced file | ||
| 2043 | Source_File The referencing source file | ||
| 2044 | Target_File The referenced target file | ||
| 2045 | END | ||
| 2046 | title => qq(Bad references at start), | ||
| 2047 | facet => $facet, aspectColor=>q(red), | ||
| 2048 | head => <<END, | ||
| 2049 | Xref found NNNN bad references on DDDD at the start of processing | ||
| 2050 | |||
| 2051 | Depending on the options chosen, Xref might fix or ameliorate these references | ||
| 2052 | so that the actual number of references with problems is lower than this. The | ||
| 2053 | best way to find out how many references still need fixing after doing an Xref | ||
| 2054 | run with fixing options enabled is to do another run with all of these options | ||
| 2055 | disabled. The braver elements will be asking why we do not account for these | ||
| 2056 | improvements in flight - that would significantly increase code complexity | ||
| 2057 | while taking a lot of effort to validate yet produce no significant | ||
| 2058 | improvements - so it is much easier not to when a second run suffices. | ||
| 2059 | END | ||
| 2060 | csv => 1, wide =>1, summarize=>1, | ||
| 2061 | file => fpe(q(bad), q(references), q(txt))); | ||
| 2062 | |||
| 2063 | formatTables($xref, \@good, # Report good references | ||
| 2064 | columns => <<END, | ||
| 2065 | Href A good href attribute | ||
| 2066 | Source_File The referencing source file | ||
| 2067 | END | ||
| 2068 | title => qq(Good references at start), | ||
| 2069 | facet => $facet, aspectColor=>q(green), | ||
| 2070 | head => <<END, | ||
| 2071 | Xref found NNNN good references on DDDD at the start of processing. | ||
| 2072 | END | ||
| 2073 | csv => 1, wide =>1, summarize=>1, | ||
| 2074 | file => fpe(q(good), q(references), q(txt))); | ||
| 2075 | # {} # From multiverse to universe 2019.08.29 - no need - we are in series | ||
| 2076 | } # checkReferences | ||
| 2077 | |||
| 2078 | sub reportGuidHrefs($) #P Report on guid hrefs | ||
| 2079 | {my ($xref) = @_; # Cross referencer | ||
| 2080 | |||
| 2081 | my %guidToFile; # Map guids to files | ||
| 2082 | if (my $xrefTopicIds = $xref->topicIds) | ||
| 2083 | {for my $file(sort keys %{$xrefTopicIds}) # Each input file containing a topic id | ||
| 2084 | {my $id = $xrefTopicIds->{$file}; # Each href in the file which will start with guid | ||
| 2085 | next unless defined $id; | ||
| 2086 | next unless $id =~ m(\bguid-)is; # Check guid appears somewhere in href | ||
| 2087 | $guidToFile{$id} = $file; # We report duplicates in reportDuplicateTopicIds | ||
| 2088 | } | ||
| 2089 | } | ||
| 2090 | |||
| 2091 | my @bad; my @good; # Good and bad guid hrefs | ||
| 2092 | for my $file(sort keys %{$xref->guidHrefs}) # Each input file which will be absolute | ||
| 2093 | {my $sourceTopicId = $xref->topicIds->{$file}; | ||
| 2094 | for my $href(sort keys %{$xref->guidHrefs->{$file}}) # Each href in the file which will start with guid | ||
| 2095 | {my ($tag, $lineLocation) = @{$xref->guidHrefs->{$file}{$href}}; # Tag of node and location in source file of node doing the referencing | ||
| 2096 | # 2019.08.29 The following line does not appear to be needed - it is happening to late to affect anything | ||
| 2097 | $xref->fixRefs->{$file}{$href}++ unless $xref->fixRefs->{$file}{$href}; # Avoid double counting - all guid hrefs will be fixed if we are fixing hrefs as both good and bad will fail. | ||
| 2098 | |||
| 2099 | if ($href =~ m(#)) # Href with # | ||
| 2100 | {my ($guid, $topic, $id) = split m(#|\/), $href, 3; # Guid, topic, remainder | ||
| 2101 | my $targetFile = $guidToFile{$guid}; # Locate file defining guid | ||
| 2102 | |||
| 2103 | if (!defined $targetFile) # No definition of this guid | ||
| 2104 | {push @bad, # Report missing guid | ||
| 2105 | ["No such guid defined", $tag, $href, $lineLocation, q(), | ||
| 2106 | $sourceTopicId, $targetFile, $file]; | ||
| 2107 | next; | ||
| 2108 | } | ||
| 2109 | |||
| 2110 | my $targetFileId = $xref->topicIds->{$targetFile} // ''; # Actual id in target file | ||
| 2111 | |||
| 2112 | my $bad = sub | ||
| 2113 | {push @bad, | ||
| 2114 | [@_, $tag, $href, $lineLocation, $targetFileId, $sourceTopicId, | ||
| 2115 | $targetFile, $file]; | ||
| 2116 | }; | ||
| 2117 | |||
| 2118 | my $good = sub | ||
| 2119 | {push @good, | ||
| 2120 | [$href, $tag, $lineLocation, $targetFile, $file]; | ||
| 2121 | }; | ||
| 2122 | |||
| 2123 | if (!-e $targetFile) # Existence of file | ||
| 2124 | {$bad->(q(No such file)); | ||
| 2125 | } | ||
| 2126 | elsif (defined $topic) # Topic defined so it must be an xref | ||
| 2127 | {if ($topic ne $guid) | ||
| 2128 | {$bad->(q(Guid does not match topic id)); | ||
| 2129 | } | ||
| 2130 | elsif (defined $id) | ||
| 2131 | {if (my $i = $xref->ids->{$targetFile}{$id}) # Check id exists in target file | ||
| 2132 | {if ($i == 1) | ||
| 2133 | {&$good; | ||
| 2134 | } | ||
| 2135 | else | ||
| 2136 | {$bad->(q(Duplicate id in topic)); | ||
| 2137 | } | ||
| 2138 | } | ||
| 2139 | $bad->(q(No such id in topic)); | ||
| 2140 | } | ||
| 2141 | else | ||
| 2142 | {&$good; | ||
| 2143 | } | ||
| 2144 | } | ||
| 2145 | else | ||
| 2146 | {&$good; | ||
| 2147 | } | ||
| 2148 | } | ||
| 2149 | elsif ($tag eq q(image)) # Image reference | ||
| 2150 | {my $guid = $href =~ s(guid|-) ()igsr; | ||
| 2151 | if (my $image = $xref->inputFolderImages->{$guid}) | ||
| 2152 | {push @good, [$tag, $href, $lineLocation, $image, $file]; | ||
| 2153 | } | ||
| 2154 | else | ||
| 2155 | {push @bad, [qq(No such image guid defined), $tag, $href, | ||
| 2156 | $lineLocation, q(), $sourceTopicId, q(), $file]; | ||
| 2157 | } | ||
| 2158 | } | ||
| 2159 | else # No # in href and not an image so it must be a bookmap element | ||
| 2160 | {my $targetFile = $guidToFile{$href}; | ||
| 2161 | if (!defined $targetFile) # No such guid | ||
| 2162 | {push @bad, [qq(No such guid defined), $tag, $href, | ||
| 2163 | $lineLocation, q(), $sourceTopicId, q(), $file]; | ||
| 2164 | } | ||
| 2165 | elsif (!-e $targetFile) # Actual file name | ||
| 2166 | {push @bad, my $p = [qq(No such file), $tag, $href, | ||
| 2167 | $lineLocation, q(), $sourceTopicId, $targetFile, $file]; | ||
| 2168 | } | ||
| 2169 | elsif ($xref->fixBadRefs) # The file exists and we want to fix such references | ||
| 2170 | {# 2019.08.29 the following line happens too late to be useful | ||
| 2171 | $xref->fixRefs->{$file}{$href}++; | ||
| 2172 | } | ||
| 2173 | else | ||
| 2174 | {push @good, [$tag, $href, $lineLocation, $targetFile, $file]; | ||
| 2175 | } | ||
| 2176 | } | ||
| 2177 | } | ||
| 2178 | } | ||
| 2179 | |||
| 2180 | $xref->badGuidHrefs = {map {$$_[7]=>$_} @bad}; # Bad references | ||
| 2181 | |||
| 2182 | my $in = $xref->inputFolder//''; | ||
| 2183 | formatTables($xref, \@bad, | ||
| 2184 | columns => <<END, | ||
| 2185 | Reason The reason why the href failed to resolve | ||
| 2186 | Tag The tag of the node doing the referencing | ||
| 2187 | Href The href of the node doing the referencing | ||
| 2188 | Line_Location The line location where the href occurred in the source file | ||
| 2189 | Target_Topic_Id The actual id of the topic in the target file | ||
| 2190 | Source_Topic_Id The topic id in the source file | ||
| 2191 | Target_File The target file | ||
| 2192 | Source_file The source file containing the reference | ||
| 2193 | END | ||
| 2194 | title =>qq(Unresolved GUID hrefs), | ||
| 2195 | head =>qq(Xref found NNNN unresolved GUID hrefs on DDDD), | ||
| 2196 | summarize=>1, | ||
| 2197 | wide =>1, | ||
| 2198 | file =>(fpe(q(bad), qw(guidHrefs txt)))); | ||
| 2199 | |||
| 2200 | formatTables($xref, \@good, | ||
| 2201 | columns => <<END, | ||
| 2202 | Tag The tag containing the href | ||
| 2203 | Href The href of the node doing the referencing | ||
| 2204 | Line_Location The line location where the href occurred in the source file | ||
| 2205 | Source_File The source file containing the reference | ||
| 2206 | Target_File The target file | ||
| 2207 | END | ||
| 2208 | title =>qq(Resolved GUID hrefs), | ||
| 2209 | head =>qq(Xref found NNNN Resolved GUID hrefs on DDDD), | ||
| 2210 | file =>(fpe(q(good), qw(guidHrefs txt)))); | ||
| 2211 | |||
| 2212 | {badGuidHrefs => $xref->badGuidHrefs, # From multiverse to universe | ||
| 2213 | } | ||
| 2214 | } # reportGuidHrefs | ||
| 2215 | |||
| 2216 | sub reportImages($) #P Reports on images and references to images | ||
| 2217 | {my ($xref) = @_; # Cross referencer | ||
| 2218 | |||
| 2219 | my %bad; # Missing images | ||
| 2220 | for my $fail($xref->fixedRefsBad->@*) # References that failed | ||
| 2221 | {my ($reason, $type, $attr, $ref, $file) = @$fail; | ||
| 2222 | next unless $type =~ m(\Aimage\Z)is; | ||
| 2223 | my $i = absFromAbsPlusRel($file, $ref); | ||
| 2224 | $bad{$i}++; | ||
| 2225 | } | ||
| 2226 | my @bad = map {[$bad{$_}, $_]} sort keys %bad; | ||
| 2227 | |||
| 2228 | my $facet = q(Image files); | ||
| 2229 | |||
| 2230 | formatTables($xref, \@bad, # Report missing images references | ||
| 2231 | columns => <<END, | ||
| 2232 | Count The number of unresolved references to this missing image | ||
| 2233 | Image_File The file name of the missing image | ||
| 2234 | END | ||
| 2235 | title => qq(Missing images), | ||
| 2236 | facet => $facet, aspectColor => qw(red), | ||
| 2237 | head => qq(Xref found NNNN missing images on DDDD), | ||
| 2238 | summarize => 1, zero => 1, | ||
| 2239 | file => fpe(qw(bad missing_images txt))); | ||
| 2240 | |||
| 2241 | |||
| 2242 | my %good; # Good images | ||
| 2243 | for my $topic(sort keys $xref->images->%*) # Topics containing images | ||
| 2244 | {for my $ref (sort keys $xref->images->{$topic}->%*) # Image references | ||
| 2245 | {my $i = absFromAbsPlusRel($topic, $ref); | ||
| 2246 | if (!$bad{$i}) | ||
| 2247 | {$good{$i}++; | ||
| 2248 | } | ||
| 2249 | } | ||
| 2250 | } | ||
| 2251 | |||
| 2252 | my @good = map {[$good{$_}, $_]} sort keys %good; # Report good images | ||
| 2253 | |||
| 2254 | formatTables($xref, \@good, | ||
| 2255 | columns => <<END, | ||
| 2256 | Count The number of references to this image | ||
| 2257 | Image_File The file name of the image | ||
| 2258 | END | ||
| 2259 | title => qq(Image files), | ||
| 2260 | facet => $facet, aspectColor => qw(green), | ||
| 2261 | head=>qq(Xref found NNNN image files on DDDD), | ||
| 2262 | summarize => 1, zero => 1, | ||
| 2263 | file=>(fpe(qw(good image_files txt)))); | ||
| 2264 | |||
| 2265 | {missingImageFiles => \%bad, | ||
| 2266 | goodImageFiles => \%good} | ||
| 2267 | |||
| 2268 | } # reportImages | ||
| 2269 | |||
| 2270 | sub reportParseFailed($) #P Report failed parses | ||
| 2271 | {my ($xref) = @_; # Cross referencer | ||
| 2272 | |||
| 2273 | formatTables($xref, | ||
| 2274 | hashOfCountsToArray($xref->parseFailed), | ||
| 2275 | columns => <<END, | ||
| 2276 | Source The file that failed to parse as an absolute file path | ||
| 2277 | END | ||
| 2278 | title=>qq(Files failed to parse), | ||
| 2279 | head=>qq(Xref found NNNN files failed to parse on DDDD), | ||
| 2280 | file=>(my $f = fpe(qw(bad parseFailed txt)))); | ||
| 2281 | {} # From multiverse to universe | ||
| 2282 | } # reportParseFailed | ||
| 2283 | |||
| 2284 | sub reportXml1($) #P Report bad xml on line 1 | ||
| 2285 | {my ($xref) = @_; # Cross referencer | ||
| 2286 | |||
| 2287 | formatTables($xref, [map {[$_]} sort keys %{$xref->badXml1}], | ||
| 2288 | columns => <<END, | ||
| 2289 | Source The source file containing bad xml on line | ||
| 2290 | END | ||
| 2291 | title=>qq(Bad Xml line 1), | ||
| 2292 | head=>qq(Xref found NNNN Files with the incorrect xml on line 1 on DDDD), | ||
| 2293 | file=>(my $f = fpe(qw(bad xmlLine1 txt)))); | ||
| 2294 | {} # From multiverse to universe | ||
| 2295 | } # reportXml1 | ||
| 2296 | |||
| 2297 | sub reportXml2($) #P Report bad xml on line 2 | ||
| 2298 | {my ($xref) = @_; # Cross referencer | ||
| 2299 | |||
| 2300 | formatTables($xref, [map {[$_]} sort keys %{$xref->badXml2}], | ||
| 2301 | columns => <<END, | ||
| 2302 | Source The source file containing bad xml on line | ||
| 2303 | END | ||
| 2304 | title=>qq(Bad Xml line 2), | ||
| 2305 | head=>qq(Xref found NNNN Files with the incorrect xml on line 2 on DDDD), | ||
| 2306 | file=>(my $f = fpe(qw(bad xmlLine2 txt)))); | ||
| 2307 | {} # From multiverse to universe | ||
| 2308 | } # reportXml2 | ||
| 2309 | |||
| 2310 | sub reportDocTypeCount($) #P Report doc type count | ||
| 2311 | {my ($xref) = @_; # Cross referencer | ||
| 2312 | |||
| 2313 | my %d; | ||
| 2314 | if (my $xrefDocType = $xref->docType) | ||
| 2315 | {for my $f(sort keys %{$xrefDocType}) | ||
| 2316 | {my $d = $xrefDocType->{$f}; | ||
| 2317 | $d{$d}++ | ||
| 2318 | } | ||
| 2319 | } | ||
| 2320 | |||
| 2321 | formatTables($xref, | ||
| 2322 | hashOfCountsToArray(\%d), | ||
| 2323 | columns => <<END, | ||
| 2324 | DocType The root element tag of this document | ||
| 2325 | END | ||
| 2326 | title=>qq(Document types), | ||
| 2327 | head=>qq(Xref found NNNN different doc types on DDDD), | ||
| 2328 | file=>(fpe(qw(count docTypes txt)))); | ||
| 2329 | {} # From multiverse to universe | ||
| 2330 | } # reportDocTypeCount | ||
| 2331 | |||
| 2332 | sub reportTagCount($) #P Report tag counts | ||
| 2333 | {my ($xref) = @_; # Cross referencer | ||
| 2334 | |||
| 2335 | my %d; | ||
| 2336 | if (my $xrefTagCount = $xref->tagCount) | ||
| 2337 | {for my $f(sort keys %{$xrefTagCount}) | ||
| 2338 | {for my $t(sort keys %{$xrefTagCount->{$f}}) | ||
| 2339 | {my $d = $xrefTagCount->{$f}{$t}; | ||
| 2340 | $d{$t} += $d; | ||
| 2341 | } | ||
| 2342 | } | ||
| 2343 | } | ||
| 2344 | |||
| 2345 | formatTables($xref, hashOfCountsToArray(\%d), | ||
| 2346 | columns => <<END, | ||
| 2347 | Tag The tag name of the xml node | ||
| 2348 | Count The number of times this tag name occurs | ||
| 2349 | END | ||
| 2350 | title=>qq(Tags), | ||
| 2351 | head=>qq(Xref found NNNN different tags on DDDD), | ||
| 2352 | file=>(fpe(qw(count tags txt)))); | ||
| 2353 | {} # From multiverse to universe | ||
| 2354 | } # reportTagCount | ||
| 2355 | |||
| 2356 | sub reportTagsAndTextsCount($) #P Report tags and texts counts | ||
| 2357 | {my ($xref) = @_; # Cross referencer | ||
| 2358 | |||
| 2359 | my $tags = 1; $tags += $xref->tags ->{$_}||0 for keys %{$xref->tags}; | ||
| 2360 | my $texts = 1; $texts += $xref->texts->{$_}||0 for keys %{$xref->texts}; | ||
| 2361 | |||
| 2362 | my @t; | ||
| 2363 | push @t, [q(Tags), $tags]; | ||
| 2364 | push @t, [q(Texts), $texts]; | ||
| 2365 | my $ratio = $xref->tagsTextsRatio = $tags/$texts; | ||
| 2366 | push @t, [q(Tags to Texts), sprintf("%7.4f", $ratio)]; | ||
| 2367 | |||
| 2368 | |||
| 2369 | formatTables($xref, \@t, | ||
| 2370 | columns => <<END, | ||
| 2371 | Item A tag or some text | ||
| 2372 | Count The number of times the tag or text occurs | ||
| 2373 | END | ||
| 2374 | title => q(Tags to Texts Ratio), | ||
| 2375 | head => q(Xref found the following tag and text counts on DDDD), | ||
| 2376 | file => (fpe(qw(count tagsAndTexts txt)))); | ||
| 2377 | |||
| 2378 | {tagsTextsRatio => $xref->tagsTextsRatio, # From multiverse to universe | ||
| 2379 | } | ||
| 2380 | } # reportTagsAndTextsCount | ||
| 2381 | |||
| 2382 | sub reportLtGt($) #P Report items found between < and > | ||
| 2383 | {my ($xref) = @_; # Cross referencer | ||
| 2384 | |||
| 2385 | my %d; | ||
| 2386 | if (my $xrefLtgt = $xref->ltgt) | ||
| 2387 | {for my $f(sort keys %{$xrefLtgt}) | ||
| 2388 | {for my $t(sort keys %{$xrefLtgt->{$f}}) | ||
| 2389 | {$d{$t} += $xrefLtgt->{$f}{$t}; | ||
| 2390 | } | ||
| 2391 | } | ||
| 2392 | } | ||
| 2393 | |||
| 2394 | formatTables($xref, [map {[$d{$_}, nws($_)]} sort keys %d], | ||
| 2395 | columns => <<END, | ||
| 2396 | Count The number of times this text was found | ||
| 2397 | Text The text found between < and >. The white space has been normalized to make better use of the display. | ||
| 2398 | END | ||
| 2399 | title=>qq(Text found between < and >), | ||
| 2400 | head=><<END, | ||
| 2401 | Xref found NNNN different text items between < and > on DDDD | ||
| 2402 | END | ||
| 2403 | file=>(fpe(qw(count ltgt txt)))); | ||
| 2404 | {} # From multiverse to universe | ||
| 2405 | } # reportLtGt | ||
| 2406 | |||
| 2407 | sub reportAttributeCount($) #P Report attribute counts | ||
| 2408 | {my ($xref) = @_; # Cross referencer | ||
| 2409 | |||
| 2410 | my %d; | ||
| 2411 | if (my $xrefAttributeCount = $xref->attributeCount) | ||
| 2412 | {for my $f(sort keys %{$xrefAttributeCount}) | ||
| 2413 | {for my $t(sort keys %{$xrefAttributeCount->{$f}}) | ||
| 2414 | {my $d = $xrefAttributeCount->{$f}{$t}; | ||
| 2415 | $d{$t} += $d; | ||
| 2416 | } | ||
| 2417 | } | ||
| 2418 | } | ||
| 2419 | |||
| 2420 | my @c; | ||
| 2421 | for my $t(sort keys %d) | ||
| 2422 | {push @c, [$d{$t}, $t]; | ||
| 2423 | } | ||
| 2424 | |||
| 2425 | formatTables($xref, [@c], # Attribute count report | ||
| 2426 | columns => <<END, | ||
| 2427 | Count The number of times this attribute appears | ||
| 2428 | Attribute The attribute name being counted | ||
| 2429 | END | ||
| 2430 | title=>qq(Attributes), | ||
| 2431 | head=>qq(Xref found NNNN different attributes on DDDD), | ||
| 2432 | file=>(my $f = fpe(qw(count attributes txt)))); | ||
| 2433 | {} # From multiverse to universe | ||
| 2434 | } # reportAttributeCount | ||
| 2435 | |||
| 2436 | sub reportAttributeNameAndValueCounts($) #P Report attribute value counts | ||
| 2437 | {my ($xref) = @_; # Cross referencer | ||
| 2438 | |||
| 2439 | my %d; | ||
| 2440 | if (my $xrefAttributeNamesAndValuesCount = $xref->attributeNamesAndValuesCount) | ||
| 2441 | {for my $f(sort keys %{$xrefAttributeNamesAndValuesCount}) | ||
| 2442 | {for my $a(sort keys %{$xrefAttributeNamesAndValuesCount->{$f}}) | ||
| 2443 | {for my $v(sort keys %{$xrefAttributeNamesAndValuesCount->{$f}{$a}}) | ||
| 2444 | {my $c = $xrefAttributeNamesAndValuesCount->{$f}{$a}{$v}; | ||
| 2445 | $d{$a}{$v} += $c; | ||
| 2446 | } | ||
| 2447 | } | ||
| 2448 | } | ||
| 2449 | } | ||
| 2450 | |||
| 2451 | my @D; | ||
| 2452 | for my $a(sort keys %d) | ||
| 2453 | {for my $v(sort keys %{$d{$a}}) | ||
| 2454 | {push @D, [$d{$a}{$v}, firstNChars($v, 128), $a]; # Otherwise the report can get surprisingly wide | ||
| 2455 | } | ||
| 2456 | } | ||
| 2457 | |||
| 2458 | my @d = sort {$$a[2] cmp $$b[2]} | ||
| 2459 | sort {$$b[0] <=> $$a[0]} @D; | ||
| 2460 | |||
| 2461 | |||
| 2462 | formatTables($xref, \@d, | ||
| 2463 | columns => <<END, | ||
| 2464 | Count The number of times this value occurs | ||
| 2465 | Value The value being counted | ||
| 2466 | Attribute The attribute on which the value appears | ||
| 2467 | END | ||
| 2468 | summarize => 1, | ||
| 2469 | title => qq(Attribute value counts), | ||
| 2470 | head => qq(Xref found NNNN attribute value combinations on DDDD), | ||
| 2471 | file => (fpe(qw(count attributeNamesAndValues txt)))); | ||
| 2472 | {} # From multiverse to universe | ||
| 2473 | } # reportAttributeNameAndValueCounts | ||
| 2474 | |||
| 2475 | sub reportValidationErrors($) #P Report the files known to have validation errors | ||
| 2476 | {my ($xref) = @_; # Cross referencer | ||
| 2477 | |||
| 2478 | my $e = $xref->validationErrors; | ||
| 2479 | |||
| 2480 | formatTables($xref, [map {[$$e{$_}, $_]} sort keys %$e], # | ||
| 2481 | columns => <<END, | ||
| 2482 | Count The number of valiation errors in the file | ||
| 2483 | File A file with xml validation errors | ||
| 2484 | END | ||
| 2485 | title=>qq(Topics with xml validation errors), | ||
| 2486 | head=><<END, | ||
| 2487 | Xref found NNNN topics with xml validation errors on DDDD | ||
| 2488 | END | ||
| 2489 | file=>(fpe(qw(bad validationErrors txt)))); | ||
| 2490 | {} # From multiverse to universe | ||
| 2491 | } # reportValidationErrors | ||
| 2492 | |||
| 2493 | sub reportTables($) #P Report on tables that have problems | ||
| 2494 | {my ($xref) = @_; # Cross referencer | ||
| 2495 | |||
| 2496 | formatTables($xref, $xref->badTables, | ||
| 2497 | columns => <<END, | ||
| 2498 | Reason Reason bookmap failed | ||
| 2499 | Attributes The tag and attributes of the table element in question | ||
| 2500 | Location The location at which the error was detected | ||
| 2501 | Source_File The file in which the error was detected | ||
| 2502 | END | ||
| 2503 | title=>qq(Tables with errors), | ||
| 2504 | head=><<END, | ||
| 2505 | Xref found NNNN table errors on DDDD | ||
| 2506 | END | ||
| 2507 | summarize=>1, | ||
| 2508 | file=>(fpe(qw(bad tables txt)))); | ||
| 2509 | {} # From multiverse to universe | ||
| 2510 | } # reportTables | ||
| 2511 | |||
| 2512 | sub reportFileExtensionCount($) #P Report file extension counts | ||
| 2513 | {my ($xref) = @_; # Cross referencer | ||
| 2514 | |||
| 2515 | my $folder = $xref->inputFolder; | ||
| 2516 | formatTables($xref, | ||
| 2517 | hashOfCountsToArray(countFileExtensions($folder)), | ||
| 2518 | columns => <<END, | ||
| 2519 | Ext An extension that appears in the input corpus | ||
| 2520 | Count The number of times the extension appears in the corpus. | ||
| 2521 | END | ||
| 2522 | title=>qq(File extensions), | ||
| 2523 | head=><<END, | ||
| 2524 | Xref found NNNN different file extensions on DDDD in folder: $folder | ||
| 2525 | END | ||
| 2526 | file=>(fpe(qw(count fileExtensions txt)))); | ||
| 2527 | {} # From multiverse to universe | ||
| 2528 | } # reportFileExtensionCount | ||
| 2529 | |||
| 2530 | sub reportFileTypes($) #P Report file type counts - takes too long in series | ||
| 2531 | {my ($xref) = @_; # Cross referencer | ||
| 2532 | |||
| 2533 | my $ft = countFileTypes($xref->inputFolder, $xref->maximumNumberOfProcesses); # Count file types | ||
| 2534 | |||
| 2535 | formatTables($xref, | ||
| 2536 | hashOfCountsToArray($ft), | ||
| 2537 | columns=><<END, | ||
| 2538 | Type A file type that appears in the input corpus | ||
| 2539 | Count The number of times the file type appears in the corpus. | ||
| 2540 | END | ||
| 2541 | title=>qq(Files types), | ||
| 2542 | head=><<END, | ||
| 2543 | Xref found NNNN different file types on DDDD | ||
| 2544 | END | ||
| 2545 | file=>(my $f = fpe(qw(count fileTypes txt)))); | ||
| 2546 | {} # From multiverse to universe | ||
| 2547 | } # reportFileTypes | ||
| 2548 | |||
| 2549 | sub reportExternalXrefs($) #P Report external xrefs missing other attributes | ||
| 2550 | {my ($xref) = @_; # Cross referencer | ||
| 2551 | |||
| 2552 | my @s; | ||
| 2553 | if (my $xrefXrefBadScope = $xref->xrefBadScope) | ||
| 2554 | {for my $f(sort keys %{$xrefXrefBadScope}) | ||
| 2555 | {my $sourceTopicId = $xref->topicIds->{$f}; | ||
| 2556 | for my $h(sort keys %{$xrefXrefBadScope->{$f}}) | ||
| 2557 | {my $s = $xrefXrefBadScope->{$f}{$h}; | ||
| 2558 | push @s, [q(Bad scope attribute), $h, $s, $sourceTopicId, $f]; | ||
| 2559 | } | ||
| 2560 | } | ||
| 2561 | } | ||
| 2562 | |||
| 2563 | if (my $xrefXrefBadFormat = $xref->xrefBadFormat) | ||
| 2564 | {for my $f(sort keys %{$xrefXrefBadFormat}) | ||
| 2565 | {my $sourceTopicId = $xref->topicIds->{$f}; | ||
| 2566 | for my $h(sort keys %{$xrefXrefBadFormat->{$f}}) | ||
| 2567 | {my $s = $xrefXrefBadFormat->{$f}{$h}; | ||
| 2568 | push @s, [q(Bad format attribute), $h, $s, $sourceTopicId, $f]; | ||
| 2569 | } | ||
| 2570 | } | ||
| 2571 | } | ||
| 2572 | |||
| 2573 | formatTables($xref, \@s, | ||
| 2574 | columns => <<END, | ||
| 2575 | Reason The reason why the xref is unsatisfactory | ||
| 2576 | Href The href attribute of the xref in question | ||
| 2577 | Xref_Statement The xref statement in question | ||
| 2578 | Source_Topic_Id The topic id of the source file containing file containing the bad external xref | ||
| 2579 | File The file containing the xref statement in question | ||
| 2580 | END | ||
| 2581 | title=>qq(Bad external xrefs), | ||
| 2582 | head=>qq(Xref found bad external xrefs on DDDD), | ||
| 2583 | file=>(my $f = fpe(qw(bad externalXrefs txt)))); | ||
| 2584 | {} # From multiverse to universe | ||
| 2585 | } # reportExternalXrefs | ||
| 2586 | |||
| 2587 | sub reportMaxZoomOut($) #P Text located via Max Zoom In | ||
| 2588 | {my ($xref) = @_; # Cross referencer | ||
| 2589 | return {} unless my $names = $xref->maxZoomIn; # No point if maxZoomIn was not specified | ||
| 2590 | |||
| 2591 | my @names = (qw(File_Name Title), sort keys %$names); # Column Headers | ||
| 2592 | my $columns = join "\n", @names; # Column Headers as lines suitable for format tables | ||
| 2593 | my %names = map {$names[$_]=>$_} keys @names; # Assign regular expression names to columns in the output table/csv | ||
| 2594 | |||
| 2595 | my @f; | ||
| 2596 | if (my $xrefMaxZoomOut = $xref->maxZoomOut // {}) | ||
| 2597 | {for my $f(sort keys %$xrefMaxZoomOut) # One row per file processed showing which regular expression names matched | ||
| 2598 | {my @n = ($f, $xref->title->{$f}); | ||
| 2599 | my $c = 0; | ||
| 2600 | for my $n(sort keys %{$xrefMaxZoomOut->{$f}}) | ||
| 2601 | {$n[$names{$n}] += $xrefMaxZoomOut->{$f}{$n}; | ||
| 2602 | ++$c; | ||
| 2603 | } | ||
| 2604 | push @f, [@n] if $c; # Only save a row if it has something in it | ||
| 2605 | } | ||
| 2606 | } | ||
| 2607 | |||
| 2608 | if (my $xrefMaxZoomOut = $xref->maxZoomOut // {}) | ||
| 2609 | {for my $f(sort keys %$xrefMaxZoomOut) | ||
| 2610 | {my $t = $xref->title->{$f}; | ||
| 2611 | my $d = $xrefMaxZoomOut->{$f}; | ||
| 2612 | $xrefMaxZoomOut->{$f} = {title=>$t, data=>$d}; | ||
| 2613 | } | ||
| 2614 | } | ||
| 2615 | |||
| 2616 | formatTables($xref, [sort {$$a[0] cmp $$b[0]} @f], # Sort by file name | ||
| 2617 | columns => $columns, | ||
| 2618 | title => qq(Max Zoom In Matches), | ||
| 2619 | head => <<END, | ||
| 2620 | Xref found NNNN file matches on DDDD | ||
| 2621 | END | ||
| 2622 | file =>(fpe(qw(lists maxZoom txt))), | ||
| 2623 | summarize => 1); | ||
| 2624 | |||
| 2625 | dumpFile(fpe(qw(lists maxZoom data)), $xref->maxZoomOut); # Dump the search results | ||
| 2626 | |||
| 2627 | {maxZoomOut => $xref->maxZoomOut, # From multiverse to universe | ||
| 2628 | } | ||
| 2629 | } # reportMaxZoomOut | ||
| 2630 | |||
| 2631 | sub reportTopicDetails($) #P Things that occur once in each file | ||
| 2632 | {my ($xref) = @_; # Cross referencer | ||
| 2633 | |||
| 2634 | my @t; | ||
| 2635 | for my $f(sort @{$xref->inputFiles}) | ||
| 2636 | {push @t, [$xref->docType ->{$f}//q(), | ||
| 2637 | $xref->topicIds->{$f}//q(), | ||
| 2638 | $xref->author ->{$f}//q(), | ||
| 2639 | $xref->title ->{$f}//q(), | ||
| 2640 | $f, | ||
| 2641 | ]; | ||
| 2642 | } | ||
| 2643 | |||
| 2644 | formatTables($xref, \@t, | ||
| 2645 | columns => <<END, | ||
| 2646 | Tag The outermost tag | ||
| 2647 | Id The id on the outermost tag | ||
| 2648 | Author The author of the topic | ||
| 2649 | Title The title of the topic | ||
| 2650 | File The source file name as a relative file name | ||
| 2651 | END | ||
| 2652 | title=>qq(Topics), | ||
| 2653 | head=><<END, | ||
| 2654 | Xref found NNNN topics on DDDD | ||
| 2655 | END | ||
| 2656 | file=>(fpe(qw(lists topics txt))), | ||
| 2657 | summarize=>1); | ||
| 2658 | {} # From multiverse to universe | ||
| 2659 | } # reportTopicDetails | ||
| 2660 | |||
| 2661 | sub reportTopicReuse($) #P Count how frequently each topic is reused | ||
| 2662 | {my ($xref) = @_; # Cross referencer | ||
| 2663 | |||
| 2664 | my %t; | ||
| 2665 | if (my $xrefBookMapRefs = $xref->bookMapRefs) | ||
| 2666 | {for my $f(sort keys %{$xrefBookMapRefs}) | ||
| 2667 | {for my $t(sort keys %{$xrefBookMapRefs->{$f}}) | ||
| 2668 | {my $file = absFromAbsPlusRel($f, $t); | ||
| 2669 | $t{$file}{$f}++; | ||
| 2670 | } | ||
| 2671 | } | ||
| 2672 | } | ||
| 2673 | |||
| 2674 | for my $t(keys %t) # Eliminate bookmaprefs that are used only once | ||
| 2675 | {if (keys (%{$t{$t}}) <= 1) | ||
| 2676 | {delete $t{$t}; | ||
| 2677 | } | ||
| 2678 | } | ||
| 2679 | |||
| 2680 | my @t; | ||
| 2681 | for my $t(keys %t) # Target | ||
| 2682 | {for my $s(keys %{$t{$t}}) # Source | ||
| 2683 | {push @t, [scalar(keys %{$t{$t}}), $t{$t}{$s}, $t, $s]; | ||
| 2684 | } | ||
| 2685 | } | ||
| 2686 | |||
| 2687 | my $t = [sort {$a->[0] <=> $b->[0]} # Order report | ||
| 2688 | sort {$a->[2] cmp $b->[2]} @t]; | ||
| 2689 | |||
| 2690 | for my $i(keys @$t) # Deduplicate first column from third column | ||
| 2691 | {next unless $i; | ||
| 2692 | my $a = $t->[$i-1]; | ||
| 2693 | my $b = $t->[$i]; | ||
| 2694 | $b->[0] = '' if $a->[2] eq $b->[2]; | ||
| 2695 | } | ||
| 2696 | |||
| 2697 | formatTables($xref, $t, | ||
| 2698 | columns => <<END, | ||
| 2699 | Reuse The number of times the target topic is reused over all topics | ||
| 2700 | Count The number of times the target topic is reused in the source topic | ||
| 2701 | Target The topic that is being reused == the target of reuse | ||
| 2702 | Source The topic that is referencing the reused topic | ||
| 2703 | END | ||
| 2704 | title=>qq(Topic Reuses), | ||
| 2705 | head=><<END, | ||
| 2706 | Xref found NNNN topics that are currently being reused on DDDD | ||
| 2707 | END | ||
| 2708 | file=>(fpe(qw(lists topicReuse txt))), | ||
| 2709 | zero=>1, # Reuse is very unlikely because the matching criteria is the MD5 sum | ||
| 2710 | summarize=>1); | ||
| 2711 | {} # From multiverse to universe | ||
| 2712 | } # reportTopicReuse | ||
| 2713 | |||
| 2714 | # References might need fixing either because they are invalid or because we are | ||
| 2715 | # deguidizing | ||
| 2716 | |||
| 2717 | sub reportFixRefs($) #P Report of hrefs that need to be fixed | ||
| 2718 | {my ($xref) = @_; # Cross referencer | ||
| 2719 | |||
| 2720 | my @r; | ||
| 2721 | if (my $xrefFixRefs = $xref->fixRefs) | ||
| 2722 | {for my $f(sort keys %{$xrefFixRefs}) | ||
| 2723 | {for my $h(sort keys %{$xrefFixRefs->{$f}}) | ||
| 2724 | {push @r, [$h, $f]; | ||
| 2725 | } | ||
| 2726 | } | ||
| 2727 | } | ||
| 2728 | |||
| 2729 | formatTables($xref, \@r, | ||
| 2730 | columns => <<END, | ||
| 2731 | Reference The reference to be fixed | ||
| 2732 | Source The topic that contains the reference | ||
| 2733 | END | ||
| 2734 | title=>qq(References to fix), | ||
| 2735 | head=><<END, | ||
| 2736 | Xref found NNNN hrefs that should be fixed on DDDD | ||
| 2737 | END | ||
| 2738 | file=>(fpe(qw(lists fixRefs txt))), | ||
| 2739 | zero=>1, | ||
| 2740 | summarize=>1); | ||
| 2741 | {} # From multiverse to universe | ||
| 2742 | } # reportFixRefs | ||
| 2743 | |||
| 2744 | sub reportSourceFiles($) #P Source file for each topic | ||
| 2745 | {my ($xref) = @_; # Cross referencer | ||
| 2746 | my @r; | ||
| 2747 | if (my $xrefTargetTopicToInputFiles = $xref->targetTopicToInputFiles) | ||
| 2748 | {for my $f(sort keys %{$xrefTargetTopicToInputFiles}) # File | ||
| 2749 | {my $s = $xrefTargetTopicToInputFiles->{$f}; # Source file for topic | ||
| 2750 | push @r, [$f, join ', ', sort keys %$s] if $s; | ||
| 2751 | } | ||
| 2752 | } | ||
| 2753 | |||
| 2754 | formatTables($xref, [sort {$$a[0]cmp $$b[0]} @r], | ||
| 2755 | columns => <<END, | ||
| 2756 | Topic The topic file | ||
| 2757 | Source The source file(s) from which the topic was obtained | ||
| 2758 | END | ||
| 2759 | title=>qq(Source file for each topic), | ||
| 2760 | head=><<END, | ||
| 2761 | Xref found the source files for NNNN topics on DDDD | ||
| 2762 | END | ||
| 2763 | file=>(fpe(qw(lists source_file_for_each_topic txt))), | ||
| 2764 | summarize=>1); | ||
| 2765 | {} # From multiverse to universe | ||
| 2766 | } # reportSourceFiles | ||
| 2767 | |||
| 2768 | sub reportReferencesFromBookMaps($) #P Topics and images referenced from bookmaps | ||
| 2769 | {my ($xref) = @_; # Cross referencer | ||
| 2770 | my %bi; # Bookmap to image | ||
| 2771 | my %bt; # Bookmap to topics | ||
| 2772 | my %ib; # Images to bookmaps | ||
| 2773 | my %tb; # Topics to bookmaps | ||
| 2774 | my @bi; # Bookmap to image report | ||
| 2775 | my @bt; # Bookmap to topic report | ||
| 2776 | my @ib; # Images to bookmaps reports | ||
| 2777 | my @tb; # Topics to bookmaps reports | ||
| 2778 | |||
| 2779 | my $imageRefsFromTopic = sub # Image references from a topic | ||
| 2780 | {my ($b, $t) = @_; # Book map, topic | ||
| 2781 | for my $I(sort keys %{$xref->imagesReferencedFromTopics->{$t}}) # Image href | ||
| 2782 | {my $i = absFromAbsPlusRel($t, $I); | ||
| 2783 | push @bi, my $d = [$I, -e $i ? 1 : '', $i, $t, $b]; | ||
| 2784 | $bi{$b}{$i}++; # Images from bookmap | ||
| 2785 | $ib{$i}{$b}++; # Images to bookmaps | ||
| 2786 | } | ||
| 2787 | }; | ||
| 2788 | |||
| 2789 | if (my $xrefBookMapRefs = $xref->bookMapRefs) # Book map as that is the only kind of file containing a topic ref | ||
| 2790 | {for my $b(sort keys %{$xrefBookMapRefs}) # Book map as that is the only kind of file containing a topic ref | ||
| 2791 | {for my $T(sort keys %{$xrefBookMapRefs->{$b}}) # Topic href | ||
| 2792 | {my $t = absFromAbsPlusRel($b, $T); | ||
| 2793 | |||
| 2794 | push @bt, [$T, (-e $T ? 1 : q()), $t, $b]; # Report book map to topic | ||
| 2795 | $bt{$b}{$t}++; # Book maps to topics | ||
| 2796 | $tb{$t}{$b}++; # Topics to book maps | ||
| 2797 | |||
| 2798 | &$imageRefsFromTopic($b, $t); | ||
| 2799 | } | ||
| 2800 | |||
| 2801 | for my $C(sort keys %{$xref->conRefs->{$b}}) # Conref | ||
| 2802 | {my ($t) = parseDitaRef($C, $b); | ||
| 2803 | &$imageRefsFromTopic($b, $t); | ||
| 2804 | } | ||
| 2805 | } | ||
| 2806 | } | ||
| 2807 | |||
| 2808 | for my $i(sort keys %ib) # Image | ||
| 2809 | {for my $b(sort keys %{$ib{$i}}) # Bookmap | ||
| 2810 | {my $c = $ib{$i}{$b}; | ||
| 2811 | push @ib, [$c, $i, $b]; | ||
| 2812 | } | ||
| 2813 | } | ||
| 2814 | |||
| 2815 | for my $t(sort keys %tb) # Topic | ||
| 2816 | {for my $b(sort keys %{$tb{$t}}) # Bookmap | ||
| 2817 | {my $c = $tb{$t}{$b}; | ||
| 2818 | push @tb, [$c, $t, $b]; | ||
| 2819 | } | ||
| 2820 | } | ||
| 2821 | # ALL FILES (below) ARE FULLY QUALIFIED! | ||
| 2822 | $xref->topicsToReferringBookMaps = \%tb; # Topics to referring bookmaps | ||
| 2823 | $xref->topicsReferencedFromBookMaps = \%bt; # Topics referenced from bookmaps | ||
| 2824 | $xref->imagesReferencedFromBookMaps = \%bi; # Images referenced from bookmaps | ||
| 2825 | $xref->imagesToRefferingBookMaps = \%ib; # Images to bookmaps | ||
| 2826 | |||
| 2827 | formatTables($xref, \@bi, | ||
| 2828 | columns => <<END, | ||
| 2829 | Href The href that contains an image reference | ||
| 2830 | Exists Whether the referenced image exists or not | ||
| 2831 | Image The name of the image file | ||
| 2832 | Topic The topic that referenced the image | ||
| 2833 | Bookmap The book map that referenced the topic | ||
| 2834 | END | ||
| 2835 | title=>qq(Images referenced from bookmaps), | ||
| 2836 | head=><<END, | ||
| 2837 | Xref found NNNN images referenced from bookmaps via topics on DDDD | ||
| 2838 | END | ||
| 2839 | file=>(fpe(qw(lists images_from_bookmaps txt))), | ||
| 2840 | zero=>1, | ||
| 2841 | summarize=>1); | ||
| 2842 | |||
| 2843 | formatTables($xref, \@bt, | ||
| 2844 | columns => <<END, | ||
| 2845 | Reference The topic reference | ||
| 2846 | Exists Whether the referenced topic exists or not | ||
| 2847 | Topic The topic that referenced the image | ||
| 2848 | Bookmap The book map that referenced the topic | ||
| 2849 | END | ||
| 2850 | title=>qq(Topics referenced from bookmaps), | ||
| 2851 | head=><<END, | ||
| 2852 | Xref found NNNN topics referenced from bookmaps via topics on DDDD | ||
| 2853 | END | ||
| 2854 | file=>(fpe(qw(lists bookmaps_to_topics txt))), | ||
| 2855 | zero=>1, | ||
| 2856 | summarize=>1); | ||
| 2857 | |||
| 2858 | formatTables($xref, \@tb, | ||
| 2859 | columns => <<END, | ||
| 2860 | Count The number of times this topic is reffered to by this bookmap | ||
| 2861 | Topic The topic in question | ||
| 2862 | Bookmap A bookmap that refers to this topic | ||
| 2863 | END | ||
| 2864 | title=>qq(Topics to referring bookmaps), | ||
| 2865 | head=><<END, | ||
| 2866 | Xref found NNNN topics referred to from bookmaps on DDDD | ||
| 2867 | END | ||
| 2868 | file=>(fpe(qw(lists topics_to_bookmaps txt))), | ||
| 2869 | zero=>1, | ||
| 2870 | summarize=>1); | ||
| 2871 | |||
| 2872 | formatTables($xref, \@ib, | ||
| 2873 | columns => <<END, | ||
| 2874 | Count The number of times this image is reffered to by this bookmap | ||
| 2875 | Image The image in question | ||
| 2876 | Bookmap A bookmap that refers to this image | ||
| 2877 | END | ||
| 2878 | title=>qq(Images to referring bookmaps), | ||
| 2879 | head=><<END, | ||
| 2880 | Xref found NNNN images referred to from bookmaps on DDDD | ||
| 2881 | END | ||
| 2882 | file=>(fpe(qw(lists images_to_bookmaps txt))), | ||
| 2883 | zero=>1, | ||
| 2884 | summarize=>1); | ||
| 2885 | } # reportReferencesFromBookMaps | ||
| 2886 | |||
| 2887 | sub reportExteriorMaps($) #P Maps that are not referenced by any other map | ||
| 2888 | {my ($xref) = @_; # Cross referencer | ||
| 2889 | |||
| 2890 | my %r; # Map reference count | ||
| 2891 | if (my $xrefTopicsReferencedFromBookMaps = $xref->topicsReferencedFromBookMaps) | ||
| 2892 | {for my $b(sort keys %{$xrefTopicsReferencedFromBookMaps}) # Each map | ||
| 2893 | {$r{$b}++; # Save map | ||
| 2894 | for my $t(sort keys %{$xrefTopicsReferencedFromBookMaps->{$b}}) # Each topic reference from the map | ||
| 2895 | {if (my $d = $xref->docType->{$t}) # Document type | ||
| 2896 | {if ($d =~ m(map\Z)s) # Its a map | ||
| 2897 | {$r{$t}--; # It has been referenced | ||
| 2898 | } | ||
| 2899 | } | ||
| 2900 | } | ||
| 2901 | } | ||
| 2902 | } | ||
| 2903 | |||
| 2904 | my %u; # Unreferenced maps | ||
| 2905 | for my $b(sort keys %r) # Each map | ||
| 2906 | {$u{$b}++ if $r{$b} > 0; # Unreferenced map | ||
| 2907 | } | ||
| 2908 | |||
| 2909 | formatTables($xref, | ||
| 2910 | hashOfCountsToArray(\%u), | ||
| 2911 | columns => <<END, | ||
| 2912 | Bookmap An exterior bookmap | ||
| 2913 | END | ||
| 2914 | title=>qq(Exterior bookmaps), | ||
| 2915 | head=><<END, | ||
| 2916 | Xref found NNNN exterior bookmaps on DDDD | ||
| 2917 | |||
| 2918 | An exterior map is one that is not referenced from any other map in the corpus. | ||
| 2919 | END | ||
| 2920 | file=>(fpe(qw(lists exterior_maps txt))), | ||
| 2921 | summarize=>1); | ||
| 2922 | |||
| 2923 | {exteriorMaps => \%u, # Exterior maps == maps that are not referenced by another map | ||
| 2924 | } # From multiverse to universe | ||
| 2925 | } # reportExteriorMaps | ||
| 2926 | |||
| 2927 | sub reportTopicsNotReferencedFromBookMaps($) #P Topics not referenced from bookmaps | ||
| 2928 | {my ($xref) = @_; # Cross referencer | ||
| 2929 | |||
| 2930 | my %r; # Topics referenced from a bookmap | ||
| 2931 | if (my $xrefTopicsReferencedFromBookMaps = $xref->topicsReferencedFromBookMaps) | ||
| 2932 | {for my $b(sort keys %{$xrefTopicsReferencedFromBookMaps}) # Book maps | ||
| 2933 | {for my $f(sort keys %{$xrefTopicsReferencedFromBookMaps->{$b}}) # Topics referenced from bookmaps | ||
| 2934 | {$r{$f}++; # Topic reference count | ||
| 2935 | } | ||
| 2936 | } | ||
| 2937 | } | ||
| 2938 | |||
| 2939 | my @n; # Topic not referenced from a bookmap | ||
| 2940 | if (my $xrefDocType = $xref->docType) | ||
| 2941 | {for my $f(sort keys %{$xrefDocType}) # Input xml files | ||
| 2942 | {next if $xrefDocType ->{$f} =~ m(map\Z)s; | ||
| 2943 | push @n, $f unless $r{$f}; # Topic not referenced from a bookmap | ||
| 2944 | } | ||
| 2945 | } | ||
| 2946 | |||
| 2947 | my $facet = q(Topics referenced from bookmaps); # Facet | ||
| 2948 | |||
| 2949 | my $n = $xref->topicsNotReferencedFromBookMaps = {map {$_=>1} @n}; #Topics not referenced by any bookmap | ||
| 2950 | |||
| 2951 | formatTables($xref, | ||
| 2952 | [map {[$r{$_}, swapFilePrefix($_, $xref->inputFolder)]} sort keys %r], | ||
| 2953 | columns => <<END, | ||
| 2954 | Count Number of referring bookmaps | ||
| 2955 | Topic Topic referenced from one or more bookmaps | ||
| 2956 | END | ||
| 2957 | title => qq(Topics referenced by bookmaps), | ||
| 2958 | facet => $facet, aspectColor=>q(green), | ||
| 2959 | head => <<END, | ||
| 2960 | Xref found NNNN topics that are referenced by one or more bookmaps on DDDD | ||
| 2961 | END | ||
| 2962 | file => fpe(qw(good topics_referenced_from_bookmaps txt)), | ||
| 2963 | summarize => 1); | ||
| 2964 | |||
| 2965 | formatTables($xref, | ||
| 2966 | [map {[swapFilePrefix($_, $xref->inputFolder)]} sort {$a cmp $b} @n], | ||
| 2967 | columns => <<END, | ||
| 2968 | Topic Unreferenced topic file name | ||
| 2969 | END | ||
| 2970 | title => qq(Topics not referenced by any bookmap), | ||
| 2971 | facet => $facet, aspectColor=>q(red), | ||
| 2972 | head => <<END, | ||
| 2973 | Xref found NNNN topics that are not referenced by any bookmaps on DDDD | ||
| 2974 | END | ||
| 2975 | file => fpe(qw(bad topics_not_referenced_from_bookmaps txt)), | ||
| 2976 | summarize => 1); | ||
| 2977 | |||
| 2978 | {topicsNotReferencedFromBookMaps=>$n | ||
| 2979 | } | ||
| 2980 | } # reportTopicsNotReferencedFromBookMaps | ||
| 2981 | |||
| 2982 | |||
| 2983 | sub reportTableDimensions($) #P Report table dimensions | ||
| 2984 | {my ($xref) = @_; # Cross referencer | ||
| 2985 | |||
| 2986 | my %d = %{$xref->tableDimensions}; | ||
| 2987 | my %r; # Topic not referenced from a bookmap | ||
| 2988 | for my $f(keys %d) # Files with tables | ||
| 2989 | {for my $c(keys %{$d{$f}}) # Table columns | ||
| 2990 | {for my $r(keys %{$d{$f}{$c}}) # Table rows | ||
| 2991 | {push @{$r{$c}{$r}}, $f; | ||
| 2992 | } | ||
| 2993 | } | ||
| 2994 | } | ||
| 2995 | |||
| 2996 | my @d; # Topic not referenced from a bookmap | ||
| 2997 | for my $c(sort {$a <=> $b} keys %r) # Table columns | ||
| 2998 | {for my $r(sort {$a <=> $b} keys %{$r{$c}}) # Table rows | ||
| 2999 | {my ($f, @f) = @{$r{$c}{$r}}; | ||
| 3000 | push @d, [$c, $r, $f]; | ||
| 3001 | push @d, [q(), q(), $_] for @f; | ||
| 3002 | } | ||
| 3003 | } | ||
| 3004 | |||
| 3005 | formatTables($xref, [@d], | ||
| 3006 | columns => <<END, | ||
| 3007 | Columns Number of columns in table | ||
| 3008 | Rows Number of rows in table | ||
| 3009 | File A file that contains a table with this many columns and rows | ||
| 3010 | END | ||
| 3011 | title=>qq(Table dimensions), | ||
| 3012 | head=><<END, | ||
| 3013 | Xref found NNNN table dimensions on DDDD | ||
| 3014 | END | ||
| 3015 | file=>(fpe(qw(count table_dimensions txt))), | ||
| 3016 | summarize=>1); | ||
| 3017 | |||
| 3018 | {} | ||
| 3019 | } # reportTableDimensions | ||
| 3020 | |||
| 3021 | sub reportOtherMeta($) #P Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map. | ||
| 3022 | {my ($xref) = @_; # Cross referencer | ||
| 3023 | |||
| 3024 | my %t; my %b; my %B; # Othermeta at topic level, othermeta to migrate to bookmap, othermeta already in bookmap | ||
| 3025 | for my $b(sort keys $xref->bookMapRefs->%*) | ||
| 3026 | {for my $n(sort keys $xref->otherMeta->{$b}->%*) # Name of othermeta | ||
| 3027 | {for my $c(sort keys $xref->otherMeta->{$b}->{$n}->%*) # Content of other meta | ||
| 3028 | {$b{$b}{$n}{$c}++; | ||
| 3029 | $B{$b}{$n}{$c}++; | ||
| 3030 | } | ||
| 3031 | } | ||
| 3032 | |||
| 3033 | for my $r(sort keys $xref->bookMapRefs->{$b}->%*) # Each topic reached from the bookmap | ||
| 3034 | {my $t = absFromAbsPlusRel($b, $r); # Topic references are relative | ||
| 3035 | if (my $o = $xref->otherMeta->{$t}) | ||
| 3036 | {for my $n(sort keys $o->%*) | ||
| 3037 | {for my $c(sort keys $o->{$n}->%*) | ||
| 3038 | {$t{$t}{$n}{$c}{$b}++; # Othermeta by topic | ||
| 3039 | $b{$b}{$n}{$c}++; # Put topic othermeta in bookmap | ||
| 3040 | } | ||
| 3041 | } | ||
| 3042 | } | ||
| 3043 | } | ||
| 3044 | } | ||
| 3045 | |||
| 3046 | if (1) # Bookmaps and topics with duplicate othermeta should be empty | ||
| 3047 | {my @d; | ||
| 3048 | for my $b(sort keys %B) # Bookmaps | ||
| 3049 | {for my $n(sort keys $B{$b}->%*) | ||
| 3050 | {if (my $N = keys $B{$b}{$n}->%*) | ||
| 3051 | {if ($N > 1) | ||
| 3052 | {my ($c, @c) = sort keys $B{$b}{$n}->%*; | ||
| 3053 | push @d, [$b, $n, $N, $c], map {[(q()) x 3, $_]} @c; | ||
| 3054 | } | ||
| 3055 | } | ||
| 3056 | } | ||
| 3057 | } | ||
| 3058 | |||
| 3059 | for my $t(sort keys %t) # Topics | ||
| 3060 | {for my $n(sort keys $t{$t}->%*) | ||
| 3061 | {if (my $N = keys $t{$t}{$n}->%*) | ||
| 3062 | {if ($N > 1) | ||
| 3063 | {my ($c, @c) = sort keys $t{$t}{$n}->%*; | ||
| 3064 | push @d, [$t, $n, $N, $c], map {[(q()) x 3, $_]} @c; | ||
| 3065 | } | ||
| 3066 | } | ||
| 3067 | } | ||
| 3068 | } | ||
| 3069 | |||
| 3070 | formatTables($xref, $xref->otherMetaDuplicatesSeparately = [@d], | ||
| 3071 | columns => <<END, | ||
| 3072 | Source Bookmaps or topic files with duplicate othermeta data | ||
| 3073 | Name Duplicated othermeta name field | ||
| 3074 | Count Number of duplicates | ||
| 3075 | Content Othermeta content | ||
| 3076 | END | ||
| 3077 | title=>q(Duplicate othermeta data in bookmaps and topics considered separately), | ||
| 3078 | head =>q(Found NNNN duplicate othermeta items on DDDD), | ||
| 3079 | clearUpLeft => -1, summarize=>1, | ||
| 3080 | file =>fpe(qw(other_meta duplicates_separately txt))); | ||
| 3081 | } | ||
| 3082 | |||
| 3083 | if (1) # Report duplicate othermeta in bookmaps with called topics othermeta included | ||
| 3084 | {my @d; | ||
| 3085 | |||
| 3086 | for my $b(sort keys %b) # Bookmaps | ||
| 3087 | {for my $n(sort keys $b{$b}->%*) | ||
| 3088 | {if (my $N = keys $b{$b}{$n}->%*) | ||
| 3089 | {if ($N > 1) | ||
| 3090 | {my ($c, @c) = sort keys $b{$b}{$n}->%*; | ||
| 3091 | push @d, [$b, $n, $N, $c], map {[(q()) x 3, $_]} @c; | ||
| 3092 | } | ||
| 3093 | } | ||
| 3094 | } | ||
| 3095 | } | ||
| 3096 | |||
| 3097 | formatTables($xref, $xref->otherMetaDuplicatesCombined = [@d], | ||
| 3098 | columns => <<END, | ||
| 3099 | Source Bookmap with duplicate othermeta with called topics othermeta included | ||
| 3100 | Name Duplicated othermeta name field | ||
| 3101 | Count Number of duplicates | ||
| 3102 | Content Othermeta content | ||
| 3103 | END | ||
| 3104 | title=>q(Duplicate othermeta in bookmaps with called topic othermeta included), | ||
| 3105 | head =>q(Found NNNN duplicate othermeta items on DDDD), | ||
| 3106 | clearUpLeft => -1, summarize=>1, | ||
| 3107 | file =>fpe(qw(other_meta duplicates txt))); | ||
| 3108 | } | ||
| 3109 | |||
| 3110 | my %o; # Topic overrides | ||
| 3111 | for my $t(sort keys %t) # Find topic othermeta which must override bookmap othermeta | ||
| 3112 | {for my $n(sort keys $t{$t}->%*) | ||
| 3113 | {for my $c(sort keys $t{$t}{$n}->%*) | ||
| 3114 | {for my $b(sort keys $t{$t}{$n}{$c}->%*) | ||
| 3115 | {if (!$b{$b}{$n}{$c} or keys($b{$b}{$n}->%*) != 1) # Override the othermeta from the bookmap unless the bookmap agrees that there is only value for this name | ||
| 3116 | {$o{$t}{$n}{$c}{$b}++; | ||
| 3117 | } | ||
| 3118 | } | ||
| 3119 | } | ||
| 3120 | } | ||
| 3121 | } | ||
| 3122 | |||
| 3123 | if (1) # Keep in topic because we cannot push the meta data to all the calling bookmaps | ||
| 3124 | {my @k; | ||
| 3125 | for my $t(sort keys %o) | ||
| 3126 | {for my $n(sort keys $o{$t}->%*) | ||
| 3127 | {for my $c(sort keys $o{$t}{$n}->%*) | ||
| 3128 | {push @k, [$t, $n, $c, sort keys $o{$t}{$n}{$c}->%*]; # The bookmaps that cause the over ride | ||
| 3129 | } | ||
| 3130 | } | ||
| 3131 | } | ||
| 3132 | |||
| 3133 | formatTables($xref, $xref->otherMetaRemainWithTopic = [@k], | ||
| 3134 | columns => <<END, | ||
| 3135 | Topic Topic file name | ||
| 3136 | Name Othermeta name field to be retained | ||
| 3137 | Content Othermeta name content to be retained | ||
| 3138 | Bookmaps One or more bookmaps that prevented the migration of this othermeta to the calling bookmaps | ||
| 3139 | END | ||
| 3140 | title=>q(Othermeta kept in topics because calling bookmaps disagree), | ||
| 3141 | clearUpLeft => -1, summarize=>1, | ||
| 3142 | head =>qq(Found NNNN othermeta items that must remain in topic on DDDD), | ||
| 3143 | file =>fpe(qw(other_meta must_remain_in_topic txt))); | ||
| 3144 | } | ||
| 3145 | |||
| 3146 | if (1) # Report othermeta pushed to the bookmaps | ||
| 3147 | {my @p; | ||
| 3148 | for my $t(sort keys %t) | ||
| 3149 | {for my $n(sort keys $t{$t}->%*) | ||
| 3150 | {for my $c(sort keys $t{$t}{$n}->%*) | ||
| 3151 | {for my $b(sort keys $t{$t}{$n}{$c}->%*) | ||
| 3152 | {push @p, [$t, $n, $c, $b] unless $o{$t}{$n}{$c}{$b} # Can be pushed to bookmap | ||
| 3153 | } | ||
| 3154 | } | ||
| 3155 | } | ||
| 3156 | } | ||
| 3157 | |||
| 3158 | formatTables($xref, $xref->otherMetaPushToBookMap = [@p], | ||
| 3159 | columns => <<END, | ||
| 3160 | Topic Topic file name | ||
| 3161 | Name Othermeta name field to be retained | ||
| 3162 | Content Othermeta name content to be retained | ||
| 3163 | Bookmap The bookmap data othermeta can be migrated to | ||
| 3164 | END | ||
| 3165 | title=>q(Othermeta data that can be pushed to the calling bookmaps), | ||
| 3166 | clearUpLeft => -1, summarize=>1, | ||
| 3167 | head =>qq(Found NNNN othermeta items that can be pushed to the calling bookmaps on DDDD), | ||
| 3168 | file =>fpe(qw(other_meta push_to_book_maps txt))); | ||
| 3169 | } | ||
| 3170 | |||
| 3171 | if (1) # Report bookmap othermeta before topic othermeta has been included | ||
| 3172 | {my @b; | ||
| 3173 | |||
| 3174 | for my $b(sort keys %B) | ||
| 3175 | {for my $n(sort keys $B{$b}->%*) | ||
| 3176 | {for my $c(sort keys $B{$b}{$n}->%*) | ||
| 3177 | {push @b, [$b, $n, scalar(keys $B{$b}{$n}->%*), $c]; | ||
| 3178 | |||
| 3179 | } | ||
| 3180 | } | ||
| 3181 | } | ||
| 3182 | |||
| 3183 | formatTables($xref, $xref->otherMetaBookMapsBeforeTopicIncludes = [@b], | ||
| 3184 | columns => <<END, | ||
| 3185 | Bookmap Bookmap file name | ||
| 3186 | Name Bookmap othermeta name | ||
| 3187 | Count Number of distinct values for this othermeta name in this bookmap | ||
| 3188 | Content Othermeta content for this name | ||
| 3189 | END | ||
| 3190 | title=> q(Bookmap othermeta before topic othermeta has been included), | ||
| 3191 | clearUpLeft => -1, summarize=>1, | ||
| 3192 | head => qq(Xref found NNNN Bookmap othermeta tags before topic othermeta was included), | ||
| 3193 | file => fpe(qw(other_meta book_maps_before_topics_included txt))); | ||
| 3194 | } | ||
| 3195 | |||
| 3196 | if (1) # Report bookmap othermeta after topic othermeta has been included | ||
| 3197 | {my @b; | ||
| 3198 | |||
| 3199 | for my $b(sort keys %b) | ||
| 3200 | {for my $n(sort keys $b{$b}->%*) | ||
| 3201 | {for my $c(sort keys $b{$b}{$n}->%*) | ||
| 3202 | {push @b, [$b, $n, scalar(keys $b{$b}{$n}->%*), $c]; | ||
| 3203 | } | ||
| 3204 | } | ||
| 3205 | } | ||
| 3206 | |||
| 3207 | formatTables($xref, $xref->otherMetaBookMapsAfterTopicIncludes = [@b], | ||
| 3208 | columns => <<END, | ||
| 3209 | Bookmap Bookmap file name | ||
| 3210 | Name Bookmap or topic othermeta name | ||
| 3211 | Count Number of distinct values for this othermeta name | ||
| 3212 | Content Othermeta content for this name | ||
| 3213 | END | ||
| 3214 | title=> q(Bookmap othermeta data after topic othermeta has been included), | ||
| 3215 | clearUpLeft => -1, summarize=>1, | ||
| 3216 | head => qq(Xref found NNNN Bookmap othermeta tags after topic othermeta was included on DDDD), | ||
| 3217 | file =>fpe(qw(other_meta book_maps_after_topics_included txt))); | ||
| 3218 | } | ||
| 3219 | |||
| 3220 | {otherMetaDuplicatesSeparately => $xref->otherMetaDuplicatesSeparately, | ||
| 3221 | otherMetaDuplicatesCombined => $xref->otherMetaDuplicatesCombined, | ||
| 3222 | otherMetaRemainWithTopic => $xref->otherMetaRemainWithTopic, | ||
| 3223 | otherMetaPushToBookMap => $xref->otherMetaPushToBookMap, | ||
| 3224 | otherMetaBookMapsBeforeTopicIncludes => $xref->otherMetaBookMapsBeforeTopicIncludes, | ||
| 3225 | otherMetaBookMapsAfterTopicIncludes => $xref->otherMetaBookMapsAfterTopicIncludes, | ||
| 3226 | } | ||
| 3227 | |||
| 3228 | } # reportOtherMeta | ||
| 3229 | |||
| 3230 | sub createSubjectSchemeMap($) #P Create a subject scheme map from othermeta | ||
| 3231 | {my ($xref) = @_; # Cross referencer | ||
| 3232 | |||
| 3233 | my %o; # Consolidated other meta | ||
| 3234 | for my $f(sort keys $xref->otherMeta->%*) # Each file containing othermeta | ||
| 3235 | {for my $n(sort keys $xref->otherMeta->{$f}->%*) # Name of othermeta | ||
| 3236 | {for my $c(sort keys $xref->otherMeta->{$f}->{$n}->%*) # Content of other meta | ||
| 3237 | {$o{$n}{$c}++; | ||
| 3238 | } | ||
| 3239 | } | ||
| 3240 | } | ||
| 3241 | |||
| 3242 | if (my $out = $xref->subjectSchemeMap) # SubjectSchemeMap requested by supplying an output folder for the maps | ||
| 3243 | {my $makeNavTitle = sub # Create a nav title using the Gearhart NavTitle Method | ||
| 3244 | {my ($c) = @_; # Othermeta content | ||
| 3245 | ucfirst $c =~ s(_) ( )gsr # The Gearhart NavTitle method | ||
| 3246 | }; | ||
| 3247 | |||
| 3248 | my @m; | ||
| 3249 | for my $n(sort keys %o) # Layout map entries | ||
| 3250 | {next if $n =~ m(\Atopic_type\Z)i; | ||
| 3251 | my $t = &$makeNavTitle($n); | ||
| 3252 | push @m, qq(<subjectHead id="$n" navtitle="$t">); | ||
| 3253 | |||
| 3254 | my %c; # Normalize all the othermeta | ||
| 3255 | for my $content(sort keys $o{$n}->%*) | ||
| 3256 | {for my $c(split m(\s+)s, $content) | ||
| 3257 | {$c{$c}++ | ||
| 3258 | } | ||
| 3259 | } | ||
| 3260 | for my $c(sort keys %c) # Write the normalized othermeta | ||
| 3261 | {my $t = &$makeNavTitle($c); | ||
| 3262 | push @m, qq(<subjectdef keys="$c" navtitle="$t"/>); | ||
| 3263 | } | ||
| 3264 | |||
| 3265 | push @m, qq(</subjectHead>); | ||
| 3266 | } | ||
| 3267 | |||
| 3268 | my $m = join "\n", q(<subjectScheme>), @m, q(</subjectScheme>); # Map | ||
| 3269 | my $x = Data::Edit::Xml::new($m); # Parse map | ||
| 3270 | owf($out, $x->ditaPrettyPrintWithHeaders); # Print map | ||
| 3271 | } | ||
| 3272 | |||
| 3273 | {otherMetaConsolidated=>\%o} # Consolidated meta data | ||
| 3274 | } # createSubjectSchemeMap | ||
| 3275 | |||
| 3276 | sub writeClassificationHtml($$) #P Write classification tree as html | ||
| 3277 | {my ($xref, $classification) = @_; # Cross referencer, {title=>{subject=>{file=>++}}} | ||
| 3278 | push my @h, <<END; # Outer table | ||
| 3279 | <style> | ||
| 3280 | .sizeLargeBold | ||
| 3281 | {font-size: 1.2em; | ||
| 3282 | font-weight: bold; | ||
| 3283 | } | ||
| 3284 | </style> | ||
| 3285 | <table id="selectBySubject" border="1" cellpadding="10"> | ||
| 3286 | <tr><th>Select by Subject<th>Topics that have been selected | ||
| 3287 | END | ||
| 3288 | push @h, <<END; # Table in column 1 of titles | ||
| 3289 | <tr><td><table border="1" cellpadding="10"> | ||
| 3290 | <tr><th>Topic Subject<th>Subject Ref | ||
| 3291 | END | ||
| 3292 | |||
| 3293 | my $iTitle = 0; my $iSubject = 0; # Id generator's for titles and subjects | ||
| 3294 | my @json; # Json holding titles and subjects | ||
| 3295 | for my $t(sort keys %$classification) # Each title | ||
| 3296 | {++$iTitle; | ||
| 3297 | push @h, <<END; # Hidden table of subjects | ||
| 3298 | <tr> | ||
| 3299 | <td valign="top"><a id="clickOnTitle_$iTitle" | ||
| 3300 | onclick="clickOnTitle($iTitle)">$t</a> | ||
| 3301 | <td><div style="display: none" id="title_$iTitle"> | ||
| 3302 | <table border="0" cellpadding="10"> | ||
| 3303 | END | ||
| 3304 | for my $s(sort keys $classification->{$t}->%*) # Each subject | ||
| 3305 | {++$iSubject; | ||
| 3306 | for my $f(sort keys $classification->{$t}{$s}->%*) # Load topics for title and subject | ||
| 3307 | {push @json, qq(topicDetails.push([$iTitle, $iSubject, "$f"]);); | ||
| 3308 | } | ||
| 3309 | push @h, <<END; # Click on subject | ||
| 3310 | <tr><td><span id="clickOnSubject_$iSubject" | ||
| 3311 | onclick='clickOnSubject($iTitle, $iSubject)'>$s</span> | ||
| 3312 | END | ||
| 3313 | } | ||
| 3314 | push @h, <<END; # End hidden table of subjects | ||
| 3315 | </table></div> | ||
| 3316 | END | ||
| 3317 | } | ||
| 3318 | push @h, <<END; # End of table in column 1 | ||
| 3319 | </table> | ||
| 3320 | END | ||
| 3321 | |||
| 3322 | push @h, <<END; # Column 2 - selected topics | ||
| 3323 | <td valign="top"> | ||
| 3324 | <div id="div"></div> | ||
| 3325 | END | ||
| 3326 | |||
| 3327 | my $json = join "\n", @json; # Json containing title and subjects to topics | ||
| 3328 | push @h, <<END; # Outer table | ||
| 3329 | </table> | ||
| 3330 | <script> | ||
| 3331 | const topicDetails = []; // [Title number, subject number, topic] | ||
| 3332 | const topicsSelected = {}; // Topics that have been selected | ||
| 3333 | const titleExpanded = []; // Title has been expanded | ||
| 3334 | |||
| 3335 | $json; | ||
| 3336 | |||
| 3337 | function clickOnTitle(title) // Expand a title to see its subjects | ||
| 3338 | {const t = document.getElementById("clickOnTitle_"+title); // Button | ||
| 3339 | const T = document.getElementById("title_"+title); // Div | ||
| 3340 | |||
| 3341 | T.style.display = titleExpanded[title] ? "none" : "block"; // Expand/contract div | ||
| 3342 | titleExpanded[title] = !titleExpanded[title]; | ||
| 3343 | } | ||
| 3344 | |||
| 3345 | function clickOnSubject(title, subject) // Add or remove topics from the hash of topics selected | ||
| 3346 | {const sli = document.getElementById("clickOnSubject_"+subject); | ||
| 3347 | |||
| 3348 | if (!sli.classList.contains("sizeLargeBold")) // Add as not emphasized | ||
| 3349 | {for(let d of topicDetails) | ||
| 3350 | {if (d[1] == subject) | ||
| 3351 | {const s = d[2]; | ||
| 3352 | if (topicsSelected[s]) topicsSelected[s]++; else topicsSelected[s] = 1; | ||
| 3353 | } | ||
| 3354 | } | ||
| 3355 | sli.classList.add("sizeLargeBold"); | ||
| 3356 | } | ||
| 3357 | else // Remove as emphasized | ||
| 3358 | {for(let d of topicDetails) | ||
| 3359 | {if (d[1] == subject) | ||
| 3360 | {const s = d[2]; | ||
| 3361 | if (topicsSelected && topicsSelected[s] > 0) topicsSelected[s]-- | ||
| 3362 | else topicsSelected[s] = 0; | ||
| 3363 | } | ||
| 3364 | } | ||
| 3365 | sli.classList.remove("sizeLargeBold"); | ||
| 3366 | } | ||
| 3367 | |||
| 3368 | if (true) // If any of the subjects are selected select teh title as well | ||
| 3369 | {let count = 0; | ||
| 3370 | for(let d of topicDetails) | ||
| 3371 | {if (d[0] == title) | ||
| 3372 | {const s = document.getElementById("clickOnSubject_"+d[1]); | ||
| 3373 | if (s.classList.contains("sizeLargeBold")) ++count; | ||
| 3374 | } | ||
| 3375 | } | ||
| 3376 | const t = document.getElementById("clickOnTitle_"+title); | ||
| 3377 | if (count > 0) t.classList.add ("sizeLargeBold"); | ||
| 3378 | else t.classList.remove("sizeLargeBold"); | ||
| 3379 | } | ||
| 3380 | |||
| 3381 | let h = ''; // Create html to show the list of selected topics | ||
| 3382 | for (let [topic, count] of Object.entries(topicsSelected)) | ||
| 3383 | {if (count > 0) | ||
| 3384 | {h += '<tr><td><a href="client.pl?getFile='+ // Anchor for topic file | ||
| 3385 | topic+'">'+topic+'</a><td align="right">'+count+"\\n"; | ||
| 3386 | } | ||
| 3387 | } | ||
| 3388 | |||
| 3389 | const d = document.getElementById("div"); | ||
| 3390 | d.innerHTML = '<table border="0" cellpadding="10">'+h+'</table>'; | ||
| 3391 | } | ||
| 3392 | |||
| 3393 | </script> | ||
| 3394 | END | ||
| 3395 | |||
| 3396 | if (1) # Classification html written as a report | ||
| 3397 | {my $f = fpe($xref->reports, qw(other_meta classificationScheme html)); | ||
| 3398 | owf($f, join "\n", @h); | ||
| 3399 | } | ||
| 3400 | } # writeClassificationHtml | ||
| 3401 | |||
| 3402 | sub createClassificationMap($$$) #P Create a classification map for each bookmap | ||
| 3403 | {my ($xref, $bookMap, $classification) = @_; # Cross referencer, bookmap to classify, classification scheme | ||
| 3404 | |||
| 3405 | for my $b($bookMap) # Book map | ||
| 3406 | {my $out = fpn($b).classificationMapSuffix; | ||
| 3407 | |||
| 3408 | push my @o, <<END; # Other meta fields for classification map | ||
| 3409 | <map> | ||
| 3410 | <topicmeta> | ||
| 3411 | <othermeta name="appaapps.classificationMap" content="yes"/> | ||
| 3412 | <othermeta name="appaapps.classificationMap.source" content="$bookMap"/> | ||
| 3413 | </topicmeta> | ||
| 3414 | END | ||
| 3415 | |||
| 3416 | for my $f(sort keys $xref->topicsReferencedFromBookMaps->{$b}->%*) # Book map topics | ||
| 3417 | {next unless my $tag = $xref->baseTag->{$f}; # Check that the referenced topic exists locally | ||
| 3418 | |||
| 3419 | my $F = $tag !~ m(map\Z)i ? $f : fpn($f).classificationMapSuffix; # Reference classification map rather than actual map | ||
| 3420 | |||
| 3421 | my $r = relFromAbsAgainstAbs($F, $out); | ||
| 3422 | |||
| 3423 | push @o, <<END; | ||
| 3424 | <topicref href="$r"> | ||
| 3425 | <topicmeta> | ||
| 3426 | <keywords/> | ||
| 3427 | </topicmeta> | ||
| 3428 | END | ||
| 3429 | |||
| 3430 | for my $n(sort keys $xref->otherMeta->{$f}->%*) # Name of other meta in book map topic | ||
| 3431 | {next if $n =~ m(\Atopic_type\Z)i; | ||
| 3432 | my ($navTitle) = sort keys $xref->otherMeta->{$f}{$n}->%*; | ||
| 3433 | |||
| 3434 | push @o, <<END; | ||
| 3435 | <topicsubject outputclass="$n" navtitle="$navTitle"> | ||
| 3436 | END | ||
| 3437 | for my $C(sort keys $xref->otherMeta->{$f}{$n}->%*) # Content of other meta | ||
| 3438 | {my @c = split /\s+/, $C; # Von Sabine gefunden! | ||
| 3439 | for my $c(sort @c) | ||
| 3440 | {push @o, <<END; | ||
| 3441 | <subjectref keyref="$c"></subjectref> | ||
| 3442 | END | ||
| 3443 | $$classification{$navTitle}{$c}{$f}++ # Classification scheme for file | ||
| 3444 | } | ||
| 3445 | } | ||
| 3446 | push @o, <<END; | ||
| 3447 | </topicsubject> | ||
| 3448 | END | ||
| 3449 | } | ||
| 3450 | |||
| 3451 | push @o, <<END; | ||
| 3452 | </topicref> | ||
| 3453 | END | ||
| 3454 | } | ||
| 3455 | |||
| 3456 | push @o, q(</map>); | ||
| 3457 | |||
| 3458 | if (1) # Write classification map | ||
| 3459 | {my $o = join "\n", @o; | ||
| 3460 | my $x = Data::Edit::Xml::new $o; | ||
| 3461 | my $y = $x->ditaPrettyPrintWithHeaders; | ||
| 3462 | owf($out, $y); | ||
| 3463 | } | ||
| 3464 | |||
| 3465 | if (my $x = Data::Edit::Xml::new($b)) # Edit an xml file retaining any existing XML headers and lint trailers | ||
| 3466 | {my $href = relFromAbsAgainstAbs($out, $b); | ||
| 3467 | my $topicRef = <<END; | ||
| 3468 | <mapref href="$href" processing-role="resource-only" format="ditamap" type="xml" outputclass="classification-ditamap" scope="local"/> | ||
| 3469 | END | ||
| 3470 | |||
| 3471 | if (my $t = $x->go_topicmeta || $x->go_title) # Position the classification map reference | ||
| 3472 | {$t->putNextAsText($topicRef); | ||
| 3473 | editXml($b, $b, -p $x); | ||
| 3474 | } | ||
| 3475 | else # Put the classification map reference first if no other place for it | ||
| 3476 | {$x->putFirstAsText($topicRef); | ||
| 3477 | editXml($b, $b, -p $x); | ||
| 3478 | } | ||
| 3479 | } | ||
| 3480 | } | ||
| 3481 | |||
| 3482 | } # createClassificationMap | ||
| 3483 | |||
| 3484 | sub createClassificationMaps($) #P Create classification maps for each bookmap | ||
| 3485 | {my ($xref) = @_; # Cross referencer | ||
| 3486 | return {} unless $xref->classificationMaps; # Only if requested | ||
| 3487 | |||
| 3488 | my $c = {}; # Classification hash | ||
| 3489 | for my $b(sort keys $xref->bookMapRefs->%*) # Ideally this should be in parallel - but it only takes 1/10 of the Xref time and is not used in most Xrefs anyway so it can probably continue in series for the moment at 2019.11.04 | ||
| 3490 | {createClassificationMap($xref, $b, $c); | ||
| 3491 | } | ||
| 3492 | writeClassificationHtml($xref, $c); # Write html to show the classification scheme | ||
| 3493 | |||
| 3494 | {} # Consolidated meta data | ||
| 3495 | } # createClassificationMaps | ||
| 3496 | |||
| 3497 | sub reportSimilarTopicsByTitle($) #P Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names | ||
| 3498 | {my ($xref) = @_; # Cross referencer | ||
| 3499 | |||
| 3500 | my %t; | ||
| 3501 | for my $File(@{$xref->inputFiles}) # Each input file | ||
| 3502 | {my $F = fn $File; | ||
| 3503 | my $f = $F =~ s([0-9a-f]{32}\Z) (_)gsr; # Remove md5 sum from file name | ||
| 3504 | $t{$f}{$F}++; | ||
| 3505 | } | ||
| 3506 | |||
| 3507 | for my $t(keys %t) # Eliminate files that have no similar counter parts | ||
| 3508 | {if (keys (%{$t{$t}}) <= 1) | ||
| 3509 | {delete $t{$t}; | ||
| 3510 | } | ||
| 3511 | } | ||
| 3512 | |||
| 3513 | my @t; | ||
| 3514 | for my $t(keys %t) # Target | ||
| 3515 | {for my $s(keys %{$t{$t}}) # Source | ||
| 3516 | {push @t, [scalar(keys %{$t{$t}}), $t, $s]; | ||
| 3517 | } | ||
| 3518 | } | ||
| 3519 | |||
| 3520 | my $t = [sort {$b->[0] <=> $a->[0]} # Order report so that most numerous are first | ||
| 3521 | sort {$a->[1] cmp $b->[1]} @t]; | ||
| 3522 | |||
| 3523 | for my $i(keys @$t) # Deduplicate first column from third column | ||
| 3524 | {next unless $i; | ||
| 3525 | my $a = $t->[$i-1]; | ||
| 3526 | my $b = $t->[$i]; | ||
| 3527 | $b->[0] = '' if $a->[1] eq $b->[1]; | ||
| 3528 | } | ||
| 3529 | |||
| 3530 | formatTables($xref, $t, | ||
| 3531 | columns => <<END, | ||
| 3532 | Similar The number of topics similar to this one | ||
| 3533 | Prefix The prefix of the target file names being used for matching | ||
| 3534 | Source Topics that have the current prefix | ||
| 3535 | END | ||
| 3536 | title => qq(Similar topics), | ||
| 3537 | head => <<END, | ||
| 3538 | Xref found NNNN topics that might be similar on DDDD | ||
| 3539 | END | ||
| 3540 | clearUpLeft => -1, summarize=>1, zero=>1, | ||
| 3541 | file => fpe(qw(similar byTitle txt))); | ||
| 3542 | {} # From multiverse to universe | ||
| 3543 | } # reportSimilarTopicsByTitle | ||
| 3544 | |||
| 3545 | sub reportSimilarTopicsByVocabulary($) #P Report topics likely to be similar on the basis of their vocabulary | ||
| 3546 | {my ($xref) = @_; # Cross referencer | ||
| 3547 | my $l = $xref->matchTopics; # Match level | ||
| 3548 | my $p = int($l * 100); # Match level as a percentage | ||
| 3549 | |||
| 3550 | my @m = grep {scalar(@$_) > 1} # Partition into like topics based on vocabulary - select the partitions with more than one element | ||
| 3551 | setPartitionOnIntersectionOverUnionOfHashStringSets($l, $xref->vocabulary); | ||
| 3552 | |||
| 3553 | my @t; | ||
| 3554 | for my $a(@m) # Each block of matching topics | ||
| 3555 | {my ($first, @rest) = @$a; | ||
| 3556 | push @t, [scalar(@$a), $first], map {[q(), $_]} @rest; | ||
| 3557 | push @t, [q(), q()]; | ||
| 3558 | } | ||
| 3559 | |||
| 3560 | my $m = @m; | ||
| 3561 | |||
| 3562 | formatTables($xref, \@t, | ||
| 3563 | columns => <<END, | ||
| 3564 | Similar The number of similar topics in this block | ||
| 3565 | Topic One of the similar topics | ||
| 3566 | END | ||
| 3567 | title=>qq(Topics with similar vocabulary with $p % confidence), | ||
| 3568 | head=><<END, | ||
| 3569 | Xref found $m groups of topics that have similar vocabulary with $p % confidence on DDDD | ||
| 3570 | END | ||
| 3571 | clearUpLeft => -1, summarize=>1, zero=>1, | ||
| 3572 | file=>(my $f = fpe(qw(similar byVocabulary txt)))); | ||
| 3573 | {} # From multiverse to universe | ||
| 3574 | } # reportSimilarTopicsByVocabulary | ||
| 3575 | |||
| 3576 | sub reportWordsByFile($) #P Index words to the files they occur in | ||
| 3577 | {my ($xref) = @_; # Cross referencer | ||
| 3578 | |||
| 3579 | my %w; # Words to files | ||
| 3580 | for my $f(sort keys $xref->vocabulary->%*) # Each file containing words | ||
| 3581 | {my $t = lc $xref->vocabulary->{$f}; | ||
| 3582 | my @w = split /\s+/, $t =~ s([^a-z]) ( )gsr; # Only allow letters in words | ||
| 3583 | for my $w(@w) | ||
| 3584 | {$w{$w}{$f}++ if length($w) < 16; # No one will want to type 16 letter words | ||
| 3585 | } | ||
| 3586 | } | ||
| 3587 | |||
| 3588 | my $N = keys %w; # Number of words | ||
| 3589 | for my $w(keys %w) # Delete useless words == appears in more than the square root of the number of files as they are not selective enough | ||
| 3590 | {my $n = keys %{$w{$w}}; | ||
| 3591 | if ($n * $n > $N) # Not selective enough | ||
| 3592 | {delete $w{$w}; | ||
| 3593 | } | ||
| 3594 | } | ||
| 3595 | |||
| 3596 | my %t; # Files to words | ||
| 3597 | for my $w(keys %w) # Delete useless words == appears in more than the square root of the number of files as they are not selective enough | ||
| 3598 | {for my $t(keys %{$w{$w}}) | ||
| 3599 | {$t{$t}{$w} = $w{$w}{$t}; | ||
| 3600 | } | ||
| 3601 | } | ||
| 3602 | |||
| 3603 | if (my $out = $xref->indexWordsFolder) # Save the indexed words | ||
| 3604 | {makePath($out); | ||
| 3605 | store \%w, fpe($out, qw(words_to_topics data)); # Index word to topics | ||
| 3606 | store \%t, fpe($out, qw(topics_to_words data)); # Index topics to words | ||
| 3607 | store $xref->title, fpe($out, qw(topics_to_titles data)); # Index topics to titles | ||
| 3608 | } | ||
| 3609 | |||
| 3610 | {indexedWords => $N, # Retain number of indexed words - if we need the actual index it is in the file | ||
| 3611 | } # From multiverse to universe | ||
| 3612 | } # reportWordsByFile | ||
| 3613 | |||
| 3614 | sub reportMd5Sum($) #P Report files with identical md5 sums | ||
| 3615 | {my ($xref) = @_; # Cross referencer | ||
| 3616 | |||
| 3617 | my %m; # {md5}{file}++ | ||
| 3618 | if (my $xrefMd5Sum = $xref->md5Sum) | ||
| 3619 | {for my $f(sort keys %{$xrefMd5Sum}) | ||
| 3620 | {if (my $m = $xrefMd5Sum->{$f}) | ||
| 3621 | {$m{$m}{$f}++; | ||
| 3622 | } | ||
| 3623 | } | ||
| 3624 | } | ||
| 3625 | |||
| 3626 | for my $m(keys %m) # Remove files with unique md5 sums | ||
| 3627 | {if (1 == keys %{$m{$m}}) | ||
| 3628 | {delete $m{$m}; | ||
| 3629 | } | ||
| 3630 | } | ||
| 3631 | |||
| 3632 | my @r; my $M; # Report files with same md5 sum clearing up and left | ||
| 3633 | for my $m(sort keys %m) | ||
| 3634 | {for my $f(sort keys %{$m{$m}}) | ||
| 3635 | {push @r, [$m, scalar(keys %{$m{$m}}), $f] unless $M; | ||
| 3636 | push @r, [q(), q(), $f] if $M; | ||
| 3637 | $M = $m; | ||
| 3638 | } | ||
| 3639 | $M = undef; | ||
| 3640 | } | ||
| 3641 | |||
| 3642 | formatTables($xref, \@r, | ||
| 3643 | columns => <<END, | ||
| 3644 | Md5_Sum The md5 sum in question | ||
| 3645 | Count The number of files with this md5 sum | ||
| 3646 | File A file that has this md5 sum | ||
| 3647 | END | ||
| 3648 | title=>qq(Files with identical md5 sums), | ||
| 3649 | head=><<END, | ||
| 3650 | Xref found NNNN files with identical md5 sums on DDDD | ||
| 3651 | |||
| 3652 | Such files are very likely to be identical and thus duplications of each other. | ||
| 3653 | END | ||
| 3654 | file=>(fpe(qw(bad same_md5_sum txt))), | ||
| 3655 | summarize=>1); | ||
| 3656 | |||
| 3657 | {md5SumDuplicates=>\%m, # From multiverse to universe | ||
| 3658 | } | ||
| 3659 | } # reportMd5Sum | ||
| 3660 | |||
| 3661 | sub reportOlBody($) #P ol under body - indicative of a task | ||
| 3662 | {my ($xref) = @_; # Cross referencer | ||
| 3663 | |||
| 3664 | my $select = sub # Select files with specified body | ||
| 3665 | {my ($body) = @_; | ||
| 3666 | my %b = %{$xref->olBody}; | ||
| 3667 | for my $b(keys %b) | ||
| 3668 | {if (my $tag = $xref->baseTag->{$b}) | ||
| 3669 | {if ($tag ne $body) | ||
| 3670 | {delete $b{$b} if $tag ne $body; | ||
| 3671 | } | ||
| 3672 | } | ||
| 3673 | } | ||
| 3674 | %b | ||
| 3675 | }; | ||
| 3676 | |||
| 3677 | my %c = $select->(q(conbody)); | ||
| 3678 | |||
| 3679 | formatTables($xref, | ||
| 3680 | [map {[$c{$_}, $_]} sort {$c{$b} <=> $c{$a}} sort keys %c], | ||
| 3681 | columns => <<END, | ||
| 3682 | Count Number of ol under a conbody tag | ||
| 3683 | File_Name The name of the file containing an ol under conbody | ||
| 3684 | END | ||
| 3685 | title=>qq(ol under conbody indicative of task), | ||
| 3686 | head=><<END, | ||
| 3687 | Xref found NNNN files with ol under a conbody tag on DDDD. | ||
| 3688 | |||
| 3689 | ol under a conbody tag is often indicative of steps in a task. | ||
| 3690 | END | ||
| 3691 | file=>(fpe(qw(bad olUnderConBody txt))), | ||
| 3692 | summarize=>1); | ||
| 3693 | |||
| 3694 | my %t = $select->(q(taskbody)); | ||
| 3695 | |||
| 3696 | formatTables($xref, | ||
| 3697 | [map {[$t{$_}, $_]} sort {$t{$b} <=> $t{$a}} sort keys %t], | ||
| 3698 | columns => <<END, | ||
| 3699 | Count Number of ol under a taskbody tag | ||
| 3700 | File_Name The name of the file containing an ol under taskbody | ||
| 3701 | END | ||
| 3702 | title=>qq(ol under taskbody indicative of steps), | ||
| 3703 | head=><<END, | ||
| 3704 | Xref found NNNN files with ol under a taskbody tag on DDDD. | ||
| 3705 | |||
| 3706 | ol under a taskbody tag is often indicative of steps in a task. | ||
| 3707 | END | ||
| 3708 | file=>(fpe(qw(bad olUnderTaskBody txt))), | ||
| 3709 | summarize=>1); | ||
| 3710 | {} # From multiverse to universe | ||
| 3711 | } # reportOlBody | ||
| 3712 | |||
| 3713 | sub reportHrefUrlEncoding($) #P href needs url encoding | ||
| 3714 | {my ($xref) = @_; # Cross referencer | ||
| 3715 | |||
| 3716 | my @b; | ||
| 3717 | if (my $xrefHrefUrlEncoding = $xref->hrefUrlEncoding) | ||
| 3718 | {for my $f (sort keys %{$xrefHrefUrlEncoding}) | ||
| 3719 | {for my $l(sort keys %{$xrefHrefUrlEncoding->{$f}}) | ||
| 3720 | {push @b, [$xrefHrefUrlEncoding->{$f}{$l}, $l, $f]; | ||
| 3721 | } | ||
| 3722 | } | ||
| 3723 | } | ||
| 3724 | |||
| 3725 | formatTables($xref, [@b], | ||
| 3726 | columns => <<END, | ||
| 3727 | Href Href that needs url encoding | ||
| 3728 | Line_location Line location | ||
| 3729 | File_Name The file containing the href that needs url encoding | ||
| 3730 | END | ||
| 3731 | title=>qq(Hrefs that need url encoding), | ||
| 3732 | head=><<END, | ||
| 3733 | Xref found NNNN locations where an href needs to be url encoded on DDDD. | ||
| 3734 | END | ||
| 3735 | file=>(fpe(qw(bad hrefs_that_need_url_encoding txt))), | ||
| 3736 | summarize=>1); | ||
| 3737 | {} # From multiverse to universe | ||
| 3738 | } # reportHrefUrlEncoding | ||
| 3739 | |||
| 3740 | sub reportConRefMatching($) #P Report conref matching | ||
| 3741 | {my ($xref) = @_; # Cross referencer | ||
| 3742 | |||
| 3743 | my %r; # The number of references to each target id | ||
| 3744 | if (my $xrefConRefs = $xref->conRefs) | ||
| 3745 | {for my $file(sort keys %{$xrefConRefs}) # Each file that has a conref | ||
| 3746 | {for my $ref (sort keys %{$xrefConRefs->{$file}}) # Each conref | ||
| 3747 | {my ($rf, $rt, $ri) = parseDitaRef($ref, $file); # Parse the dita ref | ||
| 3748 | $r{$rf}{$ri}++; # Count the number of references to this id | ||
| 3749 | } | ||
| 3750 | } | ||
| 3751 | } | ||
| 3752 | |||
| 3753 | my @r; my $N = 0; # Number of tags mismatched | ||
| 3754 | if (my $xrefConRefs = $xref->conRefs) | ||
| 3755 | {for my $file(sort keys %{$xrefConRefs}) # Each file that has a conref | ||
| 3756 | {for my $ref(sort keys %{$xrefConRefs->{$file}}) # Each conref target file | ||
| 3757 | {my ($rf, $rt, $ri) = parseDitaRef($ref, $file); # Parse the dita ref | ||
| 3758 | if (my $targetTags = $xref->idTags->{$rf}{$ri}) # Array of target tags that have this id | ||
| 3759 | {for my $t(@$targetTags) # Each target tag | ||
| 3760 | {for my $st(sort keys %{$xrefConRefs->{$file}{$ref}}) # Each conref | ||
| 3761 | {$N++ if my $tagMisMatch = $st ne $t ? q(**) : q(); # Check for mismatch between source and target tags | ||
| 3762 | push @r, [$r{$rf}{$ri}, $tagMisMatch, $st, $t, $ref, $file, $rf]; | ||
| 3763 | } | ||
| 3764 | } | ||
| 3765 | } | ||
| 3766 | } | ||
| 3767 | } | ||
| 3768 | } | ||
| 3769 | |||
| 3770 | formatTables($xref, [@r], | ||
| 3771 | columns => <<END, | ||
| 3772 | Count Number of references to this target | ||
| 3773 | Mismatch ** if there is a mismatch between the source and target tags | ||
| 3774 | Source_tag The source tag | ||
| 3775 | Target_tag The target tag | ||
| 3776 | Reference The reference value from source to target | ||
| 3777 | Source_File The source file | ||
| 3778 | Target_File The target file | ||
| 3779 | END | ||
| 3780 | title=>qq(Conrefs), | ||
| 3781 | head=><<END.fixingRun($xref), | ||
| 3782 | Xref found NNNN conrefs on DDDD with $N mismatches** | ||
| 3783 | END | ||
| 3784 | file=>(my $f = fpe(qw(lists conrefs txt))), | ||
| 3785 | summarize=>1); | ||
| 3786 | |||
| 3787 | {} # From multiverse to universe | ||
| 3788 | } # reportConRefMatching | ||
| 3789 | |||
| 3790 | sub reportPublicIds($) #P Report public ids in use | ||
| 3791 | {my ($xref) = @_; # Cross referencer | ||
| 3792 | |||
| 3793 | my %p; my $missing = 0; # Public Id counts | ||
| 3794 | if (my $xrefPublicId = $xref->publicId) | ||
| 3795 | {for my $f(sort keys %{$xrefPublicId}) # Each file | ||
| 3796 | {if (my $p = $xrefPublicId->{$f}) | ||
| 3797 | {$p{$p}++ | ||
| 3798 | } | ||
| 3799 | else | ||
| 3800 | {++$missing | ||
| 3801 | } | ||
| 3802 | } | ||
| 3803 | } | ||
| 3804 | |||
| 3805 | my @p; # Public Id counts | ||
| 3806 | for my $p(sort keys %p) | ||
| 3807 | {push @p, [$p{$p}, $p]; | ||
| 3808 | } | ||
| 3809 | |||
| 3810 | formatTables($xref, [@p], | ||
| 3811 | columns => <<END, | ||
| 3812 | Count Number of references to this public id | ||
| 3813 | Public_Id A public Id found in the input corpus | ||
| 3814 | END | ||
| 3815 | title=>qq(Public Ids), | ||
| 3816 | head=><<END, | ||
| 3817 | Xref found NNNN public ids in use on DDDD | ||
| 3818 | |||
| 3819 | $missing files have a missing public id. | ||
| 3820 | END | ||
| 3821 | file=>(my $f = fpe(qw(lists public_ids txt))), | ||
| 3822 | summarize=>1); | ||
| 3823 | |||
| 3824 | {} # From multiverse to universe | ||
| 3825 | } # reportPublicIds | ||
| 3826 | |||
| 3827 | sub reportRequiredCleanUps($) #P Report required clean ups | ||
| 3828 | {my ($xref) = @_; # Cross referencer | ||
| 3829 | |||
| 3830 | my @r; my %r; # Required clean ups | ||
| 3831 | if (my $xrefRequiredCleanUp = $xref->requiredCleanUp) | ||
| 3832 | {for my $f(sort keys %{$xrefRequiredCleanUp}) # Each file | ||
| 3833 | {for my $t(sort keys %{$xrefRequiredCleanUp->{$f}}) # Each clean up | ||
| 3834 | {push @r, [$xrefRequiredCleanUp->{$f}{$t}, firstNChars($t, 80), $f]; | ||
| 3835 | $r{$t}++; | ||
| 3836 | } | ||
| 3837 | } | ||
| 3838 | } | ||
| 3839 | |||
| 3840 | my $N = scalar keys %r; | ||
| 3841 | my $F = scalar keys %{$xref->requiredCleanUp}; | ||
| 3842 | |||
| 3843 | formatTables($xref, [@r], | ||
| 3844 | columns => <<END, | ||
| 3845 | Count Number of required clean ups | ||
| 3846 | Clean_Up The text of the clean up request | ||
| 3847 | File_Name File name containing the required clean up requests | ||
| 3848 | END | ||
| 3849 | title => qq(Required cleans ups by file), | ||
| 3850 | head => <<END, | ||
| 3851 | Xref found $F files with $N required-cleanups on DDDD | ||
| 3852 | END | ||
| 3853 | file=>(fpe(qw(lists required_clean_ups txt))), | ||
| 3854 | summarize=>1); | ||
| 3855 | |||
| 3856 | {} # From multiverse to universe | ||
| 3857 | } # reportRequiredCleanUps | ||
| 3858 | |||
| 3859 | sub reportUrls($) #P Report urls that fail to resolve | ||
| 3860 | {my ($xref) = @_; # Cross referencer | ||
| 3861 | |||
| 3862 | my %u; # Urls | ||
| 3863 | for my $f(sort keys $xref->urls->%*) # Each file | ||
| 3864 | {for my $u(sort keys$xref->urls->{$f}->%*) # Each url | ||
| 3865 | {$u{$u}{$f}++; # Urls by file | ||
| 3866 | } | ||
| 3867 | } | ||
| 3868 | |||
| 3869 | my %f; # Failing urls | ||
| 3870 | runInParallel($xref->maximumNumberOfProcesses, sub | ||
| 3871 | {my ($url) = @_; | ||
| 3872 | qx(curl -Is --connect-timeout 5 $url 2>&1 1>/dev/null); | ||
| 3873 | [$url, $?] # Fail if non zero return code | ||
| 3874 | }, sub | ||
| 3875 | {my (@results) = @_; # Consolidate results | ||
| 3876 | for my $r(@results) | ||
| 3877 | {my ($url, $code) = @$r; | ||
| 3878 | next unless $code; | ||
| 3879 | $f{$url}++; | ||
| 3880 | } | ||
| 3881 | }, sort keys %u); | ||
| 3882 | |||
| 3883 | my %b; my %g; # {url}{file} ++ : bad or good | ||
| 3884 | for my $f(sort keys $xref->urls->%*) # Each file | ||
| 3885 | {for my $u(sort keys$xref->urls->{$f}->%*) # Each url | ||
| 3886 | {$b{$u}{$f}++ if $f{$u}; # Bad url | ||
| 3887 | $g{$u}{$f}++ if !$f{$u}; # Good url | ||
| 3888 | } | ||
| 3889 | } | ||
| 3890 | |||
| 3891 | my @bad; my @good; my $bu = 0; my $buf = 0; my $gu = 0; my $guf = 0; # Report url status | ||
| 3892 | for my $u(sort keys %u) | ||
| 3893 | {if ($f{$u}) # Failing url | ||
| 3894 | {push @bad, [$u]; # Url | ||
| 3895 | ++$bu; | ||
| 3896 | for my $f(sort keys $u{$u}->%*) # Occurs in these files | ||
| 3897 | {push @bad, [q(), $f]; | ||
| 3898 | ++$buf; | ||
| 3899 | } | ||
| 3900 | } | ||
| 3901 | else # Successful url | ||
| 3902 | {push @good, [$u]; | ||
| 3903 | ++$gu; | ||
| 3904 | for my $f(sort keys $u{$u}->%*) # Occurs in these files | ||
| 3905 | {push @good, [q(), $f]; | ||
| 3906 | ++$guf; | ||
| 3907 | } | ||
| 3908 | } | ||
| 3909 | } | ||
| 3910 | |||
| 3911 | formatTables($xref, [@bad], | ||
| 3912 | columns => <<END, | ||
| 3913 | Url The url being tested | ||
| 3914 | Files The files the url occurs in | ||
| 3915 | END | ||
| 3916 | title => qq(Urls that fail), | ||
| 3917 | head => <<END, | ||
| 3918 | Xref found $bu failing urls in $buf files on DDDD | ||
| 3919 | END | ||
| 3920 | file=>(fpe(qw(bad urls txt))), | ||
| 3921 | summarize=>1); | ||
| 3922 | |||
| 3923 | formatTables($xref, [@good], | ||
| 3924 | columns => <<END, | ||
| 3925 | Url The url being tested | ||
| 3926 | Files The files the url occurs in | ||
| 3927 | END | ||
| 3928 | title => qq(Urls that pass), | ||
| 3929 | head => <<END, | ||
| 3930 | Xref found $gu good urls in $guf files on DDDD | ||
| 3931 | END | ||
| 3932 | file=>(fpe(qw(good urls txt))), | ||
| 3933 | summarize=>1); | ||
| 3934 | |||
| 3935 | {urlsBad => {%b}, | ||
| 3936 | urlsGood => {%g}, | ||
| 3937 | } # From multiverse to universe | ||
| 3938 | } # reportUrls | ||
| 3939 | |||
| 3940 | sub addNavTitlesToOneMap($$) #P Fix navtitles in one map | ||
| 3941 | {my ($xref, $file) = @_; # Xref results, file to fix | ||
| 3942 | my $changes = 0; # Number of successful changes | ||
| 3943 | my @r; # Count of tags changed | ||
| 3944 | |||
| 3945 | my $x = Data::Edit::Xml::new($file); # Parse xml - should parse OK else otherwise how did we find out that this file needed to be fixed | ||
| 3946 | |||
| 3947 | $x->by(sub # Each node | ||
| 3948 | {my ($o) = @_; | ||
| 3949 | if ($o->at(qr(\A(appendix|chapter|mapref|topicref)\Z)is)) # Nodes that take nav titles | ||
| 3950 | |||
| 3951 | {if (my $h = $o->href) # href to target | ||
| 3952 | {if ($h =~ m(\AGUID-)is) # Target by guid | ||
| 3953 | {if (my $target = $xref->guidToFile->{$h}) # Absolute target name | ||
| 3954 | {if (my $title = $xref->title->{$target}) # Nav title | ||
| 3955 | {$o->set(navtitle=>$title); # Set nav title | ||
| 3956 | push @r, [q(set by guid), $h, $title, $target, $file]; # Record set | ||
| 3957 | ++$changes; | ||
| 3958 | } | ||
| 3959 | else # No such target file | ||
| 3960 | {push @r, [q(No title for guid target), -A $o, $target, $file]; | ||
| 3961 | } | ||
| 3962 | } | ||
| 3963 | else # No mapping from guid to target file | ||
| 3964 | {push @r, [q(No file for guid), -A $o, $target, $file]; | ||
| 3965 | } | ||
| 3966 | } | ||
| 3967 | else # Target by file name | ||
| 3968 | {my $target = absFromAbsPlusRel($file, $h); # Absolute target name | ||
| 3969 | if (my $title = $xref->title->{$target}) # Nav title | ||
| 3970 | {$o->set(navtitle=>$title); # Set nav title | ||
| 3971 | push @r, [q(set), $h, $title, $target, $file]; # Record set | ||
| 3972 | ++$changes; | ||
| 3973 | } | ||
| 3974 | else | ||
| 3975 | {push @r, [q(No title for target), -A $o, $target, $file]; | ||
| 3976 | } | ||
| 3977 | } | ||
| 3978 | } | ||
| 3979 | else | ||
| 3980 | {push @r, [q(No href), -A $o, q(), $file]; | ||
| 3981 | } | ||
| 3982 | } | ||
| 3983 | }); | ||
| 3984 | |||
| 3985 | if ($changes) # Replace xml in source file if we changed anything successfully | ||
| 3986 | {editXml($file, $file, -p $x); # Edit xml | ||
| 3987 | } | ||
| 3988 | |||
| 3989 | \@r # Return report of actions taken | ||
| 3990 | } # addNavTitlesToOneMap | ||
| 3991 | |||
| 3992 | sub addNavTitlesToMaps($) #P Add nav titles to files containing maps. | ||
| 3993 | {my ($xref) = @_; # Xref results | ||
| 3994 | my @r; # Additions made | ||
| 3995 | my @files = | ||
| 3996 | sort | ||
| 3997 | grep {$xref->baseTag->{$_} =~ m(map\Z)s} # Files containing maps | ||
| 3998 | keys %{$xref->baseTag}; # Files with any base tags | ||
| 3999 | |||
| 4000 | if (@files) # Add nav titles to files | ||
| 4001 | {processFilesInParallel # Process each file in parallel | ||
| 4002 | sub | ||
| 4003 | {my ($file) = @_; # File to process | ||
| 4004 | $xref->addNavTitlesToOneMap($file); # Process one input file | ||
| 4005 | }, | ||
| 4006 | sub {push @r, deSquareArray(@_)}, @files; | ||
| 4007 | } | ||
| 4008 | |||
| 4009 | my @Bad; | ||
| 4010 | my @Good; | ||
| 4011 | for my $r(@r) | ||
| 4012 | {if ($$r[0] =~ m(\ANo)s) | ||
| 4013 | {push @Bad, $r; | ||
| 4014 | } | ||
| 4015 | else | ||
| 4016 | {shift @$r; | ||
| 4017 | push @Good, $r; | ||
| 4018 | } | ||
| 4019 | } | ||
| 4020 | |||
| 4021 | my @bad = sort {$$a[3] cmp $$b[3]} sort {$$a[1] cmp $$b[1]} # Sort results else we will get them in varying orders | ||
| 4022 | sort {$$a[0] cmp $$b[0]} @Bad; | ||
| 4023 | my @good = sort {$$a[2] cmp $$b[2]} sort {$$a[0] cmp $$b[0]} | ||
| 4024 | sort {$$a[3] cmp $$b[3]} sort {$$a[1] cmp $$b[1]} @Good; | ||
| 4025 | |||
| 4026 | formatTables($xref, $xref->badNavTitles = \@bad, | ||
| 4027 | columns => <<END, | ||
| 4028 | Reason The reason why a nav title was not added | ||
| 4029 | Statement The source xml statement requesting a navtitle | ||
| 4030 | Title The title of the the navtitle attribute | ||
| 4031 | Target_File The target of the href | ||
| 4032 | Source_File The source file being editted | ||
| 4033 | END | ||
| 4034 | summarize=>1, | ||
| 4035 | title=>qq(Failing Nav titles), | ||
| 4036 | head=><<END, | ||
| 4037 | Xref was unable to add NNNN navtitles as requested by the addNavTitles attribute on DDDD | ||
| 4038 | END | ||
| 4039 | file=>(my $f = fpe(qw(bad nav_titles txt)))); | ||
| 4040 | |||
| 4041 | formatTables($xref, $xref->goodNavTitles = \@good, | ||
| 4042 | columns => <<END, | ||
| 4043 | Statement The source xml statement requesting a navtitle | ||
| 4044 | Title The title of the the navtitle attribute | ||
| 4045 | Target_File The target of the href | ||
| 4046 | Source_File The source file being editted | ||
| 4047 | END | ||
| 4048 | summarize=>1, | ||
| 4049 | title=>qq(Succeding Nav titles), | ||
| 4050 | head=><<END, | ||
| 4051 | Xref was able to add NNNN navtitles as requested by the addNavTitles parameter on DDDD | ||
| 4052 | END | ||
| 4053 | file=>(fpe(qw(good nav_titles txt)))); | ||
| 4054 | |||
| 4055 | {badNavTitles => $xref->badNavTitles, | ||
| 4056 | goodNavTitles => $xref->goodNavTitles, | ||
| 4057 | } # From multiverse to universe | ||
| 4058 | } # addNavTitlesToMaps | ||
| 4059 | |||
| 4060 | sub oxygenProjectFileMetaData #P Meta data for the oxygen project files | ||
| 4061 | {if (isSubInPackage(q(Data::Edit::Xml::Xref), q(xprMetaData))) | ||
| 4062 | {return &Data::Edit::Xml::Xref::xprMetaData(@_) # Call supplied sub to get optional meta data | ||
| 4063 | } | ||
| 4064 | q(<meta/>) # Default | ||
| 4065 | } # oxygenProjectFileMetaData | ||
| 4066 | |||
| 4067 | sub createOxygenProjectFile($$$) #P Create an Oxygen project file for the specified bookmap | ||
| 4068 | {my ($xref, $bm, $xprName) = @_; # Xref, Bookmap, xpr name from bookmap | ||
| 4069 | |||
| 4070 | my @mapRefs = ($bm); # Include this map in the project file | ||
| 4071 | |||
| 4072 | my $extractRefs = sub # Extract references | ||
| 4073 | {my ($field) = @_; # Field to extract | ||
| 4074 | my %hash; | ||
| 4075 | |||
| 4076 | for my $bm(@mapRefs) # Initial plus referenced book maps | ||
| 4077 | {for my $file(sort keys %{$xref->topicsReferencedFromBookMaps->{$bm}}) # Topics from referenced book maps | ||
| 4078 | {for my $href(sort keys %{$xref->{$field}->{$file}}) # Href | ||
| 4079 | {if (my ($t) = parseDitaRef($href, $file)) # Fully qualified target | ||
| 4080 | {$hash{$t}++; # Referenced file | ||
| 4081 | } | ||
| 4082 | } | ||
| 4083 | } | ||
| 4084 | } | ||
| 4085 | %hash | ||
| 4086 | }; | ||
| 4087 | |||
| 4088 | my $extractRefsAndWrapWithFileName = sub # Extract some hrefs from the xref and wrap them with file name relative to the specified absolute file | ||
| 4089 | {my ($field, $xpr) = @_; # Field to extract, oxygen project file | ||
| 4090 | my %hash = &$extractRefs($field, $xpr); | ||
| 4091 | |||
| 4092 | my @f; # File names | ||
| 4093 | for my $file(sort keys %hash) | ||
| 4094 | {my $r = relFromAbsAgainstAbs $file, $xpr; | ||
| 4095 | push @f, qq(<file name="$r"/>); | ||
| 4096 | } | ||
| 4097 | |||
| 4098 | join "\n", @f; | ||
| 4099 | }; | ||
| 4100 | |||
| 4101 | my $extractRefsAndWrapWithFolderName = sub # Extract some hrefs from the xref and wrap them with containing folder names | ||
| 4102 | {my ($field, $xpr) = @_; # Field to extract, oxygen project file | ||
| 4103 | my %hash = &$extractRefs($field, $xpr); | ||
| 4104 | |||
| 4105 | my %f; # File names | ||
| 4106 | for my $file(sort keys %hash) | ||
| 4107 | {my $r = fp relFromAbsAgainstAbs $file, $xpr; | ||
| 4108 | $f{$r}++ | ||
| 4109 | } | ||
| 4110 | |||
| 4111 | my @f; # Folder names | ||
| 4112 | for my $f(sort keys %f) | ||
| 4113 | {push @f, qq(<folder path="$f"/>); | ||
| 4114 | } | ||
| 4115 | |||
| 4116 | join "\n", @f; | ||
| 4117 | }; | ||
| 4118 | |||
| 4119 | my $extractTopicsNotConrefs = sub # Extract non conreffed topicrefs | ||
| 4120 | {my ($xpr) = @_; # Project file being built | ||
| 4121 | my %conRefs = &$extractRefs(q(conRefs), $xpr); | ||
| 4122 | |||
| 4123 | my %p; my %s; # File names, files already seen | ||
| 4124 | for my $bm(@mapRefs) # Each bookmap in this xpr | ||
| 4125 | {for my $file(sort keys $xref->topicsReferencedFromBookMaps->{$bm}->%*) # Field to extract | ||
| 4126 | {if (my $docType = $xref->docType->{$file}) # Include files only once even if they are referenced multiple times | ||
| 4127 | {if ($docType !~ m(map\Z)s and !$s{$file}++) # Include files only once even if they are referenced multiple times | ||
| 4128 | {if (!$conRefs{$file}) | ||
| 4129 | {my $t = detagString($xref->title->{$file} // $file); | ||
| 4130 | my $r = relFromAbsAgainstAbs $file, $xpr; | ||
| 4131 | $p{fp $r}++ | ||
| 4132 | } | ||
| 4133 | } | ||
| 4134 | } | ||
| 4135 | } | ||
| 4136 | } | ||
| 4137 | |||
| 4138 | my @p; # Folder names | ||
| 4139 | for my $p(sort keys %p) | ||
| 4140 | {push @p, qq(<folder path="$p"/>); | ||
| 4141 | } | ||
| 4142 | join "\n", sort @p; | ||
| 4143 | }; | ||
| 4144 | |||
| 4145 | my $extractTopicsNotConrefs22 = sub # Extract non conreffed topicrefs | ||
| 4146 | {my ($xpr) = @_; # Project file being built | ||
| 4147 | my %conRefs = &$extractRefs(q(conRefs), $xpr); | ||
| 4148 | |||
| 4149 | my @f; my %s; # File names, files already seen | ||
| 4150 | for my $bm(@mapRefs) # Each bookmap in this xpr | ||
| 4151 | {for my $file(sort keys $xref->topicsReferencedFromBookMaps->{$bm}->%*) # Field to extract | ||
| 4152 | {if (my $docType = $xref->docType->{$file}) # Include files only once even if they are referenced multiple times | ||
| 4153 | {if ($docType !~ m(map\Z)s and !$s{$file}++) # Include files only once even if they are referenced multiple times | ||
| 4154 | {if (!$conRefs{$file}) | ||
| 4155 | {my $t = detagString($xref->title->{$file} // $file); | ||
| 4156 | my $r = relFromAbsAgainstAbs $file, $xpr; | ||
| 4157 | push @f, <<END | ||
| 4158 | <file name="$r"/> | ||
| 4159 | END | ||
| 4160 | } | ||
| 4161 | } | ||
| 4162 | } | ||
| 4163 | } | ||
| 4164 | } | ||
| 4165 | |||
| 4166 | join "\n", sort @f; | ||
| 4167 | }; | ||
| 4168 | |||
| 4169 | my $formatTargetsFolder = sub # Recreate the targets folder structure so that they can get from their old folder structure to the new flattened files | ||
| 4170 | {my ($xpr) = @_; # Oxygen project file | ||
| 4171 | |||
| 4172 | my @r; | ||
| 4173 | my $r; $r = sub # Target files for each file in the bookmaps in this project file | ||
| 4174 | {my ($files) = @_; | ||
| 4175 | for my $f(sort keys %$files) | ||
| 4176 | {if (ref($$files{$f})) | ||
| 4177 | {push @r, qq(<folder name="$f">); | ||
| 4178 | &$r($$files{$f}); | ||
| 4179 | push @r, qq(</folder>); | ||
| 4180 | } | ||
| 4181 | else | ||
| 4182 | {my $F = $$files{$f}; # Find first containing bookmap | ||
| 4183 | for my $bm(@mapRefs) | ||
| 4184 | {if ($xref->topicsReferencedFromBookMaps->{$bm}{$F} or | ||
| 4185 | $xref->imagesReferencedFromBookMaps->{$bm}{$F}) | ||
| 4186 | {my $t = detagString($xref->title->{$F} // $f); | ||
| 4187 | my $r = relFromAbsAgainstAbs $F, $xpr; | ||
| 4188 | push @r, <<END; | ||
| 4189 | <folder name="$f" navTitle="$t"> | ||
| 4190 | <file name="$r"/> | ||
| 4191 | </folder> | ||
| 4192 | END | ||
| 4193 | last; | ||
| 4194 | } | ||
| 4195 | } | ||
| 4196 | } | ||
| 4197 | } | ||
| 4198 | }; | ||
| 4199 | |||
| 4200 | join "\n", @r; | ||
| 4201 | }; | ||
| 4202 | |||
| 4203 | my $mapRefsFrom; $mapRefsFrom = sub # Parse referenced bookmaps recursively | ||
| 4204 | {my ($bookMap, $href) = @_; # Bookmap, href in bookmap which refers to this bookmap | ||
| 4205 | |||
| 4206 | my $bm = absFromAbsPlusRel($bookMap, $href); # Referenced bookmap | ||
| 4207 | push @mapRefs, $bm; # Save referenced bookmap | ||
| 4208 | |||
| 4209 | if (my $x = eval {Data::Edit::Xml::new($bm)}) # Parse bookmap | ||
| 4210 | {$x->by(sub # Traverse referenced bookmap | ||
| 4211 | {my ($o, $p, $q) = @_; | ||
| 4212 | if ($o->at_mapref) # Map reference in referenced bookmap | ||
| 4213 | {if (my $h = $o->href) | ||
| 4214 | {&$mapRefsFrom($bm, $h); | ||
| 4215 | } | ||
| 4216 | } | ||
| 4217 | elsif ($o->at_topicref) # Map referenced via topic ref in referenced bookmap | ||
| 4218 | {if ($o->attrX_format =~ m(\Aditamap\Z)i) | ||
| 4219 | {if (my $h = $o->href) | ||
| 4220 | {&$mapRefsFrom($bm, $h); | ||
| 4221 | } | ||
| 4222 | } | ||
| 4223 | } | ||
| 4224 | }); | ||
| 4225 | } | ||
| 4226 | }; | ||
| 4227 | |||
| 4228 | my $x = Data::Edit::Xml::new($bm); # Parse initial bookmap | ||
| 4229 | my $title; # Title from initial bookmap | ||
| 4230 | |||
| 4231 | $x->by(sub # Traverse initial bookmap | ||
| 4232 | {my ($o, $p, $q) = @_; | ||
| 4233 | if ($o->at(qr(title\Z))) | ||
| 4234 | {$title = $o->stringText; | ||
| 4235 | } | ||
| 4236 | elsif ($o->at_mapref) # Map ref in initial bookmap | ||
| 4237 | {if (my $h = $o->href) | ||
| 4238 | {&$mapRefsFrom($bm, $h, 1); | ||
| 4239 | } | ||
| 4240 | } | ||
| 4241 | elsif ($o->at_topicref) # Map referenced via a topic reference in the initial bookmap | ||
| 4242 | {if ($o->attrX_format =~ m(\Aditamap\Z)i) | ||
| 4243 | {if (my $h = $o->href) | ||
| 4244 | {&$mapRefsFrom($bm, $h, 1); | ||
| 4245 | } | ||
| 4246 | } | ||
| 4247 | } | ||
| 4248 | }); | ||
| 4249 | |||
| 4250 | $title //= q(); # Default title | ||
| 4251 | my $out = &$xprName($bm, $title); # Oxygen Project File name from bookmap name or title | ||
| 4252 | |||
| 4253 | my $base = fn $bm; # Remove MD5 sum to create an acceptable name but arbitrary name for the project tree | ||
| 4254 | $base =~ s(_[a-f0-9]{32}\Z) ()i; # Match book map name | ||
| 4255 | my $baseExt = relFromAbsAgainstAbs($bm, $out); # Targeted book map that we are creating an xpr file for | ||
| 4256 | my $name = fpe($base, q(xpr)); # Project tree name | ||
| 4257 | |||
| 4258 | my $mapRefs = join "\n", | ||
| 4259 | map | ||
| 4260 | {q(<file name=").relFromAbsAgainstAbs($_, $out).q("/>) # Xpr file out has references to bookmaps in other folders | ||
| 4261 | } @mapRefs; | ||
| 4262 | |||
| 4263 | my $resources = &$extractRefsAndWrapWithFolderName(q(conRefs), $out); | ||
| 4264 | my $images = &$extractRefsAndWrapWithFileName (q(images), $out); | ||
| 4265 | my $topics = &$extractTopicsNotConrefs ($out); | ||
| 4266 | |||
| 4267 | my $metaData = &oxygenProjectFileMetaData; # Add meta data if we are doing this for real otherwise ignore it as it is bulky and gets in the way | ||
| 4268 | |||
| 4269 | my $X = eval {$x->new(<<END)}; | ||
| 4270 | <?xml version="1.0" encoding="UTF-8"?> | ||
| 4271 | <project version="21.1"> | ||
| 4272 | $metaData | ||
| 4273 | <projectTree name="$name"> | ||
| 4274 | <folder masterFiles="true" name="Master Files"> | ||
| 4275 | $mapRefs | ||
| 4276 | </folder> | ||
| 4277 | <folder name="$title Project"> | ||
| 4278 | <folder name="$title Master Map"> | ||
| 4279 | <file name="$baseExt"/> | ||
| 4280 | </folder> | ||
| 4281 | <folder name="Images"> | ||
| 4282 | $images | ||
| 4283 | </folder> | ||
| 4284 | <folder name="Resources"> | ||
| 4285 | $resources | ||
| 4286 | </folder> | ||
| 4287 | <folder name="Topics"> | ||
| 4288 | $topics | ||
| 4289 | </folder> | ||
| 4290 | </folder> | ||
| 4291 | <folder name="CCX Content Repo (All)"> | ||
| 4292 | <folder path="../"/> | ||
| 4293 | </folder> | ||
| 4294 | </projectTree> | ||
| 4295 | </project> | ||
| 4296 | END | ||
| 4297 | |||
| 4298 | # $X->go_projectTree->last->by(sub # Cut out empty folders under the last folder otherwise we get lots of empty folders that contain no files because the files in them were not relevant to this bookmap | ||
| 4299 | # {my ($f) = @_; | ||
| 4300 | # $f->cutIfEmpty_folder; | ||
| 4301 | # }); | ||
| 4302 | |||
| 4303 | if ($@) | ||
| 4304 | {cluck "Unable to parse generated oxygen project file:\n$@" | ||
| 4305 | } | ||
| 4306 | else | ||
| 4307 | {my $text = Data::Edit::Xml::xmlHeader -p $X; | ||
| 4308 | |||
| 4309 | owf($out, $text); | ||
| 4310 | } | ||
| 4311 | } # createOxygenProjectFile | ||
| 4312 | |||
| 4313 | sub createOxygenProjectMapFiles($) #P Create Oxygen project files from Xref results | ||
| 4314 | {my ($xref) = @_; # Cross referencer | ||
| 4315 | |||
| 4316 | my $xprName = sub # Method of choosing xpr file name | ||
| 4317 | {my $x = $xref->oxygenProjects; # Method specified by caller | ||
| 4318 | if (isSubInPackage(q(Data::Edit::Xml::Xref), q(xprName))) | ||
| 4319 | {return sub {&Data::Edit::Xml::Xref::xprName(@_)} # Call supplied sub to generate xpr file name | ||
| 4320 | } | ||
| 4321 | return sub {my ($bm) = @_; setFileExtension($bm, q(xpr))} unless ref $x; # xpr file mirrors book map | ||
| 4322 | }->(); | ||
| 4323 | |||
| 4324 | processFilesInParallel | ||
| 4325 | sub | ||
| 4326 | {my ($bm) = @_; | ||
| 4327 | createOxygenProjectFile($xref, $bm, $xprName); | ||
| 4328 | }, | ||
| 4329 | sub {}, sort keys $xref->exteriorMaps->%*; # Only for exterior book maps - i.e. only book maps that not referenced by any other book map. | ||
| 4330 | |||
| 4331 | {} # Multiverse to universe | ||
| 4332 | } # createOxygenProjectMapFiles | ||
| 4333 | |||
| 4334 | sub oneBadRef($$$) #P Check one reference and return the first error encountered or B<undef> if no errors encountered. Relies on L<topicIds> to test files present and test the B<topicId> is valid, relies on L<ids> to check that the referenced B<id> is valid. | ||
| 4335 | {my ($xref, $file, $href) = @_; # Cross referencer, file containing reference, reference | ||
| 4336 | |||
| 4337 | my $fileExists = sub # Check that the specified file exists by looking for the topic id which L<Dita> guarantees will exist | ||
| 4338 | {my ($file) = @_; # File to check | ||
| 4339 | return 1 if $xref->topicIds->{$file}; # File exists | ||
| 4340 | my $decodedTarget = wwwDecode($file); # Decode file name by expanding % signs to see if we can get a match | ||
| 4341 | return 2 if $xref->topicIds->{$decodedTarget}; # File exists after decoding % signs | ||
| 4342 | return 3 if -e $file; # Images | ||
| 4343 | undef # Cannot locate file | ||
| 4344 | }; | ||
| 4345 | |||
| 4346 | if ($href =~ m(#)) # Full Dita href | ||
| 4347 | {my $sourceTopicId = $xref->topicIds->{$file}; # Source id for referencing file | ||
| 4348 | my ($target, $topicId, $id) = parseDitaRef($href, $file, $sourceTopicId); # Parse full Dita href | ||
| 4349 | my $targetFile = absFromAbsPlusRel($file, $target//$file); # Absolute target file which might be the current file | ||
| 4350 | my $targetTopicId = $xref->topicIds->{$targetFile}; # Topic Id of target file | ||
| 4351 | |||
| 4352 | my $bad = sub # Report a bad reference | ||
| 4353 | {my ($r) = @_; # Reason | ||
| 4354 | [$r, $href, $target, $topicId, $id, $sourceTopicId, | ||
| 4355 | $targetTopicId, $file, $targetFile]; | ||
| 4356 | }; | ||
| 4357 | |||
| 4358 | return &$bad(q(No such file)) unless &$fileExists($targetFile); # Check target file exists | ||
| 4359 | return &$bad(q(No topic id)) unless $targetTopicId; # Check the target has a topic id | ||
| 4360 | return &$bad(q(Topic id does not match)) | ||
| 4361 | unless $targetTopicId eq $topicId or $topicId eq q(); # Check topic id of referenced topic against supplied topicId. It is safe to assume that the target does have topic id as Dita requires one | ||
| 4362 | |||
| 4363 | if ($id) # Checkid if supplied | ||
| 4364 | {my $i = $xref->ids->{$target}{$id}; # Number of ids in the target topic with this value | ||
| 4365 | return &$bad(q(No such id in target topic)) unless $i; # No such id | ||
| 4366 | return &$bad(q(Duplicated id in target topic)) # Duplicate ids | ||
| 4367 | unless $i == 1 or $i == 2 && $id eq $topicId; # Using dita topic references we can cope with one internal id that duplicates the topic id | ||
| 4368 | } | ||
| 4369 | } | ||
| 4370 | |||
| 4371 | else # No # in href | ||
| 4372 | {my $targetFile = absFromAbsPlusRel($file, $href); | ||
| 4373 | return [q(No such file), $href, $href, q(), q(), q(), q(), | ||
| 4374 | $file, $targetFile] unless &$fileExists($targetFile); # Check target file exists | ||
| 4375 | } | ||
| 4376 | |||
| 4377 | undef # No error to report | ||
| 4378 | } # oneBadRef | ||
| 4379 | |||
| 4380 | #D0 | ||
| 4381 | # podDocumentation | ||
| 4382 | =pod | ||
| 4383 | |||
| 4384 | =encoding utf-8 | ||
| 4385 | |||
| 4386 | =head1 Name | ||
| 4387 | |||
| 4388 | Data::Edit::Xml::Xref - Cross reference Dita XML, match topics and ameliorate missing references. | ||
| 4389 | |||
| 4390 | =head1 Synopsis | ||
| 4391 | |||
| 4392 | L<Xref> scans an entire document corpus looking primarily for problems with | ||
| 4393 | references between the files in the corpus; it reports any opportunities for | ||
| 4394 | improvements it finds and makes changes to the corpus to implement these | ||
| 4395 | improvements if so requested taking advantage of parallelism where ever | ||
| 4396 | possible. | ||
| 4397 | |||
| 4398 | The following example checks the references in a corpus of Dita XML | ||
| 4399 | documents held in folder L<inputFolder|/inputFolder>: | ||
| 4400 | |||
| 4401 | use Data::Edit::Xml::Xref; | ||
| 4402 | |||
| 4403 | my $x = xref(inputFolder => q(in), | ||
| 4404 | fixBadRefs => 1, | ||
| 4405 | flattenFolder => q(out2), | ||
| 4406 | matchTopics => 0.9, | ||
| 4407 | ); | ||
| 4408 | |||
| 4409 | The cross reference analysis can be requested as a L<status line|/statusLine>: | ||
| 4410 | |||
| 4411 | ok nws($x->statusLine) eq nws(<<END); | ||
| 4412 | Xref: 108 references fixed, 50 bad xrefs, 16 missing image files, 16 missing image references, 13 bad first lines, 13 bad second lines, 9 bad conrefs, 9 duplicate topic ids, 9 files with bad conrefs, 9 files with bad xrefs, 8 duplicate ids, 6 bad topicrefs, 6 files not referenced, 4 invalid guid hrefs, 2 bad book maps, 2 bad tables, 1 External xrefs with no format=html, 1 External xrefs with no scope=external, 1 file failed to parse, 1 href missing | ||
| 4413 | END | ||
| 4414 | |||
| 4415 | Or as a tabular report: | ||
| 4416 | |||
| 4417 | ok nws($x->statusTable) eq nws(<<END); | ||
| 4418 | Xref: | ||
| 4419 | Count Condition | ||
| 4420 | 1 108 references fixed | ||
| 4421 | 2 50 bad xrefs | ||
| 4422 | 3 16 missing image files | ||
| 4423 | 4 16 missing image references | ||
| 4424 | 5 13 bad first lines | ||
| 4425 | 6 13 bad second lines | ||
| 4426 | 7 9 files with bad conrefs | ||
| 4427 | 8 9 bad conrefs | ||
| 4428 | 9 9 files with bad xrefs | ||
| 4429 | 10 9 duplicate topic ids | ||
| 4430 | 11 8 duplicate ids | ||
| 4431 | 12 6 bad topicrefs | ||
| 4432 | 13 6 files not referenced | ||
| 4433 | 14 4 invalid guid hrefs | ||
| 4434 | 15 2 bad book maps | ||
| 4435 | 16 2 bad tables | ||
| 4436 | 17 1 href missing | ||
| 4437 | 18 1 file failed to parse | ||
| 4438 | 19 1 External xrefs with no format=html | ||
| 4439 | 20 1 External xrefs with no scope=external | ||
| 4440 | END | ||
| 4441 | |||
| 4442 | More detailed reports are produced in the L<reports|/reports> folder: | ||
| 4443 | |||
| 4444 | $x->reports | ||
| 4445 | |||
| 4446 | and indexed by the L<reports> report: | ||
| 4447 | |||
| 4448 | reports/reports.txt | ||
| 4449 | |||
| 4450 | which contains a list of all the L<reports> generated: | ||
| 4451 | |||
| 4452 | Rows Title File | ||
| 4453 | 1 5 Attributes reports/count/attributes.txt | ||
| 4454 | 2 13 Bad Xml line 1 reports/bad/xmlLine1.txt | ||
| 4455 | 3 13 Bad Xml line 2 reports/bad/xmlLine2.txt | ||
| 4456 | 4 9 Bad conRefs reports/bad/ConRefs.txt | ||
| 4457 | 5 2 Bad external xrefs reports/bad/externalXrefs.txt | ||
| 4458 | 6 16 Bad image references reports/bad/imageRefs.txt | ||
| 4459 | 7 9 Bad topicrefs reports/bad/bookMapRefs.txt | ||
| 4460 | 8 50 Bad xRefs reports/bad/XRefs.txt | ||
| 4461 | 9 2 Bookmaps with errors reports/bad/bookMap.txt | ||
| 4462 | 10 2 Document types reports/count/docTypes.txt | ||
| 4463 | 11 8 Duplicate id definitions within files reports/bad/idDefinitionsDuplicated.txt | ||
| 4464 | 12 3 Duplicate topic id definitions reports/bad/topicIdDefinitionsDuplicated.txt | ||
| 4465 | 13 3 File extensions reports/count/fileExtensions.txt | ||
| 4466 | 14 1 Files failed to parse reports/bad/parseFailed.txt | ||
| 4467 | 15 0 Files types reports/count/fileTypes.txt | ||
| 4468 | 16 16 Files whose short names are bi-jective with their md5 sums reports/good/shortNameToMd5Sum.txt | ||
| 4469 | 17 0 Files whose short names are not bi-jective with their md5 sums reports/bad/shortNameToMd5Sum.txt | ||
| 4470 | 18 108 Fixes Applied To Failing References reports/lists/referencesFixed.txt | ||
| 4471 | 19 0 Good bookmaps reports/good/bookMap.txt | ||
| 4472 | 20 9 Good conRefs reports/good/ConRefs.txt | ||
| 4473 | 21 5 Good topicrefs reports/good/bookMapRefs.txt | ||
| 4474 | 22 8 Good xRefs reports/good/XRefs.txt | ||
| 4475 | 23 1 Guid topic definitions reports/lists/guidsToFiles.txt | ||
| 4476 | 24 2 Image files reports/good/imagesFound.txt | ||
| 4477 | 25 1 Missing hrefs reports/bad/missingHrefAttributes.txt | ||
| 4478 | 26 16 Missing image references reports/bad/imagesMissing.txt | ||
| 4479 | 27 4 Possible improvements reports/improvements.txt | ||
| 4480 | 28 2 Resolved GUID hrefs reports/good/guidHrefs.txt | ||
| 4481 | 29 2 Tables with errors reports/bad/tables.txt | ||
| 4482 | 30 23 Tags reports/count/tags.txt | ||
| 4483 | 31 11 Topic Reuses reports/lists/topicReuse.txt | ||
| 4484 | 32 0 Topic Reuses reports/lists/similar/byTitle.txt | ||
| 4485 | 33 16 Topics reports/lists/topics.txt | ||
| 4486 | 34 15 Topics with similar vocabulary reports/lists/similar/byVocabulary.txt | ||
| 4487 | 35 0 Topics with validation errors reports/bad/validationErrors.txt | ||
| 4488 | 36 0 Topics without ids reports/bad/topicIdDefinitionsMissing.txt | ||
| 4489 | 37 6 Unreferenced files reports/bad/notReferenced.txt | ||
| 4490 | 38 11 Unresolved GUID hrefs reports/bad/guidHrefs.txt | ||
| 4491 | |||
| 4492 | =head2 Add navigation titles to topic references | ||
| 4493 | |||
| 4494 | Xref will create or update the navigation titles B<navtitles> of topic refs | ||
| 4495 | B<appendix|chapter|topicref> in maps if requested by both file name and GUID | ||
| 4496 | reference: | ||
| 4497 | |||
| 4498 | addNavTitle => 1 | ||
| 4499 | |||
| 4500 | Reports of successful updates will be written to: | ||
| 4501 | |||
| 4502 | reports/good/navTitles.txt | ||
| 4503 | |||
| 4504 | Reports of unsuccessful updates will be written to: | ||
| 4505 | |||
| 4506 | reports/bad/navTitles.txt | ||
| 4507 | |||
| 4508 | =head2 Fix bad references | ||
| 4509 | |||
| 4510 | It is often desirable to ameliorate unresolved Dita href attributes so that | ||
| 4511 | incomplete content can be loaded into a content management system. The: | ||
| 4512 | |||
| 4513 | fixBadRefs => 1 | ||
| 4514 | |||
| 4515 | attribute requests that the: | ||
| 4516 | |||
| 4517 | conref and href | ||
| 4518 | |||
| 4519 | attributes be renamed to: | ||
| 4520 | |||
| 4521 | xtrf | ||
| 4522 | |||
| 4523 | if the B<conref> or B<href> attribute specification cannot be resolved in the | ||
| 4524 | current corpus by other methods of fixing failing references such as: | ||
| 4525 | L<fixDitaRefs>, L<fixRelocatedRefs> or L<fixXrefsByTitle>. | ||
| 4526 | |||
| 4527 | This feature designed by L<mailto:mim@cpan.org>. | ||
| 4528 | |||
| 4529 | =head2 Deguidize | ||
| 4530 | |||
| 4531 | Some content management systems use guids, some content management systems use | ||
| 4532 | file names as their means of identifying content. When moving from a guid to a | ||
| 4533 | file name content management system it might be necessary to replace the guids | ||
| 4534 | representing file names with the actual underlying file names. If the | ||
| 4535 | |||
| 4536 | deguidize => 1 | ||
| 4537 | |||
| 4538 | parameter is set to true, Xref will replace any such file guids with the | ||
| 4539 | underlying file name if it is present in the content being cross referenced. | ||
| 4540 | |||
| 4541 | =head2 File flattening | ||
| 4542 | |||
| 4543 | It is often desirable to flatten or reflatten the topic files in a corpus so | ||
| 4544 | that they can coexist in a single folder of a content management system without | ||
| 4545 | colliding with each other. | ||
| 4546 | |||
| 4547 | The presence of the input attribute: | ||
| 4548 | |||
| 4549 | flattenFolder => folder-to-flatten-files-into | ||
| 4550 | |||
| 4551 | causes topic files to be flattened into the named folder using the | ||
| 4552 | L<GBStandard> to generate the flattened file names. Xref will then update all | ||
| 4553 | L<Dita> references to match these new file names. If the L<flattenFolder> | ||
| 4554 | folder is the same as the L<inputFolder> then the input files are flattened in | ||
| 4555 | place. | ||
| 4556 | |||
| 4557 | =head2 Locating relocated files | ||
| 4558 | |||
| 4559 | File references in B<conref> or B<hrefs> that have a unique valid base file | ||
| 4560 | name and an invalid path can be fixed by setting the input attribute: | ||
| 4561 | |||
| 4562 | fixRelocatedRefs => 1 | ||
| 4563 | |||
| 4564 | to a true value to request that Xref should replace the incorrect paths to the | ||
| 4565 | unique bases file names with the correct path. | ||
| 4566 | |||
| 4567 | If coded in conjunction with the B<fixBadRefs> input attribute this will cause | ||
| 4568 | Xref to first try and fix any missing xrefs, any that still fail to resolve | ||
| 4569 | will then be ameliorated by moving them to the B<xtrf> attribute. | ||
| 4570 | |||
| 4571 | =head2 Fix Xrefs by Title | ||
| 4572 | |||
| 4573 | L<Dita> B<xref> tags with broken or missing B<href> attributes can sometimes be | ||
| 4574 | fixed by matching the text content of the B<xref> with the titles of topics. | ||
| 4575 | |||
| 4576 | If: | ||
| 4577 | |||
| 4578 | fixXrefsByTitle => 1 | ||
| 4579 | |||
| 4580 | is specified, L<Xref> will locate possible targets for a broken B<href> by | ||
| 4581 | matching the white space normalized L<Data::Table::Text::nws> of the text | ||
| 4582 | content of the B<xref> with the similarly normalized title of each topic that | ||
| 4583 | is referenced by any book map that refers to the topic containing the B<xref>. | ||
| 4584 | |||
| 4585 | If a single matching candidate is located then it will be used to update the | ||
| 4586 | B<href> attribute of the B<xref>. | ||
| 4587 | |||
| 4588 | =head2 Fix References in Dita To Dita Conversions | ||
| 4589 | |||
| 4590 | When converting a L<Dita> input source corpus to L<Dita> the referenced topics | ||
| 4591 | are usually renamed and flattened via the L<GBStandard>. If enabled: | ||
| 4592 | |||
| 4593 | fixDitaRefs => targets/ | ||
| 4594 | |||
| 4595 | updates valid L<Dita> references in the input corpus with the latest name for | ||
| 4596 | the referenced topic to make links that were valid in the input corpus valid in | ||
| 4597 | the output corpus as well. | ||
| 4598 | |||
| 4599 | The B<targets/> folder should contain the same set of file names as the | ||
| 4600 | original input corpus, each such file should contain the name of a B<bookmap> | ||
| 4601 | topic present in the B<inputFolder=> whose B<chapter> and B<topicref>s identify | ||
| 4602 | the new names of the files cut out and flattened from the existing input | ||
| 4603 | corpus. | ||
| 4604 | |||
| 4605 | The creation of the B<target/> folder is usually done by some other piece of | ||
| 4606 | software such as L<Data::Edit::Xml::To::Dita> as it is too complex and | ||
| 4607 | laborious to be performed reliably by hand. No validation of the contents of | ||
| 4608 | this folder is performed as it is assumed that it has been created reliably in | ||
| 4609 | software. | ||
| 4610 | |||
| 4611 | =head2 Topic Matching | ||
| 4612 | |||
| 4613 | Topics can be matched on title and vocabulary to assist authors in finding | ||
| 4614 | similar topics by specifying the: | ||
| 4615 | |||
| 4616 | matchTopics => 0.9 | ||
| 4617 | |||
| 4618 | attribute where the value of this attribute is the confidence level between 0 | ||
| 4619 | and 1. | ||
| 4620 | |||
| 4621 | Topic matching produces the reports: | ||
| 4622 | |||
| 4623 | reports/lists/similar/byTitle.txt | ||
| 4624 | reports/lists/similar/byVocabulary.txt | ||
| 4625 | |||
| 4626 | Topic matching might take some time for large input folders. | ||
| 4627 | |||
| 4628 | =head3 Title matching | ||
| 4629 | |||
| 4630 | This report can be found at: | ||
| 4631 | |||
| 4632 | reports/lists/similar/byTitle.txt | ||
| 4633 | |||
| 4634 | Title sorts topics by their titles so that topic with similar titles can be | ||
| 4635 | easily located: | ||
| 4636 | |||
| 4637 | Similar Prefix Source | ||
| 4638 | 1 14 c_Notices__ c_Notices_5614e96c7a3eaf3dfefc4a455398361b | ||
| 4639 | 2 c_Notices__ c_Notices_14a9f467215dea879d417de884c21e6d | ||
| 4640 | 3 c_Notices__ c_Notices_19011759a2f768d76581dc3bba170a44 | ||
| 4641 | 4 c_Notices__ c_Notices_aa741e6223e6cf8bc1a5ebdcf0ba867c | ||
| 4642 | 5 c_Notices__ c_Notices_f0009b28c3c273094efded5fac32b83f | ||
| 4643 | 6 c_Notices__ c_Notices_b1480ac1af812da3945239271c579bb1 | ||
| 4644 | 7 c_Notices__ c_Notices_5f3aa15d024f0b6068bd8072d4942f6d | ||
| 4645 | 8 c_Notices__ c_Notices_17c1f39e8d70c765e1fbb6c495bedb03 | ||
| 4646 | 9 c_Notices__ c_Notices_7ea35477554f979b3045feb369b69359 | ||
| 4647 | 10 c_Notices__ c_Notices_4f200259663703065d247b35d5500e0e | ||
| 4648 | 11 c_Notices__ c_Notices_e3f2eb03c23491c5e96b08424322e423 | ||
| 4649 | 12 c_Notices__ c_Notices_06b7e9b0329740fc2b50fedfecbc5a94 | ||
| 4650 | 13 c_Notices__ c_Notices_550a0d84dfc94982343f58f84d1c11c2 | ||
| 4651 | 14 c_Notices__ c_Notices_fa7e563d8153668db9ed098d0fe6357b | ||
| 4652 | 15 3 c_Overview__ c_Overview_f9e554ee9be499368841260344815f58 | ||
| 4653 | 16 c_Overview__ c_Overview_f234dc10ea3f4229d0e1ab4ad5e8f5fe | ||
| 4654 | 17 c_Overview__ c_Overview_96121d7bcd41cf8be318b96da0049e73 | ||
| 4655 | |||
| 4656 | =head3 Vocabulary matching | ||
| 4657 | |||
| 4658 | This report can be found at: | ||
| 4659 | |||
| 4660 | reports/lists/similar/byVocabulary.txt | ||
| 4661 | |||
| 4662 | Vocabulary matching compares the vocabulary of pairs of topics: topics with | ||
| 4663 | similar vocabularies within the confidence level specified are reported | ||
| 4664 | together: | ||
| 4665 | |||
| 4666 | Similar Topic | ||
| 4667 | 1 8 in/1.dita | ||
| 4668 | 2 in/2.dita | ||
| 4669 | 3 in/3.dita | ||
| 4670 | 4 in/4.dita | ||
| 4671 | 5 in/5.dita | ||
| 4672 | 6 in/6.dita | ||
| 4673 | 7 in/7.dita | ||
| 4674 | 8 in/8.dita | ||
| 4675 | 9 | ||
| 4676 | 10 2 in/map/bookmap.ditamap | ||
| 4677 | 11 in/map/bookmap2.ditamap | ||
| 4678 | 12 | ||
| 4679 | 13 2 in/act4. dita | ||
| 4680 | 14 in/act5.dita | ||
| 4681 | |||
| 4682 | =head2 Url checking | ||
| 4683 | |||
| 4684 | Xref will check urls by fetching their headers with L<curl> if the | ||
| 4685 | |||
| 4686 | validateUrls=>1 | ||
| 4687 | |||
| 4688 | is specified. A list of failing L<url>s will be written to: | ||
| 4689 | |||
| 4690 | reports/bad/urls.txt | ||
| 4691 | |||
| 4692 | while a corresponding list of passing L<url>s will be written to | ||
| 4693 | |||
| 4694 | reports/good/urls.txt | ||
| 4695 | |||
| 4696 | =head1 Description | ||
| 4697 | |||
| 4698 | Cross reference Dita XML, match topics and ameliorate missing references. | ||
| 4699 | |||
| 4700 | |||
| 4701 | Version 20200202. | ||
| 4702 | |||
| 4703 | |||
| 4704 | The following sections describe the methods in each functional area of this | ||
| 4705 | module. For an alphabetic listing of all methods by name see L<Index|/Index>. | ||
| 4706 | |||
| 4707 | |||
| 4708 | |||
| 4709 | =head1 Cross reference | ||
| 4710 | |||
| 4711 | Check the cross references in a set of Dita files and report the results. | ||
| 4712 | |||
| 4713 | =head2 xref(%attributes) | ||
| 4714 | |||
| 4715 | Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref>. | ||
| 4716 | |||
| 4717 | Parameter Description | ||
| 4718 | 1 %attributes Cross referencer attribute value pairs | ||
| 4719 | |||
| 4720 | B<Example:> | ||
| 4721 | |||
| 4722 | |||
| 4723 | lll "Test 011"; | ||
| 4724 | clearFolder(tests, 111); | ||
| 4725 | createSampleInputFilesForFixDitaRefsImproved3(tests); | ||
| 4726 | |||
| 4727 | my $y = 𝘅𝗿𝗲𝗳(inputFolder => out, reports => reportFolder); # Check results without fixes | ||
| 4728 | ok $y->statusLine eq q(Xref: 1 ref); | ||
| 4729 | |||
| 4730 | my $x = 𝘅𝗿𝗲𝗳 | ||
| 4731 | (inputFolder => out, | ||
| 4732 | reports => reportFolder, | ||
| 4733 | fixBadRefs => 1, | ||
| 4734 | fixDitaRefs => targets, | ||
| 4735 | fixedFolder => outFixed); | ||
| 4736 | |||
| 4737 | ok !$x->errors; | ||
| 4738 | |||
| 4739 | |||
| 4740 | =head1 Create test data | ||
| 4741 | |||
| 4742 | Create files to test the various capabilities provided by Xref | ||
| 4743 | |||
| 4744 | |||
| 4745 | =head2 Data::Edit::Xml::Xref Definition | ||
| 4746 | |||
| 4747 | |||
| 4748 | Attributes used by the Xref cross referencer. | ||
| 4749 | |||
| 4750 | |||
| 4751 | |||
| 4752 | |||
| 4753 | =head3 Input fields | ||
| 4754 | |||
| 4755 | |||
| 4756 | B<addNavTitles> - If true, add navtitle to outgoing bookmap references to show the title of the target topic. | ||
| 4757 | |||
| 4758 | B<changeBadXrefToPh> - Change xrefs being placed in B<M3> by L<fixBadRefs> to B<ph>. | ||
| 4759 | |||
| 4760 | B<classificationMaps> - Create classification maps if true | ||
| 4761 | |||
| 4762 | B<deguidize> - Set true to replace guids in dita references with file name. Given reference B<g1#g2/id> convert B<g1> to a file name by locating the topic with topicId B<g2>. This requires the guids to be genuinely unique. SDL guids are thought to be unique by language code but the same topic, translated to a different language might well have the same guid as the original topic with a different language code: =(de|en|es|fr). If the source is in just one language then the guid uniqueness is a reasonable assumption. If the conversion can be done in phases by language then the uniqueness of guids is again reasonably assured. L<Data::Edit::Xml::Lint> provides an alternative solution to deguidizing by using labels to record the dita reference in the input corpus for each id encountered, these references can then be resolved in the usual manner by L<Data::Edit::Xml::Lint::relint>. | ||
| 4763 | |||
| 4764 | B<deleteUnusedIds> - Delete ids (except on topics) that are not referenced in any reference in the corpus regardless of the file component of any such reference. | ||
| 4765 | |||
| 4766 | B<fixBadRefs> - Fix any remaining bad references after any all allowed attempts have been made to fix failing references by moving the failing reference to the B<xtrf> attribute i.e. placing it in B<M3> possibly renaming the tag to B<ph> if L<changeBadXrefToPh> is in effect as well. | ||
| 4767 | |||
| 4768 | B<fixDitaRefs> - Fix references in a corpus of L<Dita|http://docs.oasis-open.org/dita/dita/v1.3/os/part2-tech-content/dita-v1.3-os-part2-tech-content.html> documents that have been converted to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard> and whose target structure has been written to the named folder. | ||
| 4769 | |||
| 4770 | B<fixRelocatedRefs> - Fix references to topics that have been moved around in the out folder structure assuming that all file names are unique which they will be if they have been renamed to the GB Standard. | ||
| 4771 | |||
| 4772 | B<fixXrefsByTitle> - Try to fix invalid xrefs by the Gearhart Title Method enhanced by the Monroe map method if true | ||
| 4773 | |||
| 4774 | B<fixedFolder> - Fixed files are placed in this folder. | ||
| 4775 | |||
| 4776 | B<fixedFolderTemp> - Fixed files are placed in this folder if we are on aws but nit the session leader - this folder is then copied back to L<fixedFolder> on the session leader. | ||
| 4777 | |||
| 4778 | B<flattenFolder> - Files are renamed to the Gearhart standard and placed in this folder if set. References to the unflattened files are updated to references to the flattened files. This option will eventually be deprecated as the Dita::GB::Standard is now fully available allowing files to be easily flattened before being processed by Xref. | ||
| 4779 | |||
| 4780 | B<getFileUrl> - A url to retrieve a specified file from the server running xref used in generating html reports. The complete url is obtained by appending the fully qualified file name to this value. | ||
| 4781 | |||
| 4782 | B<html> - Generate html version of reports in this folder if supplied | ||
| 4783 | |||
| 4784 | B<indexWords> - Index words to topics and topics to words if true. | ||
| 4785 | |||
| 4786 | B<indexWordsFolder> - Folder into which to save words to topic and topics to word indexes if L<indexWords> is true. | ||
| 4787 | |||
| 4788 | B<inputFolder> - A folder containing the dita and ditamap files to be cross referenced. | ||
| 4789 | |||
| 4790 | B<matchTopics> - Match topics by title and by vocabulary to the specified confidence level between 0 and 1. This operation might take some time to complete on a large corpus. | ||
| 4791 | |||
| 4792 | B<maxZoomIn> - Optional hash of names to regular expressions to look for in each file | ||
| 4793 | |||
| 4794 | B<maximumNumberOfProcesses> - Maximum number of processes to run in parallel at any one time with a sensible default. | ||
| 4795 | |||
| 4796 | B<oxygenProjects> - Create oxygen project files for each map - the project file will have an extension of .xpr and the same name and path as the map file or the name return by your implementation of: Data::Edit::Xml::Xref::xprName($map) if present. | ||
| 4797 | |||
| 4798 | B<reports> - Reports folder: Xref will write text versions of the generated reports to files in this folder. | ||
| 4799 | |||
| 4800 | B<requestAttributeNameAndValueCounts> - Report attribute name and value counts | ||
| 4801 | |||
| 4802 | B<subjectSchemeMap> - Create a subject scheme map in the named file | ||
| 4803 | |||
| 4804 | B<suppressReferenceChecks> - Suppress reference checking - which normally happens by default - but which takes time and might be irrelevant if an earlier xref has already checked all the references. | ||
| 4805 | |||
| 4806 | B<validateUrls> - Validate urls if true by fetching their headers with L<curl|https://linux.die.net/man/1/curl> | ||
| 4807 | |||
| 4808 | |||
| 4809 | |||
| 4810 | =head3 Output fields | ||
| 4811 | |||
| 4812 | |||
| 4813 | B<allowUniquePartialMatches> - Allow unique partial matches - i.e ignore the stuff to the right of the # in a reference if doing so produces a unique result. This feature has been explicitly disabled for conrefs (PS2-561) and might need to be disabled for other types of reference as well. | ||
| 4814 | |||
| 4815 | B<attributeCount> - {file}{attribute name} == count of the different xml attributes found in the xml files. | ||
| 4816 | |||
| 4817 | B<attributeNamesAndValuesCount> - {file}{attribute name}{value} = count | ||
| 4818 | |||
| 4819 | B<author> - {file} = author of this file. | ||
| 4820 | |||
| 4821 | B<badGuidHrefs> - Bad conrefs - all. | ||
| 4822 | |||
| 4823 | B<badNavTitles> - Details of nav titles that were not resolved | ||
| 4824 | |||
| 4825 | B<badReferencesCount> - The number of bad references at the start of the run - however depending on what options were chosen Xref might ameliorate these bad references and thereby reduce this count. | ||
| 4826 | |||
| 4827 | B<badTables> - Array of tables that need fixing. | ||
| 4828 | |||
| 4829 | B<badXml1> - [Files] with a bad xml encoding header on the first line. | ||
| 4830 | |||
| 4831 | B<badXml2> - [Files] with a bad xml doc type on the second line. | ||
| 4832 | |||
| 4833 | B<baseFiles> - {base of file name}{full file name}++ Current location of the file via uniqueness guaranteed by the GB standard | ||
| 4834 | |||
| 4835 | B<baseTag> - Base Tag for each file | ||
| 4836 | |||
| 4837 | B<bookMapRefs> - {bookmap full file name}{href}{navTitle}++ References from bookmaps to topics via appendix, chapter, bookmapref. | ||
| 4838 | |||
| 4839 | B<conRefs> - {file}{href}{tag}++ : conref source detail | ||
| 4840 | |||
| 4841 | B<createReports1> - Reports requested before references fixed | ||
| 4842 | |||
| 4843 | B<createReports2> - Reports requested after references fixed | ||
| 4844 | |||
| 4845 | B<currentFolder> - The current working folder used to make absolute file names from relative ones | ||
| 4846 | |||
| 4847 | B<docType> - {file} == docType: the docType for each xml file. | ||
| 4848 | |||
| 4849 | B<duplicateIds> - [file, id] Duplicate id definitions within each file. | ||
| 4850 | |||
| 4851 | B<duplicateTopicIds> - [topicId, [files]] Files with duplicate topic ids - the id on the outermost tag. | ||
| 4852 | |||
| 4853 | B<emptyTopics> - {file} : topics where the *body is empty. | ||
| 4854 | |||
| 4855 | B<errors> - Number of significant errors as reported in L<statusLine> or 0 if no such errors found | ||
| 4856 | |||
| 4857 | B<exteriorMaps> - {exterior map} : maps that are not referenced by another map | ||
| 4858 | |||
| 4859 | B<fileExtensions> - Default file extensions to load | ||
| 4860 | |||
| 4861 | B<fixRefs> - {file}{ref} where the href or conref target is not valid. | ||
| 4862 | |||
| 4863 | B<fixedRefsBad> - [] hrefs and conrefs from L<fixRefs|/fixRefs> which were moved to the "xtrf" attribute as requested by the L<fixBadHrefs|/fixBadHrefs> attribute because the reference was invalid and could not be improved by L<deguidization|/deguidize>. | ||
| 4864 | |||
| 4865 | B<fixedRefsGB> - [] files fixed to the Gearhart-Brenan file naming standard | ||
| 4866 | |||
| 4867 | B<fixedRefsGood> - [] hrefs and conrefs from L<fixRefs|/fixRefs> which were invalid but have been fixed by L<deguidizing|/deguidize> them to a valid file name. | ||
| 4868 | |||
| 4869 | B<fixedRefsNoAction> - [] hrefs and conrefs from L<fixRefs|/fixRefs> for which no action was taken. | ||
| 4870 | |||
| 4871 | B<flattenFiles> - {old full file name} = file renamed to Gearhart-Brenan file naming standard | ||
| 4872 | |||
| 4873 | B<goodImageFiles> - {file}++ : number of references to each good image | ||
| 4874 | |||
| 4875 | B<goodNavTitles> - Details of nav titles that were resolved. | ||
| 4876 | |||
| 4877 | B<guidHrefs> - {file}{href} = location where href starts with GUID- and is thus probably a guid. | ||
| 4878 | |||
| 4879 | B<guidToFile> - {topic id which is a guid} = file defining topic id. | ||
| 4880 | |||
| 4881 | B<hrefUrlEncoding> - Hrefs that need url encoding because they contain white space. | ||
| 4882 | |||
| 4883 | B<idNotReferenced> - {file}{id}++ - id in a file that is not referenced | ||
| 4884 | |||
| 4885 | B<idReferencedCount> - {file}{id}++ - the number of times this id in this file is referenced from the rest of the corpus | ||
| 4886 | |||
| 4887 | B<idTags> - {file}{id}[tag] The tags associated with each id in a file - there might be more than one if the id is duplicated | ||
| 4888 | |||
| 4889 | B<ids> - {file}{id} - id definitions across all files. | ||
| 4890 | |||
| 4891 | B<idsRemoved> - {id}++ : Ids removed from all files | ||
| 4892 | |||
| 4893 | B<images> - {file}{href} Count of image references in each file. | ||
| 4894 | |||
| 4895 | B<imagesReferencedFromBookMaps> - {bookmap full file name}{full name of image referenced from topic referenced from bookmap}++ | ||
| 4896 | |||
| 4897 | B<imagesReferencedFromTopics> - {topic full file name}{full name of image referenced from topic}++ | ||
| 4898 | |||
| 4899 | B<imagesToRefferingBookMaps> - {image full file name}{bookmap full file name}++ : images to referring bookmaps | ||
| 4900 | |||
| 4901 | B<indexedWords> - {word}{full file name of topic the words occurs in}. | ||
| 4902 | |||
| 4903 | B<inputFileToTargetTopics> - {input file}{target file}++ : Tells us the topics an input file was split into | ||
| 4904 | |||
| 4905 | B<inputFiles> - Input files from L<inputFolder|/inputFolder>. | ||
| 4906 | |||
| 4907 | B<inputFolderImages> - {full image file name} for all files in input folder thus including any images resent | ||
| 4908 | |||
| 4909 | B<ltgt> - {text between < and >}{filename} = count giving the count of text items found between < and > | ||
| 4910 | |||
| 4911 | B<maxZoomOut> - Results from L<maxZoomIn|/maxZoomIn> where {file name}{regular expression key name in L<maxZoomIn|/maxZoomIn>}++ | ||
| 4912 | |||
| 4913 | B<md5Sum> - MD5 sum for each input file. | ||
| 4914 | |||
| 4915 | B<md5SumDuplicates> - {md5sum}{file}++ : md5 sums with more than one file | ||
| 4916 | |||
| 4917 | B<missingImageFiles> - [file, href] == Missing images in each file. | ||
| 4918 | |||
| 4919 | B<missingTopicIds> - Missing topic ids. | ||
| 4920 | |||
| 4921 | B<noHref> - Tags that should have an href but do not have one. | ||
| 4922 | |||
| 4923 | B<notReferenced> - {file name} Files in input area that are not referenced by a conref, image, bookmapref or xref tag and are not a bookmap. | ||
| 4924 | |||
| 4925 | B<olBody> - The number of ol under body by file | ||
| 4926 | |||
| 4927 | B<originalSourceFileAndIdToNewFile> - {original file}{id} = new file: Record mapping from original source file and id to the new file containing the id | ||
| 4928 | |||
| 4929 | B<otherMeta> - {original file}{othermeta name}{othermeta content}++ : the contents of the other meta tags | ||
| 4930 | |||
| 4931 | B<otherMetaBookMapsAfterTopicIncludes> - Bookmap othermeta after topic othermeta has been included | ||
| 4932 | |||
| 4933 | B<otherMetaBookMapsBeforeTopicIncludes> - Bookmap othermeta before topic othermeta has been included | ||
| 4934 | |||
| 4935 | B<otherMetaConsolidated> - {Name}{Content}++ : consolidated other meta data across entire corpus | ||
| 4936 | |||
| 4937 | B<otherMetaDuplicatesCombined> - Duplicate othermeta in bookmaps with called topics othermeta included | ||
| 4938 | |||
| 4939 | B<otherMetaDuplicatesSeparately> - Duplicate othermeta in bookmaps and topics considered separately | ||
| 4940 | |||
| 4941 | B<otherMetaPushToBookMap> - Othermeta that can be pushed to the calling book map | ||
| 4942 | |||
| 4943 | B<otherMetaRemainWithTopic> - Othermeta that must stay in the topic | ||
| 4944 | |||
| 4945 | B<parseFailed> - {file} files that failed to parse. | ||
| 4946 | |||
| 4947 | B<publicId> - {file} = Public id on Doctype | ||
| 4948 | |||
| 4949 | B<references> - {file}{reference}++ - the various references encountered | ||
| 4950 | |||
| 4951 | B<relocatedReferencesFailed> - Failing references that were not fixed by relocation | ||
| 4952 | |||
| 4953 | B<relocatedReferencesFixed> - Relocated references fixed | ||
| 4954 | |||
| 4955 | B<requiredCleanUp> - {full file name}{cleanup} = number of required-cleanups | ||
| 4956 | |||
| 4957 | B<results> - Summary of results table. | ||
| 4958 | |||
| 4959 | B<sourceTopicToTargetBookMap> - {input topic cut into multiple pieces} = output bookmap representing pieces | ||
| 4960 | |||
| 4961 | B<statusLine> - Status line summarizing the cross reference. | ||
| 4962 | |||
| 4963 | B<statusTable> - Status table summarizing the cross reference. | ||
| 4964 | |||
| 4965 | B<tableDimensions> - {file}{columns}{rows} == count | ||
| 4966 | |||
| 4967 | B<tagCount> - {file}{tags} == count of the different tag names found in the xml files. | ||
| 4968 | |||
| 4969 | B<tags> - Number of tags encountered | ||
| 4970 | |||
| 4971 | B<tagsTextsRatio> - Ratio of tags to text encountered | ||
| 4972 | |||
| 4973 | B<targetFolderContent> - {file} = bookmap file name : the target folder content which shows us where an input file went | ||
| 4974 | |||
| 4975 | B<targetTopicToInputFiles> - {current file} = the source file from which the current file was obtained | ||
| 4976 | |||
| 4977 | B<texts> - Number of texts encountered | ||
| 4978 | |||
| 4979 | B<timeEnded> - Time the run ended | ||
| 4980 | |||
| 4981 | B<timeStart> - Time the run started | ||
| 4982 | |||
| 4983 | B<title> - {full file name} = title of file. | ||
| 4984 | |||
| 4985 | B<titleToFile> - {title}{file}++ if L<fixXrefsByTitle> is in effect | ||
| 4986 | |||
| 4987 | B<topicFlattening> - {topic}{sources}++ : the source files for each topic that was flattened | ||
| 4988 | |||
| 4989 | B<topicFlatteningFactor> - Topic flattening factor - higher is better | ||
| 4990 | |||
| 4991 | B<topicIds> - {file} = topic id - the id on the outermost tag. | ||
| 4992 | |||
| 4993 | B<topicsFlattened> - Number of topics flattened | ||
| 4994 | |||
| 4995 | B<topicsNotReferencedFromBookMaps> - {topic file not referenced from any bookmap} = 1 | ||
| 4996 | |||
| 4997 | B<topicsReferencedFromBookMaps> - {bookmap full file name}{topic full file name}++ : bookmaps to topics | ||
| 4998 | |||
| 4999 | B<topicsToReferringBookMaps> - {topic full file name}{bookmap full file name}++ : topics to referring bookmaps | ||
| 5000 | |||
| 5001 | B<urls> - {topic full file name}{url}++ : urls found in each file | ||
| 5002 | |||
| 5003 | B<urlsBad> - {url}{topic full file name}++ : failing urls found in each file | ||
| 5004 | |||
| 5005 | B<urlsGood> - {url}{topic full file name}++ : passing urls found in each file | ||
| 5006 | |||
| 5007 | B<validationErrors> - True means that Lint detected errors in the xml contained in the file. | ||
| 5008 | |||
| 5009 | B<vocabulary> - The text of each topic shorn of attributes for vocabulary comparison. | ||
| 5010 | |||
| 5011 | B<xRefs> - {file}{href}++ Xrefs references. | ||
| 5012 | |||
| 5013 | B<xrefBadFormat> - External xrefs with no format=html. | ||
| 5014 | |||
| 5015 | B<xrefBadScope> - External xrefs with no scope=external. | ||
| 5016 | |||
| 5017 | B<xrefsFixedByTitle> - Xrefs fixed by locating a matching topic title from their text content. | ||
| 5018 | |||
| 5019 | |||
| 5020 | |||
| 5021 | =head1 Private Methods | ||
| 5022 | |||
| 5023 | =head2 newXref(%attributes) | ||
| 5024 | |||
| 5025 | Create a new cross referencer | ||
| 5026 | |||
| 5027 | Parameter Description | ||
| 5028 | 1 %attributes Attributes | ||
| 5029 | |||
| 5030 | =head2 xref2(%attributes) | ||
| 5031 | |||
| 5032 | Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. The possible attributes are defined in L<Data::Edit::Xml::Xref|/Data::Edit::Xml::Xref> | ||
| 5033 | |||
| 5034 | Parameter Description | ||
| 5035 | 1 %attributes Attributes of cross referencer | ||
| 5036 | |||
| 5037 | =head2 createReportsInParallel($xref, @reports) | ||
| 5038 | |||
| 5039 | Create reports in parallel | ||
| 5040 | |||
| 5041 | Parameter Description | ||
| 5042 | 1 $xref Cross referencer | ||
| 5043 | 2 @reports Reports to be run | ||
| 5044 | |||
| 5045 | =head2 createReportsInParallel1() | ||
| 5046 | |||
| 5047 | Create reports in parallel that do not require fixed references | ||
| 5048 | |||
| 5049 | |||
| 5050 | =head2 createReportsInParallel2() | ||
| 5051 | |||
| 5052 | Create reports in parallel that require fixed references | ||
| 5053 | |||
| 5054 | |||
| 5055 | =head2 countLevels($l, $h) | ||
| 5056 | |||
| 5057 | Count has elements to the specified number of levels | ||
| 5058 | |||
| 5059 | Parameter Description | ||
| 5060 | 1 $l Levels | ||
| 5061 | 2 $h Hash | ||
| 5062 | |||
| 5063 | =head2 externalReference($reference) | ||
| 5064 | |||
| 5065 | Check for an external reference | ||
| 5066 | |||
| 5067 | Parameter Description | ||
| 5068 | 1 $reference Reference to check | ||
| 5069 | |||
| 5070 | =head2 fixingRun($xref) | ||
| 5071 | |||
| 5072 | A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source. Returns a useful message describing this state of affairs. | ||
| 5073 | |||
| 5074 | Parameter Description | ||
| 5075 | 1 $xref Cross referencer | ||
| 5076 | |||
| 5077 | =head2 loadInputFiles($xref) | ||
| 5078 | |||
| 5079 | Load the names of the files to be processed | ||
| 5080 | |||
| 5081 | Parameter Description | ||
| 5082 | 1 $xref Cross referencer | ||
| 5083 | |||
| 5084 | =head2 formatTables($xref, $data, %options) | ||
| 5085 | |||
| 5086 | Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>. | ||
| 5087 | |||
| 5088 | Parameter Description | ||
| 5089 | 1 $xref Cross referencer | ||
| 5090 | 2 $data Table to be formatted | ||
| 5091 | 3 %options Options | ||
| 5092 | |||
| 5093 | =head2 hashOfCountsToArray($hash) | ||
| 5094 | |||
| 5095 | Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables> | ||
| 5096 | |||
| 5097 | Parameter Description | ||
| 5098 | 1 $hash Hash to be converted | ||
| 5099 | |||
| 5100 | =head2 reportGuidsToFiles($xref) | ||
| 5101 | |||
| 5102 | Map and report guids to files | ||
| 5103 | |||
| 5104 | Parameter Description | ||
| 5105 | 1 $xref Xref results | ||
| 5106 | |||
| 5107 | =head2 editXml($in, $out, $source) | ||
| 5108 | |||
| 5109 | Edit an xml file retaining any existing XML headers and lint trailers | ||
| 5110 | |||
| 5111 | Parameter Description | ||
| 5112 | 1 $in Input file | ||
| 5113 | 2 $out Output file | ||
| 5114 | 3 $source Source to write | ||
| 5115 | |||
| 5116 | =head2 fixReferencesInOneFile($xref, $sourceFile) | ||
| 5117 | |||
| 5118 | Fix one file by moving unresolved references to the xtrf attribute | ||
| 5119 | |||
| 5120 | Parameter Description | ||
| 5121 | 1 $xref Xref results | ||
| 5122 | 2 $sourceFile Source file to fix | ||
| 5123 | |||
| 5124 | =head2 fixReferencesParallel($xref, $file) | ||
| 5125 | |||
| 5126 | Fix the references in one file | ||
| 5127 | |||
| 5128 | Parameter Description | ||
| 5129 | 1 $xref Cross referencer | ||
| 5130 | 2 $file File to fix | ||
| 5131 | |||
| 5132 | =head2 fixReferencesResults($xref, @results) | ||
| 5133 | |||
| 5134 | Consolidate the results of fixing references. | ||
| 5135 | |||
| 5136 | Parameter Description | ||
| 5137 | 1 $xref Cross referencer | ||
| 5138 | 2 @results Results from fixReferencesInParallel | ||
| 5139 | |||
| 5140 | =head2 fixReferences($xref) | ||
| 5141 | |||
| 5142 | Fix just the file containing references using a number of techniques and report those references that cannot be so fixed. | ||
| 5143 | |||
| 5144 | Parameter Description | ||
| 5145 | 1 $xref Xref results | ||
| 5146 | |||
| 5147 | =head2 fixOneFileGB($xref, $file) | ||
| 5148 | |||
| 5149 | Fix one file to the Gearhart-Brenan standard | ||
| 5150 | |||
| 5151 | Parameter Description | ||
| 5152 | 1 $xref Xref results | ||
| 5153 | 2 $file File to fix | ||
| 5154 | |||
| 5155 | =head2 fixFilesGB($xref) | ||
| 5156 | |||
| 5157 | Rename files to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard> | ||
| 5158 | |||
| 5159 | Parameter Description | ||
| 5160 | 1 $xref Xref results | ||
| 5161 | |||
| 5162 | =head2 analyzeOneFileParallel($Xref, $iFile) | ||
| 5163 | |||
| 5164 | Analyze one input file | ||
| 5165 | |||
| 5166 | Parameter Description | ||
| 5167 | 1 $Xref Xref request | ||
| 5168 | 2 $iFile File to analyze | ||
| 5169 | |||
| 5170 | =head2 analyzeOneFileResults($xref, @x) | ||
| 5171 | |||
| 5172 | Merge a list of cross reference results into the first cross referencer in the list | ||
| 5173 | |||
| 5174 | Parameter Description | ||
| 5175 | 1 $xref Cross referencer to merge into | ||
| 5176 | 2 @x Other cross referencers | ||
| 5177 | |||
| 5178 | =head2 analyzeInputFiles($xref) | ||
| 5179 | |||
| 5180 | Analyze the input files | ||
| 5181 | |||
| 5182 | Parameter Description | ||
| 5183 | 1 $xref Cross referencer | ||
| 5184 | |||
| 5185 | =head2 reportIdRefs($xref) | ||
| 5186 | |||
| 5187 | Report the number of times each id is referenced | ||
| 5188 | |||
| 5189 | Parameter Description | ||
| 5190 | 1 $xref Cross referencer | ||
| 5191 | |||
| 5192 | =head2 removeUnusedIds($xref) | ||
| 5193 | |||
| 5194 | Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference. This is a very conservative approach which acknowledges that writers might be looking for an id if they mention it in a reference. | ||
| 5195 | |||
| 5196 | Parameter Description | ||
| 5197 | 1 $xref Cross referencer | ||
| 5198 | |||
| 5199 | =head2 reportEmptyTopics($xref) | ||
| 5200 | |||
| 5201 | Report empty topics | ||
| 5202 | |||
| 5203 | Parameter Description | ||
| 5204 | 1 $xref Cross referencer | ||
| 5205 | |||
| 5206 | =head2 reportDuplicateIds($xref) | ||
| 5207 | |||
| 5208 | Report duplicate ids | ||
| 5209 | |||
| 5210 | Parameter Description | ||
| 5211 | 1 $xref Cross referencer | ||
| 5212 | |||
| 5213 | =head2 reportDuplicateTopicIds($xref) | ||
| 5214 | |||
| 5215 | Report duplicate topic ids | ||
| 5216 | |||
| 5217 | Parameter Description | ||
| 5218 | 1 $xref Cross referencer | ||
| 5219 | |||
| 5220 | =head2 reportNoHrefs($xref) | ||
| 5221 | |||
| 5222 | Report locations where an href was expected but not found | ||
| 5223 | |||
| 5224 | Parameter Description | ||
| 5225 | 1 $xref Cross referencer | ||
| 5226 | |||
| 5227 | =head2 checkReferences($xref) | ||
| 5228 | |||
| 5229 | Check each reference, report bad references and mark them for fixing. | ||
| 5230 | |||
| 5231 | Parameter Description | ||
| 5232 | 1 $xref Cross referencer | ||
| 5233 | |||
| 5234 | =head2 reportGuidHrefs($xref) | ||
| 5235 | |||
| 5236 | Report on guid hrefs | ||
| 5237 | |||
| 5238 | Parameter Description | ||
| 5239 | 1 $xref Cross referencer | ||
| 5240 | |||
| 5241 | =head2 reportImages($xref) | ||
| 5242 | |||
| 5243 | Reports on images and references to images | ||
| 5244 | |||
| 5245 | Parameter Description | ||
| 5246 | 1 $xref Cross referencer | ||
| 5247 | |||
| 5248 | =head2 reportParseFailed($xref) | ||
| 5249 | |||
| 5250 | Report failed parses | ||
| 5251 | |||
| 5252 | Parameter Description | ||
| 5253 | 1 $xref Cross referencer | ||
| 5254 | |||
| 5255 | =head2 reportXml1($xref) | ||
| 5256 | |||
| 5257 | Report bad xml on line 1 | ||
| 5258 | |||
| 5259 | Parameter Description | ||
| 5260 | 1 $xref Cross referencer | ||
| 5261 | |||
| 5262 | =head2 reportXml2($xref) | ||
| 5263 | |||
| 5264 | Report bad xml on line 2 | ||
| 5265 | |||
| 5266 | Parameter Description | ||
| 5267 | 1 $xref Cross referencer | ||
| 5268 | |||
| 5269 | =head2 reportDocTypeCount($xref) | ||
| 5270 | |||
| 5271 | Report doc type count | ||
| 5272 | |||
| 5273 | Parameter Description | ||
| 5274 | 1 $xref Cross referencer | ||
| 5275 | |||
| 5276 | =head2 reportTagCount($xref) | ||
| 5277 | |||
| 5278 | Report tag counts | ||
| 5279 | |||
| 5280 | Parameter Description | ||
| 5281 | 1 $xref Cross referencer | ||
| 5282 | |||
| 5283 | =head2 reportTagsAndTextsCount($xref) | ||
| 5284 | |||
| 5285 | Report tags and texts counts | ||
| 5286 | |||
| 5287 | Parameter Description | ||
| 5288 | 1 $xref Cross referencer | ||
| 5289 | |||
| 5290 | =head2 reportLtGt($xref) | ||
| 5291 | |||
| 5292 | Report items found between < and > | ||
| 5293 | |||
| 5294 | Parameter Description | ||
| 5295 | 1 $xref Cross referencer | ||
| 5296 | |||
| 5297 | =head2 reportAttributeCount($xref) | ||
| 5298 | |||
| 5299 | Report attribute counts | ||
| 5300 | |||
| 5301 | Parameter Description | ||
| 5302 | 1 $xref Cross referencer | ||
| 5303 | |||
| 5304 | =head2 reportAttributeNameAndValueCounts($xref) | ||
| 5305 | |||
| 5306 | Report attribute value counts | ||
| 5307 | |||
| 5308 | Parameter Description | ||
| 5309 | 1 $xref Cross referencer | ||
| 5310 | |||
| 5311 | =head2 reportValidationErrors($xref) | ||
| 5312 | |||
| 5313 | Report the files known to have validation errors | ||
| 5314 | |||
| 5315 | Parameter Description | ||
| 5316 | 1 $xref Cross referencer | ||
| 5317 | |||
| 5318 | =head2 reportTables($xref) | ||
| 5319 | |||
| 5320 | Report on tables that have problems | ||
| 5321 | |||
| 5322 | Parameter Description | ||
| 5323 | 1 $xref Cross referencer | ||
| 5324 | |||
| 5325 | =head2 reportFileExtensionCount($xref) | ||
| 5326 | |||
| 5327 | Report file extension counts | ||
| 5328 | |||
| 5329 | Parameter Description | ||
| 5330 | 1 $xref Cross referencer | ||
| 5331 | |||
| 5332 | =head2 reportFileTypes($xref) | ||
| 5333 | |||
| 5334 | Report file type counts - takes too long in series | ||
| 5335 | |||
| 5336 | Parameter Description | ||
| 5337 | 1 $xref Cross referencer | ||
| 5338 | |||
| 5339 | =head2 reportExternalXrefs($xref) | ||
| 5340 | |||
| 5341 | Report external xrefs missing other attributes | ||
| 5342 | |||
| 5343 | Parameter Description | ||
| 5344 | 1 $xref Cross referencer | ||
| 5345 | |||
| 5346 | =head2 reportMaxZoomOut($xref) | ||
| 5347 | |||
| 5348 | Text located via Max Zoom In | ||
| 5349 | |||
| 5350 | Parameter Description | ||
| 5351 | 1 $xref Cross referencer | ||
| 5352 | |||
| 5353 | =head2 reportTopicDetails($xref) | ||
| 5354 | |||
| 5355 | Things that occur once in each file | ||
| 5356 | |||
| 5357 | Parameter Description | ||
| 5358 | 1 $xref Cross referencer | ||
| 5359 | |||
| 5360 | =head2 reportTopicReuse($xref) | ||
| 5361 | |||
| 5362 | Count how frequently each topic is reused | ||
| 5363 | |||
| 5364 | Parameter Description | ||
| 5365 | 1 $xref Cross referencer | ||
| 5366 | |||
| 5367 | =head2 reportFixRefs($xref) | ||
| 5368 | |||
| 5369 | Report of hrefs that need to be fixed | ||
| 5370 | |||
| 5371 | Parameter Description | ||
| 5372 | 1 $xref Cross referencer | ||
| 5373 | |||
| 5374 | =head2 reportSourceFiles($xref) | ||
| 5375 | |||
| 5376 | Source file for each topic | ||
| 5377 | |||
| 5378 | Parameter Description | ||
| 5379 | 1 $xref Cross referencer | ||
| 5380 | |||
| 5381 | =head2 reportReferencesFromBookMaps($xref) | ||
| 5382 | |||
| 5383 | Topics and images referenced from bookmaps | ||
| 5384 | |||
| 5385 | Parameter Description | ||
| 5386 | 1 $xref Cross referencer | ||
| 5387 | |||
| 5388 | =head2 reportExteriorMaps($xref) | ||
| 5389 | |||
| 5390 | Maps that are not referenced by any other map | ||
| 5391 | |||
| 5392 | Parameter Description | ||
| 5393 | 1 $xref Cross referencer | ||
| 5394 | |||
| 5395 | =head2 reportTopicsNotReferencedFromBookMaps($xref) | ||
| 5396 | |||
| 5397 | Topics not referenced from bookmaps | ||
| 5398 | |||
| 5399 | Parameter Description | ||
| 5400 | 1 $xref Cross referencer | ||
| 5401 | |||
| 5402 | =head2 reportTableDimensions($xref) | ||
| 5403 | |||
| 5404 | Report table dimensions | ||
| 5405 | |||
| 5406 | Parameter Description | ||
| 5407 | 1 $xref Cross referencer | ||
| 5408 | |||
| 5409 | =head2 reportOtherMeta($xref) | ||
| 5410 | |||
| 5411 | Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map. | ||
| 5412 | |||
| 5413 | Parameter Description | ||
| 5414 | 1 $xref Cross referencer | ||
| 5415 | |||
| 5416 | =head2 createSubjectSchemeMap($xref) | ||
| 5417 | |||
| 5418 | Create a subject scheme map from othermeta | ||
| 5419 | |||
| 5420 | Parameter Description | ||
| 5421 | 1 $xref Cross referencer | ||
| 5422 | |||
| 5423 | =head2 writeClassificationHtml($xref, $classification) | ||
| 5424 | |||
| 5425 | Write classification tree as html | ||
| 5426 | |||
| 5427 | Parameter Description | ||
| 5428 | 1 $xref Cross referencer | ||
| 5429 | 2 $classification {title=>{subject=>{file=>++}}} | ||
| 5430 | |||
| 5431 | =head2 createClassificationMap($xref, $bookMap, $classification) | ||
| 5432 | |||
| 5433 | Create a classification map for each bookmap | ||
| 5434 | |||
| 5435 | Parameter Description | ||
| 5436 | 1 $xref Cross referencer | ||
| 5437 | 2 $bookMap Bookmap to classify | ||
| 5438 | 3 $classification Classification scheme | ||
| 5439 | |||
| 5440 | =head2 createClassificationMaps($xref) | ||
| 5441 | |||
| 5442 | Create classification maps for each bookmap | ||
| 5443 | |||
| 5444 | Parameter Description | ||
| 5445 | 1 $xref Cross referencer | ||
| 5446 | |||
| 5447 | =head2 reportSimilarTopicsByTitle($xref) | ||
| 5448 | |||
| 5449 | Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names | ||
| 5450 | |||
| 5451 | Parameter Description | ||
| 5452 | 1 $xref Cross referencer | ||
| 5453 | |||
| 5454 | =head2 reportSimilarTopicsByVocabulary($xref) | ||
| 5455 | |||
| 5456 | Report topics likely to be similar on the basis of their vocabulary | ||
| 5457 | |||
| 5458 | Parameter Description | ||
| 5459 | 1 $xref Cross referencer | ||
| 5460 | |||
| 5461 | =head2 reportWordsByFile($xref) | ||
| 5462 | |||
| 5463 | Index words to the files they occur in | ||
| 5464 | |||
| 5465 | Parameter Description | ||
| 5466 | 1 $xref Cross referencer | ||
| 5467 | |||
| 5468 | =head2 reportMd5Sum($xref) | ||
| 5469 | |||
| 5470 | Report files with identical md5 sums | ||
| 5471 | |||
| 5472 | Parameter Description | ||
| 5473 | 1 $xref Cross referencer | ||
| 5474 | |||
| 5475 | =head2 reportOlBody($xref) | ||
| 5476 | |||
| 5477 | ol under body - indicative of a task | ||
| 5478 | |||
| 5479 | Parameter Description | ||
| 5480 | 1 $xref Cross referencer | ||
| 5481 | |||
| 5482 | =head2 reportHrefUrlEncoding($xref) | ||
| 5483 | |||
| 5484 | href needs url encoding | ||
| 5485 | |||
| 5486 | Parameter Description | ||
| 5487 | 1 $xref Cross referencer | ||
| 5488 | |||
| 5489 | =head2 reportConRefMatching($xref) | ||
| 5490 | |||
| 5491 | Report conref matching | ||
| 5492 | |||
| 5493 | Parameter Description | ||
| 5494 | 1 $xref Cross referencer | ||
| 5495 | |||
| 5496 | =head2 reportPublicIds($xref) | ||
| 5497 | |||
| 5498 | Report public ids in use | ||
| 5499 | |||
| 5500 | Parameter Description | ||
| 5501 | 1 $xref Cross referencer | ||
| 5502 | |||
| 5503 | =head2 reportRequiredCleanUps($xref) | ||
| 5504 | |||
| 5505 | Report required clean ups | ||
| 5506 | |||
| 5507 | Parameter Description | ||
| 5508 | 1 $xref Cross referencer | ||
| 5509 | |||
| 5510 | =head2 reportUrls($xref) | ||
| 5511 | |||
| 5512 | Report urls that fail to resolve | ||
| 5513 | |||
| 5514 | Parameter Description | ||
| 5515 | 1 $xref Cross referencer | ||
| 5516 | |||
| 5517 | =head2 addNavTitlesToOneMap($xref, $file) | ||
| 5518 | |||
| 5519 | Fix navtitles in one map | ||
| 5520 | |||
| 5521 | Parameter Description | ||
| 5522 | 1 $xref Xref results | ||
| 5523 | 2 $file File to fix | ||
| 5524 | |||
| 5525 | =head2 addNavTitlesToMaps($xref) | ||
| 5526 | |||
| 5527 | Add nav titles to files containing maps. | ||
| 5528 | |||
| 5529 | Parameter Description | ||
| 5530 | 1 $xref Xref results | ||
| 5531 | |||
| 5532 | =head2 oxygenProjectFileMetaData() | ||
| 5533 | |||
| 5534 | Meta data for the oxygen project files | ||
| 5535 | |||
| 5536 | |||
| 5537 | =head2 createOxygenProjectFile($xref, $bm, $xprName) | ||
| 5538 | |||
| 5539 | Create an Oxygen project file for the specified bookmap | ||
| 5540 | |||
| 5541 | Parameter Description | ||
| 5542 | 1 $xref Xref | ||
| 5543 | 2 $bm Bookmap | ||
| 5544 | 3 $xprName Xpr name from bookmap | ||
| 5545 | |||
| 5546 | =head2 createOxygenProjectMapFiles($xref) | ||
| 5547 | |||
| 5548 | Create Oxygen project files from Xref results | ||
| 5549 | |||
| 5550 | Parameter Description | ||
| 5551 | 1 $xref Cross referencer | ||
| 5552 | |||
| 5553 | =head2 oneBadRef($xref, $file, $href) | ||
| 5554 | |||
| 5555 | Check one reference and return the first error encountered or B<undef> if no errors encountered. Relies on L<topicIds> to test files present and test the B<topicId> is valid, relies on L<ids> to check that the referenced B<id> is valid. | ||
| 5556 | |||
| 5557 | Parameter Description | ||
| 5558 | 1 $xref Cross referencer | ||
| 5559 | 2 $file File containing reference | ||
| 5560 | 3 $href Reference | ||
| 5561 | |||
| 5562 | =head2 createSampleInputFilesBaseCase($in, $N) | ||
| 5563 | |||
| 5564 | Create sample input files for testing. The attribute B<inputFolder> supplies the name of the folder in which to create the sample files. | ||
| 5565 | |||
| 5566 | Parameter Description | ||
| 5567 | 1 $in Input folder | ||
| 5568 | 2 $N Number of sample files | ||
| 5569 | |||
| 5570 | =head2 createSampleInputFilesFixFolder($in) | ||
| 5571 | |||
| 5572 | Create sample input files for testing fixFolder | ||
| 5573 | |||
| 5574 | Parameter Description | ||
| 5575 | 1 $in Folder to create the files in | ||
| 5576 | |||
| 5577 | =head2 createSampleInputFilesLtGt($in) | ||
| 5578 | |||
| 5579 | Create sample input files for testing items between < and > | ||
| 5580 | |||
| 5581 | Parameter Description | ||
| 5582 | 1 $in Folder to create the files in | ||
| 5583 | |||
| 5584 | =head2 createSampleInputFilesForFixDitaRefs($in, $targets) | ||
| 5585 | |||
| 5586 | Create sample input files for fixing renamed topic refs | ||
| 5587 | |||
| 5588 | Parameter Description | ||
| 5589 | 1 $in Folder to create the files in | ||
| 5590 | 2 $targets Targets folder | ||
| 5591 | |||
| 5592 | =head2 createSampleInputFilesForFixDitaRefsXref($in) | ||
| 5593 | |||
| 5594 | Create sample input files for fixing references into renamed topics by xref | ||
| 5595 | |||
| 5596 | Parameter Description | ||
| 5597 | 1 $in Folder to create the files in | ||
| 5598 | |||
| 5599 | =head2 createSampleConRefs($in) | ||
| 5600 | |||
| 5601 | Create sample input files for fixing a conref | ||
| 5602 | |||
| 5603 | Parameter Description | ||
| 5604 | 1 $in Folder to create the files in | ||
| 5605 | |||
| 5606 | =head2 createSampleConRefMatching($in) | ||
| 5607 | |||
| 5608 | Create sample input files for matching conref source and targets | ||
| 5609 | |||
| 5610 | Parameter Description | ||
| 5611 | 1 $in Folder to create the files in | ||
| 5612 | |||
| 5613 | =head2 createSampleDuplicateMd5Sum($in) | ||
| 5614 | |||
| 5615 | Create sample input files with duplicate md5 sums | ||
| 5616 | |||
| 5617 | Parameter Description | ||
| 5618 | 1 $in Folder to create the files in | ||
| 5619 | |||
| 5620 | =head2 createSampleUnreferencedIds($in) | ||
| 5621 | |||
| 5622 | Create sample input files with unreferenced ids | ||
| 5623 | |||
| 5624 | Parameter Description | ||
| 5625 | 1 $in Folder to create the files in | ||
| 5626 | |||
| 5627 | =head2 createEmptyBody($in) | ||
| 5628 | |||
| 5629 | Create sample input files for empty body detection | ||
| 5630 | |||
| 5631 | Parameter Description | ||
| 5632 | 1 $in Folder to create the files in | ||
| 5633 | |||
| 5634 | =head2 createClassificationMapsTest($in) | ||
| 5635 | |||
| 5636 | Create sample input files for a classification map | ||
| 5637 | |||
| 5638 | Parameter Description | ||
| 5639 | 1 $in Folder to create the files in | ||
| 5640 | |||
| 5641 | =head2 createWordsToFilesTest($in) | ||
| 5642 | |||
| 5643 | Index words to file | ||
| 5644 | |||
| 5645 | Parameter Description | ||
| 5646 | 1 $in Folder to create the files in | ||
| 5647 | |||
| 5648 | =head2 createUrlTests($in) | ||
| 5649 | |||
| 5650 | Check urls | ||
| 5651 | |||
| 5652 | Parameter Description | ||
| 5653 | 1 $in Folder to create the files in | ||
| 5654 | |||
| 5655 | =head2 changeFolderAndWriteFiles($f, $D) | ||
| 5656 | |||
| 5657 | Change file structure to the current folder and write | ||
| 5658 | |||
| 5659 | Parameter Description | ||
| 5660 | 1 $f Data structure as a string | ||
| 5661 | 2 $D Target folder | ||
| 5662 | |||
| 5663 | =head2 createSampleInputFilesForFixDitaRefsImproved1($folder) | ||
| 5664 | |||
| 5665 | Create sample input files for fixing references via the targets/ folder | ||
| 5666 | |||
| 5667 | Parameter Description | ||
| 5668 | 1 $folder Folder to switch to | ||
| 5669 | |||
| 5670 | =head2 createSampleInputFilesForFixDitaRefsImproved2($folder) | ||
| 5671 | |||
| 5672 | Create sample input files for fixing conref references via the targets/ folder | ||
| 5673 | |||
| 5674 | Parameter Description | ||
| 5675 | 1 $folder Folder to switch to | ||
| 5676 | |||
| 5677 | =head2 createSampleInputFilesForFixDitaRefsImproved3($folder) | ||
| 5678 | |||
| 5679 | Create sample input files for fixing bookmap references to topics that get cut into multiple pieces | ||
| 5680 | |||
| 5681 | Parameter Description | ||
| 5682 | 1 $folder Folder to switch to | ||
| 5683 | |||
| 5684 | =head2 createSampleInputFilesForFixDitaRefsImproved4($folder) | ||
| 5685 | |||
| 5686 | Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces | ||
| 5687 | |||
| 5688 | Parameter Description | ||
| 5689 | 1 $folder Folder to switch to | ||
| 5690 | |||
| 5691 | =head2 createSampleImageTest($folder) | ||
| 5692 | |||
| 5693 | Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces | ||
| 5694 | |||
| 5695 | Parameter Description | ||
| 5696 | 1 $folder Folder to switch to | ||
| 5697 | |||
| 5698 | =head2 createTestTopicFlattening($folder) | ||
| 5699 | |||
| 5700 | Create sample input files for testing topic flattening ratio reporting | ||
| 5701 | |||
| 5702 | Parameter Description | ||
| 5703 | 1 $folder Folder to switch to | ||
| 5704 | |||
| 5705 | =head2 createTestReferencedToFlattenedTopic($folder) | ||
| 5706 | |||
| 5707 | Full reference to a topic that has been flattened | ||
| 5708 | |||
| 5709 | Parameter Description | ||
| 5710 | 1 $folder Folder to switch to | ||
| 5711 | |||
| 5712 | =head2 createTestReferenceToCutOutTopic($folder) | ||
| 5713 | |||
| 5714 | References from a topic that has been cut out to a topic that has been cut out | ||
| 5715 | |||
| 5716 | Parameter Description | ||
| 5717 | 1 $folder Folder to switch to | ||
| 5718 | |||
| 5719 | =head2 createSampleOtherMeta($out) | ||
| 5720 | |||
| 5721 | Create sample data for othermeta reports | ||
| 5722 | |||
| 5723 | Parameter Description | ||
| 5724 | 1 $out Folder | ||
| 5725 | |||
| 5726 | =head2 createTestOneNotRef($folder) | ||
| 5727 | |||
| 5728 | One topic refernced and the other not | ||
| 5729 | |||
| 5730 | Parameter Description | ||
| 5731 | 1 $folder Folder to switch to | ||
| 5732 | |||
| 5733 | =head2 createSampleTopicsReferencedFromBookMaps($in) | ||
| 5734 | |||
| 5735 | The number of times a topic is referenced from a bookmap | ||
| 5736 | |||
| 5737 | Parameter Description | ||
| 5738 | 1 $in Folder to create the files in | ||
| 5739 | |||
| 5740 | =head2 createSampleImageReferences($in) | ||
| 5741 | |||
| 5742 | Good and bad image references | ||
| 5743 | |||
| 5744 | Parameter Description | ||
| 5745 | 1 $in Folder to create the files in | ||
| 5746 | |||
| 5747 | =head2 createRequiredCleanUps($in) | ||
| 5748 | |||
| 5749 | Required clean ups report | ||
| 5750 | |||
| 5751 | Parameter Description | ||
| 5752 | 1 $in Folder to create the files in | ||
| 5753 | |||
| 5754 | =head2 createSoftConrefs($in) | ||
| 5755 | |||
| 5756 | Fix file part of conref even if the rest is invalid | ||
| 5757 | |||
| 5758 | Parameter Description | ||
| 5759 | 1 $in Folder to create the files in | ||
| 5760 | |||
| 5761 | =head2 checkXrefStructure($x, $field, @folders) | ||
| 5762 | |||
| 5763 | Check an output structure produced by Xrf | ||
| 5764 | |||
| 5765 | Parameter Description | ||
| 5766 | 1 $x Cross references | ||
| 5767 | 2 $field Field to check | ||
| 5768 | 3 @folders Folders to suppress | ||
| 5769 | |||
| 5770 | =head2 writeXrefStructure($x, $field, @folders) | ||
| 5771 | |||
| 5772 | Write the test for an Xref structure | ||
| 5773 | |||
| 5774 | Parameter Description | ||
| 5775 | 1 $x Cross referencer | ||
| 5776 | 2 $field Field | ||
| 5777 | 3 @folders Names of the folders to suppress | ||
| 5778 | |||
| 5779 | =head2 deleteVariableFields($x) | ||
| 5780 | |||
| 5781 | Remove time and other fields that do not affect the end results | ||
| 5782 | |||
| 5783 | Parameter Description | ||
| 5784 | 1 $x Cross referencer | ||
| 5785 | |||
| 5786 | =head2 testReferenceChecking() | ||
| 5787 | |||
| 5788 | Test reference checking | ||
| 5789 | |||
| 5790 | |||
| 5791 | |||
| 5792 | =head1 Index | ||
| 5793 | |||
| 5794 | |||
| 5795 | 1 L<addNavTitlesToMaps|/addNavTitlesToMaps> - Add nav titles to files containing maps. | ||
| 5796 | |||
| 5797 | 2 L<addNavTitlesToOneMap|/addNavTitlesToOneMap> - Fix navtitles in one map | ||
| 5798 | |||
| 5799 | 3 L<analyzeInputFiles|/analyzeInputFiles> - Analyze the input files | ||
| 5800 | |||
| 5801 | 4 L<analyzeOneFileParallel|/analyzeOneFileParallel> - Analyze one input file | ||
| 5802 | |||
| 5803 | 5 L<analyzeOneFileResults|/analyzeOneFileResults> - Merge a list of cross reference results into the first cross referencer in the list | ||
| 5804 | |||
| 5805 | 6 L<changeFolderAndWriteFiles|/changeFolderAndWriteFiles> - Change file structure to the current folder and write | ||
| 5806 | |||
| 5807 | 7 L<checkReferences|/checkReferences> - Check each reference, report bad references and mark them for fixing. | ||
| 5808 | |||
| 5809 | 8 L<checkXrefStructure|/checkXrefStructure> - Check an output structure produced by Xrf | ||
| 5810 | |||
| 5811 | 9 L<countLevels|/countLevels> - Count has elements to the specified number of levels | ||
| 5812 | |||
| 5813 | 10 L<createClassificationMap|/createClassificationMap> - Create a classification map for each bookmap | ||
| 5814 | |||
| 5815 | 11 L<createClassificationMaps|/createClassificationMaps> - Create classification maps for each bookmap | ||
| 5816 | |||
| 5817 | 12 L<createClassificationMapsTest|/createClassificationMapsTest> - Create sample input files for a classification map | ||
| 5818 | |||
| 5819 | 13 L<createEmptyBody|/createEmptyBody> - Create sample input files for empty body detection | ||
| 5820 | |||
| 5821 | 14 L<createOxygenProjectFile|/createOxygenProjectFile> - Create an Oxygen project file for the specified bookmap | ||
| 5822 | |||
| 5823 | 15 L<createOxygenProjectMapFiles|/createOxygenProjectMapFiles> - Create Oxygen project files from Xref results | ||
| 5824 | |||
| 5825 | 16 L<createReportsInParallel|/createReportsInParallel> - Create reports in parallel | ||
| 5826 | |||
| 5827 | 17 L<createReportsInParallel1|/createReportsInParallel1> - Create reports in parallel that do not require fixed references | ||
| 5828 | |||
| 5829 | 18 L<createReportsInParallel2|/createReportsInParallel2> - Create reports in parallel that require fixed references | ||
| 5830 | |||
| 5831 | 19 L<createRequiredCleanUps|/createRequiredCleanUps> - Required clean ups report | ||
| 5832 | |||
| 5833 | 20 L<createSampleConRefMatching|/createSampleConRefMatching> - Create sample input files for matching conref source and targets | ||
| 5834 | |||
| 5835 | 21 L<createSampleConRefs|/createSampleConRefs> - Create sample input files for fixing a conref | ||
| 5836 | |||
| 5837 | 22 L<createSampleDuplicateMd5Sum|/createSampleDuplicateMd5Sum> - Create sample input files with duplicate md5 sums | ||
| 5838 | |||
| 5839 | 23 L<createSampleImageReferences|/createSampleImageReferences> - Good and bad image references | ||
| 5840 | |||
| 5841 | 24 L<createSampleImageTest|/createSampleImageTest> - Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces | ||
| 5842 | |||
| 5843 | 25 L<createSampleInputFilesBaseCase|/createSampleInputFilesBaseCase> - Create sample input files for testing. | ||
| 5844 | |||
| 5845 | 26 L<createSampleInputFilesFixFolder|/createSampleInputFilesFixFolder> - Create sample input files for testing fixFolder | ||
| 5846 | |||
| 5847 | 27 L<createSampleInputFilesForFixDitaRefs|/createSampleInputFilesForFixDitaRefs> - Create sample input files for fixing renamed topic refs | ||
| 5848 | |||
| 5849 | 28 L<createSampleInputFilesForFixDitaRefsImproved1|/createSampleInputFilesForFixDitaRefsImproved1> - Create sample input files for fixing references via the targets/ folder | ||
| 5850 | |||
| 5851 | 29 L<createSampleInputFilesForFixDitaRefsImproved2|/createSampleInputFilesForFixDitaRefsImproved2> - Create sample input files for fixing conref references via the targets/ folder | ||
| 5852 | |||
| 5853 | 30 L<createSampleInputFilesForFixDitaRefsImproved3|/createSampleInputFilesForFixDitaRefsImproved3> - Create sample input files for fixing bookmap references to topics that get cut into multiple pieces | ||
| 5854 | |||
| 5855 | 31 L<createSampleInputFilesForFixDitaRefsImproved4|/createSampleInputFilesForFixDitaRefsImproved4> - Create sample input files for fixing bookmap reference to a topic that did not get cut into multiple pieces | ||
| 5856 | |||
| 5857 | 32 L<createSampleInputFilesForFixDitaRefsXref|/createSampleInputFilesForFixDitaRefsXref> - Create sample input files for fixing references into renamed topics by xref | ||
| 5858 | |||
| 5859 | 33 L<createSampleInputFilesLtGt|/createSampleInputFilesLtGt> - Create sample input files for testing items between < and > | ||
| 5860 | |||
| 5861 | 34 L<createSampleOtherMeta|/createSampleOtherMeta> - Create sample data for othermeta reports | ||
| 5862 | |||
| 5863 | 35 L<createSampleTopicsReferencedFromBookMaps|/createSampleTopicsReferencedFromBookMaps> - The number of times a topic is referenced from a bookmap | ||
| 5864 | |||
| 5865 | 36 L<createSampleUnreferencedIds|/createSampleUnreferencedIds> - Create sample input files with unreferenced ids | ||
| 5866 | |||
| 5867 | 37 L<createSoftConrefs|/createSoftConrefs> - Fix file part of conref even if the rest is invalid | ||
| 5868 | |||
| 5869 | 38 L<createSubjectSchemeMap|/createSubjectSchemeMap> - Create a subject scheme map from othermeta | ||
| 5870 | |||
| 5871 | 39 L<createTestOneNotRef|/createTestOneNotRef> - One topic refernced and the other not | ||
| 5872 | |||
| 5873 | 40 L<createTestReferencedToFlattenedTopic|/createTestReferencedToFlattenedTopic> - Full reference to a topic that has been flattened | ||
| 5874 | |||
| 5875 | 41 L<createTestReferenceToCutOutTopic|/createTestReferenceToCutOutTopic> - References from a topic that has been cut out to a topic that has been cut out | ||
| 5876 | |||
| 5877 | 42 L<createTestTopicFlattening|/createTestTopicFlattening> - Create sample input files for testing topic flattening ratio reporting | ||
| 5878 | |||
| 5879 | 43 L<createUrlTests|/createUrlTests> - Check urls | ||
| 5880 | |||
| 5881 | 44 L<createWordsToFilesTest|/createWordsToFilesTest> - Index words to file | ||
| 5882 | |||
| 5883 | 45 L<deleteVariableFields|/deleteVariableFields> - Remove time and other fields that do not affect the end results | ||
| 5884 | |||
| 5885 | 46 L<editXml|/editXml> - Edit an xml file retaining any existing XML headers and lint trailers | ||
| 5886 | |||
| 5887 | 47 L<externalReference|/externalReference> - Check for an external reference | ||
| 5888 | |||
| 5889 | 48 L<fixFilesGB|/fixFilesGB> - Rename files to the L<GB Standard|http://metacpan.org/pod/Dita::GB::Standard> | ||
| 5890 | |||
| 5891 | 49 L<fixingRun|/fixingRun> - A fixing run fixes problems where it can and thus induces changes which might make the updated output different from the incoming source. | ||
| 5892 | |||
| 5893 | 50 L<fixOneFileGB|/fixOneFileGB> - Fix one file to the Gearhart-Brenan standard | ||
| 5894 | |||
| 5895 | 51 L<fixReferences|/fixReferences> - Fix just the file containing references using a number of techniques and report those references that cannot be so fixed. | ||
| 5896 | |||
| 5897 | 52 L<fixReferencesInOneFile|/fixReferencesInOneFile> - Fix one file by moving unresolved references to the xtrf attribute | ||
| 5898 | |||
| 5899 | 53 L<fixReferencesParallel|/fixReferencesParallel> - Fix the references in one file | ||
| 5900 | |||
| 5901 | 54 L<fixReferencesResults|/fixReferencesResults> - Consolidate the results of fixing references. | ||
| 5902 | |||
| 5903 | 55 L<formatTables|/formatTables> - Using cross reference B<$xref> options and an array of arrays B<$data> format a report as a table using B<%options> as described in L<Data::Table::Text::formatTable> and L<Data::Table::Text::formatHtmlTable>. | ||
| 5904 | |||
| 5905 | 56 L<hashOfCountsToArray|/hashOfCountsToArray> - Convert a B<$hash> of {key} = count to an array so it can be formatted with L<formatTables> | ||
| 5906 | |||
| 5907 | 57 L<loadInputFiles|/loadInputFiles> - Load the names of the files to be processed | ||
| 5908 | |||
| 5909 | 58 L<newXref|/newXref> - Create a new cross referencer | ||
| 5910 | |||
| 5911 | 59 L<oneBadRef|/oneBadRef> - Check one reference and return the first error encountered or B<undef> if no errors encountered. | ||
| 5912 | |||
| 5913 | 60 L<oxygenProjectFileMetaData|/oxygenProjectFileMetaData> - Meta data for the oxygen project files | ||
| 5914 | |||
| 5915 | 61 L<removeUnusedIds|/removeUnusedIds> - Remove ids that do are not mentioned in any href or conref in the corpus regardless of the file component of any such reference. | ||
| 5916 | |||
| 5917 | 62 L<reportAttributeCount|/reportAttributeCount> - Report attribute counts | ||
| 5918 | |||
| 5919 | 63 L<reportAttributeNameAndValueCounts|/reportAttributeNameAndValueCounts> - Report attribute value counts | ||
| 5920 | |||
| 5921 | 64 L<reportConRefMatching|/reportConRefMatching> - Report conref matching | ||
| 5922 | |||
| 5923 | 65 L<reportDocTypeCount|/reportDocTypeCount> - Report doc type count | ||
| 5924 | |||
| 5925 | 66 L<reportDuplicateIds|/reportDuplicateIds> - Report duplicate ids | ||
| 5926 | |||
| 5927 | 67 L<reportDuplicateTopicIds|/reportDuplicateTopicIds> - Report duplicate topic ids | ||
| 5928 | |||
| 5929 | 68 L<reportEmptyTopics|/reportEmptyTopics> - Report empty topics | ||
| 5930 | |||
| 5931 | 69 L<reportExteriorMaps|/reportExteriorMaps> - Maps that are not referenced by any other map | ||
| 5932 | |||
| 5933 | 70 L<reportExternalXrefs|/reportExternalXrefs> - Report external xrefs missing other attributes | ||
| 5934 | |||
| 5935 | 71 L<reportFileExtensionCount|/reportFileExtensionCount> - Report file extension counts | ||
| 5936 | |||
| 5937 | 72 L<reportFileTypes|/reportFileTypes> - Report file type counts - takes too long in series | ||
| 5938 | |||
| 5939 | 73 L<reportFixRefs|/reportFixRefs> - Report of hrefs that need to be fixed | ||
| 5940 | |||
| 5941 | 74 L<reportGuidHrefs|/reportGuidHrefs> - Report on guid hrefs | ||
| 5942 | |||
| 5943 | 75 L<reportGuidsToFiles|/reportGuidsToFiles> - Map and report guids to files | ||
| 5944 | |||
| 5945 | 76 L<reportHrefUrlEncoding|/reportHrefUrlEncoding> - href needs url encoding | ||
| 5946 | |||
| 5947 | 77 L<reportIdRefs|/reportIdRefs> - Report the number of times each id is referenced | ||
| 5948 | |||
| 5949 | 78 L<reportImages|/reportImages> - Reports on images and references to images | ||
| 5950 | |||
| 5951 | 79 L<reportLtGt|/reportLtGt> - Report items found between < and > | ||
| 5952 | |||
| 5953 | 80 L<reportMaxZoomOut|/reportMaxZoomOut> - Text located via Max Zoom In | ||
| 5954 | |||
| 5955 | 81 L<reportMd5Sum|/reportMd5Sum> - Report files with identical md5 sums | ||
| 5956 | |||
| 5957 | 82 L<reportNoHrefs|/reportNoHrefs> - Report locations where an href was expected but not found | ||
| 5958 | |||
| 5959 | 83 L<reportOlBody|/reportOlBody> - ol under body - indicative of a task | ||
| 5960 | |||
| 5961 | 84 L<reportOtherMeta|/reportOtherMeta> - Advise in the feasibility of moving othermeta data from topics to bookmaps assuming that the othermeta data will be applied only at the head of the map rather than individually to each topic in the map. | ||
| 5962 | |||
| 5963 | 85 L<reportParseFailed|/reportParseFailed> - Report failed parses | ||
| 5964 | |||
| 5965 | 86 L<reportPublicIds|/reportPublicIds> - Report public ids in use | ||
| 5966 | |||
| 5967 | 87 L<reportReferencesFromBookMaps|/reportReferencesFromBookMaps> - Topics and images referenced from bookmaps | ||
| 5968 | |||
| 5969 | 88 L<reportRequiredCleanUps|/reportRequiredCleanUps> - Report required clean ups | ||
| 5970 | |||
| 5971 | 89 L<reportSimilarTopicsByTitle|/reportSimilarTopicsByTitle> - Report topics likely to be similar on the basis of their titles as expressed in the non Guid part of their file names | ||
| 5972 | |||
| 5973 | 90 L<reportSimilarTopicsByVocabulary|/reportSimilarTopicsByVocabulary> - Report topics likely to be similar on the basis of their vocabulary | ||
| 5974 | |||
| 5975 | 91 L<reportSourceFiles|/reportSourceFiles> - Source file for each topic | ||
| 5976 | |||
| 5977 | 92 L<reportTableDimensions|/reportTableDimensions> - Report table dimensions | ||
| 5978 | |||
| 5979 | 93 L<reportTables|/reportTables> - Report on tables that have problems | ||
| 5980 | |||
| 5981 | 94 L<reportTagCount|/reportTagCount> - Report tag counts | ||
| 5982 | |||
| 5983 | 95 L<reportTagsAndTextsCount|/reportTagsAndTextsCount> - Report tags and texts counts | ||
| 5984 | |||
| 5985 | 96 L<reportTopicDetails|/reportTopicDetails> - Things that occur once in each file | ||
| 5986 | |||
| 5987 | 97 L<reportTopicReuse|/reportTopicReuse> - Count how frequently each topic is reused | ||
| 5988 | |||
| 5989 | 98 L<reportTopicsNotReferencedFromBookMaps|/reportTopicsNotReferencedFromBookMaps> - Topics not referenced from bookmaps | ||
| 5990 | |||
| 5991 | 99 L<reportUrls|/reportUrls> - Report urls that fail to resolve | ||
| 5992 | |||
| 5993 | 100 L<reportValidationErrors|/reportValidationErrors> - Report the files known to have validation errors | ||
| 5994 | |||
| 5995 | 101 L<reportWordsByFile|/reportWordsByFile> - Index words to the files they occur in | ||
| 5996 | |||
| 5997 | 102 L<reportXml1|/reportXml1> - Report bad xml on line 1 | ||
| 5998 | |||
| 5999 | 103 L<reportXml2|/reportXml2> - Report bad xml on line 2 | ||
| 6000 | |||
| 6001 | 104 L<testReferenceChecking|/testReferenceChecking> - Test reference checking | ||
| 6002 | |||
| 6003 | 105 L<writeClassificationHtml|/writeClassificationHtml> - Write classification tree as html | ||
| 6004 | |||
| 6005 | 106 L<writeXrefStructure|/writeXrefStructure> - Write the test for an Xref structure | ||
| 6006 | |||
| 6007 | 107 L<xref|/xref> - Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. | ||
| 6008 | |||
| 6009 | 108 L<xref2|/xref2> - Check the cross references in a set of Dita files held in L<inputFolder|/inputFolder> and report the results in the L<reports|/reports> folder. | ||
| 6010 | |||
| 6011 | =head1 Installation | ||
| 6012 | |||
| 6013 | This module is written in 100% Pure Perl and, thus, it is easy to read, | ||
| 6014 | comprehend, use, modify and install via B<cpan>: | ||
| 6015 | |||
| 6016 | sudo cpan install Data::Edit::Xml::Xref | ||
| 6017 | |||
| 6018 | =head1 Author | ||
| 6019 | |||
| 6020 | L<philiprbrenan@gmail.com|mailto:philiprbrenan@gmail.com> | ||
| 6021 | |||
| 6022 | L<http://www.appaapps.com|http://www.appaapps.com> | ||
| 6023 | |||
| 6024 | =head1 Copyright | ||
| 6025 | |||
| 6026 | Copyright (c) 2016-2019 Philip R Brenan. | ||
| 6027 | |||
| 6028 | This module is free software. It may be used, redistributed and/or modified | ||
| 6029 | under the same terms as Perl itself. | ||
| 6030 | |||
| 6031 | =cut | ||
| 6032 | |||
| 6033 | |||
| 6034 | |||
| 6035 | # Tests and documentation | ||
| 6036 | |||
| 6037 | sub test | ||
| 6038 | {my $p = __PACKAGE__; | ||
| 6039 | binmode($_, ":utf8") for *STDOUT, *STDERR; | ||
| 6040 | return if eval "eof(${p}::DATA)"; | ||
| 6041 | my $s = eval "join('', <${p}::DATA>)"; | ||
| 6042 | $@ and die $@; | ||
| 6043 | eval $s; | ||
| 6044 | $@ and die $@; | ||
| 6045 | 1 | ||
| 6046 | } | ||
| 6047 | |||
| 6048 | test unless caller; | ||
| 6049 | |||
| 6050 | 1; | ||
| 6051 | # podDocumentation | ||
| 6052 | __DATA__ | ||