| 1 | #!/usr/local/bin/perl | ||
| 2 | use strict; | ||
| 3 | |||
| 4 | use XML::LibXML; | ||
| 5 | use OPC::Node; | ||
| 6 | use FB3; | ||
| 7 | use FB3::Validator; | ||
| 8 | use File::Copy; | ||
| 9 | use File::Temp; | ||
| 10 | use File::Basename qw(basename); | ||
| 11 | use Cwd qw(cwd abs_path); | ||
| 12 | use Getopt::Long; | ||
| 13 | use utf8; | ||
| 14 | |||
| 15 | my ($Help, $Force, $In, $Out, $CutChars, $XsdPath, $ImagesPath, $ImageFileName, $WorkType); | ||
| 16 | my $r = GetOptions( | ||
| 17 | 'help|h' => \$Help, | ||
| 18 | 'force|f' => \$Force, | ||
| 19 | 'in|i=s' => \$In, | ||
| 20 | 'out|o=s' => \$Out, | ||
| 21 | 'chars|c=i' => \$CutChars, | ||
| 22 | 'xsd|x=s' => \$XsdPath, # optional | ||
| 23 | 'imagespath|p=s' => \$ImagesPath, | ||
| 24 | 'type|t=s' => \$WorkType, | ||
| 25 | ); | ||
| 26 | |||
| 27 | help() if $Help; | ||
| 28 | |||
| 29 | unless ($In and $Out) { | ||
| 30 | print "\nrequired params 'in'/'out' not defined\n"; | ||
| 31 | help(); | ||
| 32 | } | ||
| 33 | |||
| 34 | if ($ImagesPath) { | ||
| 35 | die "Path $ImagesPath not found\n" unless -d $ImagesPath; | ||
| 36 | $ImagesPath = $ImagesPath.'/' if $ImagesPath !~ /\/$/; | ||
| 37 | } | ||
| 38 | |||
| 39 | my $InTrial; #мы внутри триалки | ||
| 40 | |||
| 41 | if ( not $WorkType ) { | ||
| 42 | |||
| 43 | $WorkType = 'trial'; | ||
| 44 | |||
| 45 | } elsif ( $WorkType !~ /^(trial|output)$/ ) { | ||
| 46 | |||
| 47 | print "\nparams 'type' must be trial|output\n"; | ||
| 48 | help(); | ||
| 49 | } | ||
| 50 | |||
| 51 | use constant { | ||
| 52 | NS_XLINK => 'http://www.w3.org/1999/xlink', | ||
| 53 | NS_FB3_DESCRIPTION => 'http://www.fictionbook.org/FictionBook3/description', | ||
| 54 | NS_FB3_BODY => 'http://www.fictionbook.org/FictionBook3/body', | ||
| 55 | DEFAULT_TRIAL_PERCENT => 24, | ||
| 56 | }; | ||
| 57 | |||
| 58 | my $Blocks = qr/^(p|subtitle|ol|ul|pre|table|poem|blockquote|div|subscription)$/; | ||
| 59 | |||
| 60 | use constant { | ||
| 61 | RELATION_TYPE_FB3_BOOK => | ||
| 62 | 'http://www.fictionbook.org/FictionBook3/relationships/Book', | ||
| 63 | RELATION_TYPE_FB3_BODY => | ||
| 64 | 'http://www.fictionbook.org/FictionBook3/relationships/body', | ||
| 65 | RELATION_TYPE_OPC_THUMBNAIL => | ||
| 66 | 'http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail', | ||
| 67 | RELATION_TYPE_CORE_PROP => | ||
| 68 | 'http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties', | ||
| 69 | RELATION_TYPE_FB3_IMAGES => | ||
| 70 | 'http://www.fictionbook.org/FictionBook3/relationships/image', | ||
| 71 | }; | ||
| 72 | |||
| 73 | our $Validator = FB3::Validator->new( $XsdPath ); | ||
| 74 | |||
| 75 | my $Finish; | ||
| 76 | my $CharsProcessed = 0; | ||
| 77 | my %ImageHash; | ||
| 78 | my %HrefHash; | ||
| 79 | my %IdHash; | ||
| 80 | my %NoteHash; | ||
| 81 | my %NoteIdHash; | ||
| 82 | |||
| 83 | my $TmpDir = File::Temp::tempdir( CLEANUP => 1 ); | ||
| 84 | my $FB3TmpDir = $TmpDir.'/unzipped_'.File::Basename::basename($In); | ||
| 85 | my $UnzipMessage = `/usr/bin/unzip -o -d $FB3TmpDir $In 2>&1`; | ||
| 86 | die "Unzipping fb3 error: $UnzipMessage" if $?; | ||
| 87 | |||
| 88 | my $FB3Package = FB3->new( from_dir => $FB3TmpDir ); | ||
| 89 | my $FB3Body = $FB3Package->Body; | ||
| 90 | my $BodyXML = $FB3Body->Content; | ||
| 91 | my $Parser = XML::LibXML->new(); | ||
| 92 | my $BodyDoc = $Parser->load_xml( string => $BodyXML, huge => 1 ); | ||
| 93 | my $RootNode = $BodyDoc->getDocumentElement(); | ||
| 94 | my $XPC = XML::LibXML::XPathContext->new($RootNode); | ||
| 95 | $XPC->registerNs('fb', &NS_FB3_BODY); | ||
| 96 | $XPC->registerNs('fbd', &NS_FB3_DESCRIPTION); | ||
| 97 | |||
| 98 | my $CharsFull; | ||
| 99 | |||
| 100 | if ($WorkType eq 'trial') { | ||
| 101 | $CharsFull = length($RootNode->textContent); | ||
| 102 | foreach my $TrialOnlyNode ($XPC->findnodes('/fb:fb3-body/fb:section[@output="trial-only"]',$RootNode)) { | ||
| 103 | $CharsFull -= (length($TrialOnlyNode->textContent) || 0); | ||
| 104 | } | ||
| 105 | $CutChars ||= DEFAULT_TRIAL_PERCENT * 0.01 * $CharsFull; | ||
| 106 | ProceedNodeTrial($RootNode); | ||
| 107 | } elsif ($WorkType eq 'output') { | ||
| 108 | ProceedNodeOut($RootNode); | ||
| 109 | } | ||
| 110 | |||
| 111 | CollectElementStat($RootNode); | ||
| 112 | CleanLinks($FB3Body); | ||
| 113 | CleanImages($FB3Package); | ||
| 114 | CleanNotes($FB3Package); | ||
| 115 | |||
| 116 | $BodyDoc->toFile($FB3Body->PhysicalName, 0); | ||
| 117 | |||
| 118 | if ($WorkType eq 'trial') { | ||
| 119 | |||
| 120 | my $CharsTrial = length($RootNode->textContent); | ||
| 121 | my $FB3Descr = $FB3Package->Meta; | ||
| 122 | my $DescrXML = $FB3Descr->Content; | ||
| 123 | my $DescrDoc = $Parser->load_xml( string => $DescrXML, huge => 1 ); | ||
| 124 | my $DescrNode = $DescrDoc->getDocumentElement(); | ||
| 125 | |||
| 126 | if ( my $FragNode = $XPC->findnodes("/fbd:fb3-description/fbd:fb3-fragment", $DescrDoc)->[0] ) { | ||
| 127 | $FragNode->unbindNode(); | ||
| 128 | } | ||
| 129 | |||
| 130 | my $FB3FragmentNode = $DescrDoc->createElement('fb3-fragment'); | ||
| 131 | |||
| 132 | $FB3FragmentNode->setAttribute('full_length', $CharsFull); | ||
| 133 | $FB3FragmentNode->setAttribute('fragment_length', $CharsTrial); | ||
| 134 | |||
| 135 | $DescrNode->appendChild($FB3FragmentNode); | ||
| 136 | $DescrDoc->toFile($FB3Descr->PhysicalName, 0); | ||
| 137 | } | ||
| 138 | |||
| 139 | die "Empty body in result file" unless scalar @{$XPC->findnodes("/fb:fb3-body/fb:section", $RootNode)}; | ||
| 140 | |||
| 141 | ZipFolder("$FB3TmpDir/", $Out); | ||
| 142 | if ( my $ValidationError = $Validator->Validate($Out) ) { | ||
| 143 | unless ($Force) { | ||
| 144 | unlink $Out; | ||
| 145 | die "Validation error: $ValidationError"; | ||
| 146 | } | ||
| 147 | } | ||
| 148 | |||
| 149 | sub CollectElementStat { | ||
| 150 | my $Node = shift || return; | ||
| 151 | #быстрее, чем //xpath | ||
| 152 | for my $ChildNode ($Node->childNodes) { | ||
| 153 | |||
| 154 | next if $ChildNode->nodeName eq '#text'; | ||
| 155 | |||
| 156 | CollectElementStat($ChildNode); | ||
| 157 | |||
| 158 | $ImageHash{$ChildNode->getAttribute('src')} = 1 if $ChildNode->nodeName eq 'img'; | ||
| 159 | $NoteHash{$ChildNode->getAttribute('href')} = 1 if $ChildNode->nodeName eq 'note'; | ||
| 160 | |||
| 161 | if (my $Id = $ChildNode->getAttribute('id')) { | ||
| 162 | |||
| 163 | if ($ChildNode->nodeName eq 'notebody') { | ||
| 164 | $NoteIdHash{$Id} = 1; | ||
| 165 | } else { | ||
| 166 | $IdHash{$Id} = 1; | ||
| 167 | } | ||
| 168 | } | ||
| 169 | |||
| 170 | if ($ChildNode->nodeName eq 'a') { | ||
| 171 | my $Href = $ChildNode->getAttribute('xlink:href'); | ||
| 172 | $HrefHash{$Href} = 1 if ($Href and $Href =~ s/^#//); | ||
| 173 | } | ||
| 174 | } | ||
| 175 | } | ||
| 176 | |||
| 177 | sub ProceedNodeOut { | ||
| 178 | my $Node = shift || return; | ||
| 179 | |||
| 180 | #актуален только первый уровень section | ||
| 181 | for my $ChildNode ($Node->childNodes) { | ||
| 182 | next unless $ChildNode->nodeName eq 'section'; | ||
| 183 | my $Output = $ChildNode->getAttribute('output'); | ||
| 184 | $ChildNode->unbindNode() if ($Output and $Output eq 'trial-only'); | ||
| 185 | } | ||
| 186 | } | ||
| 187 | |||
| 188 | sub ProceedNodeTrial { | ||
| 189 | my $Node = shift || return; | ||
| 190 | my $ImmortalBranch = shift || 0; # не убивать потомков | ||
| 191 | |||
| 192 | my $NodeName = $Node->nodeName; | ||
| 193 | if ($NodeName eq '#text') { | ||
| 194 | $CharsProcessed += length($Node->nodeValue); | ||
| 195 | $Node->unbindNode() if ($Finish && !$ImmortalBranch); | ||
| 196 | return; | ||
| 197 | } | ||
| 198 | |||
| 199 | my $OuterImmortal; | ||
| 200 | if ($NodeName eq 'section') { | ||
| 201 | |||
| 202 | if ( #обрабатываем логику атрибута 'output' #только внешние section | ||
| 203 | $Node->parentNode->nodeName eq 'fb3-body' | ||
| 204 | && defined $Node->getAttribute('output') | ||
| 205 | ) { | ||
| 206 | my $SectionOutput = $Node->getAttribute('output'); | ||
| 207 | if ($SectionOutput eq 'payed') { #пропускаем ноду и забываем | ||
| 208 | $Node->unbindNode(); | ||
| 209 | return $Node; | ||
| 210 | } | ||
| 211 | if ($SectionOutput eq 'trial' || $SectionOutput eq 'trial-only') { #всегда бессмертны | ||
| 212 | $OuterImmortal = 1; | ||
| 213 | $ImmortalBranch = 1; | ||
| 214 | $InTrial = 1 if $WorkType eq 'trial'; #работаем триалку и зашли внутрь нее | ||
| 215 | } elsif ($InTrial) { #триалка закончилась, ну и хорош | ||
| 216 | $InTrial = 0; | ||
| 217 | $Finish = 1; | ||
| 218 | } | ||
| 219 | } | ||
| 220 | |||
| 221 | $Node->setAttribute('first-char-pos', $CharsProcessed+1); | ||
| 222 | |||
| 223 | } elsif ($NodeName eq 'title' && ($Node->parentNode->nodeName eq 'section' || $Node->parentNode->nodeName eq 'notes')) { # заголовок секции, пригодится | ||
| 224 | $ImmortalBranch = 1; | ||
| 225 | |||
| 226 | } elsif ( $NodeName eq 'img' && (!$Finish || $ImmortalBranch) ) { | ||
| 227 | |||
| 228 | my $ImgSrc = $Node->getAttribute('src'); | ||
| 229 | $ImageHash{$ImgSrc} = 1 if $ImgSrc; | ||
| 230 | |||
| 231 | } elsif ( $NodeName eq 'note' && (!$Finish || $ImmortalBranch) ) { | ||
| 232 | |||
| 233 | my $NoteHref = $Node->getAttribute('href'); | ||
| 234 | $NoteHash{$NoteHref} = 1 if $NoteHref; | ||
| 235 | |||
| 236 | } elsif ($NodeName eq 'notebody') { | ||
| 237 | |||
| 238 | if (!$NoteHash{$Node->getAttribute('id')}) { | ||
| 239 | $Node->unbindNode(); | ||
| 240 | } else { | ||
| 241 | $ImmortalBranch = 1; | ||
| 242 | } | ||
| 243 | |||
| 244 | } | ||
| 245 | |||
| 246 | for my $ChildNode ($Node->childNodes) { | ||
| 247 | |||
| 248 | if ($ChildNode->nodeName =~ /$Blocks/) { | ||
| 249 | if ( ($CharsProcessed + length($ChildNode->textContent)) >= $CutChars ) { | ||
| 250 | $Finish = 1; | ||
| 251 | } | ||
| 252 | } | ||
| 253 | |||
| 254 | ProceedNodeTrial($ChildNode, $ImmortalBranch); | ||
| 255 | } | ||
| 256 | |||
| 257 | if ($NodeName eq 'section') { | ||
| 258 | my @SectionChildren = $Node->nonBlankChildNodes; | ||
| 259 | $Node->setAttribute('clipped', 'true') if $Finish && !$OuterImmortal; | ||
| 260 | if (scalar @SectionChildren == 0 || (scalar @SectionChildren == 1 && $SectionChildren[0]->nodeName eq 'title') | ||
| 261 | || (scalar @SectionChildren == 2 && $SectionChildren[0]->nodeName eq 'title' && ($SectionChildren[1]->nodeName eq 'epigraph' || $SectionChildren[1]->nodeName eq 'annotation')) | ||
| 262 | ) { # проверка, что в секции нет ничего кроме заголовка | ||
| 263 | $SectionChildren[1]->unbindNode() if ($SectionChildren[1] && ($SectionChildren[1]->nodeName eq 'epigraph' || $SectionChildren[1]->nodeName eq 'annotation')); | ||
| 264 | my $ClippedNode = XML::LibXML::Element->new('clipped'); | ||
| 265 | $Node->appendChild($ClippedNode); | ||
| 266 | } | ||
| 267 | } | ||
| 268 | |||
| 269 | if ($NodeName eq 'image') { | ||
| 270 | $ImageFileName = $Node->getAttribute('href'); | ||
| 271 | $ImageFileName =~ s/^#//; | ||
| 272 | # Если нет, то игнорируем, потому что мы уже получили триал и теперь уже работаем с ним | ||
| 273 | if ( -f $ImagesPath.$ImageFileName ) { | ||
| 274 | my ($x, $y) = imgsize($ImagesPath.$ImageFileName); | ||
| 275 | $CharsProcessed += 200 if ($x && $y && $x >= 100 && $y >= 100); | ||
| 276 | } | ||
| 277 | } | ||
| 278 | |||
| 279 | if ( $Finish && !$ImmortalBranch && (!$Node->firstChild || Trim(InNode($Node)) eq '') ) { | ||
| 280 | $Node->unbindNode(); # если нет потомков - рубим. | ||
| 281 | } | ||
| 282 | |||
| 283 | return $Node; | ||
| 284 | } | ||
| 285 | |||
| 286 | sub InNode { | ||
| 287 | my $Node = shift; | ||
| 288 | join "",map {$_->toString} $Node->childNodes; | ||
| 289 | } | ||
| 290 | |||
| 291 | sub Trim { | ||
| 292 | my $Str = shift; | ||
| 293 | $Str =~ s/[\n\r]//g; | ||
| 294 | $Str =~ s/^\s+//g; | ||
| 295 | $Str =~ s/\s+$//g; | ||
| 296 | return $Str; | ||
| 297 | } | ||
| 298 | |||
| 299 | sub NReplace { | ||
| 300 | my $Node = shift; | ||
| 301 | my $Parent = $Node->parentNode(); | ||
| 302 | my $Clone = $BodyDoc->createElement($Parent->nodeName()); | ||
| 303 | foreach ($Parent->getAttributes) { | ||
| 304 | $Clone->setAttribute($_->nodeName => $_->value); | ||
| 305 | } | ||
| 306 | foreach my $PNode ($Parent->childNodes()) { | ||
| 307 | if ($PNode == $Node) { | ||
| 308 | foreach ($PNode->childNodes) { | ||
| 309 | $Clone->appendChild($_); | ||
| 310 | } | ||
| 311 | next; | ||
| 312 | } | ||
| 313 | $Clone->appendChild($PNode); | ||
| 314 | } | ||
| 315 | $Parent->replaceNode($Clone); | ||
| 316 | } | ||
| 317 | |||
| 318 | sub CleanLinks { | ||
| 319 | my $FB3Body = shift; | ||
| 320 | |||
| 321 | #notebody | ||
| 322 | foreach my $Id (keys %NoteIdHash) { | ||
| 323 | next if exists $NoteHash{$Id}; | ||
| 324 | my $Ids = $XPC->findnodes('/fb:fb3-body/fb:notes/fb:notebody[@id="'.$Id.'"]'); | ||
| 325 | foreach my $NodeIds (@$Ids) { | ||
| 326 | $NodeIds->unbindNode(); | ||
| 327 | } | ||
| 328 | } | ||
| 329 | |||
| 330 | #остальные ноды | ||
| 331 | foreach my $Id (keys %IdHash) { | ||
| 332 | next if exists $HrefHash{$Id}; | ||
| 333 | my $Ids = $XPC->findnodes('//*[@id="'.$Id.'"]'); | ||
| 334 | foreach my $NodeIds (@$Ids) { | ||
| 335 | next if $NodeIds->nodeName() eq 'section'; | ||
| 336 | next if $NodeIds->nodeName() eq 'notebody'; | ||
| 337 | if ($NodeIds->nodeName eq 'span') { | ||
| 338 | NReplace($NodeIds); | ||
| 339 | } else { | ||
| 340 | $NodeIds->removeAttribute('id'); | ||
| 341 | } | ||
| 342 | } | ||
| 343 | delete $IdHash{$Id}; | ||
| 344 | } | ||
| 345 | |||
| 346 | #ссылки | ||
| 347 | foreach my $Href (keys %HrefHash) { | ||
| 348 | next if exists $IdHash{$Href}; | ||
| 349 | my $Hrefs = $XPC->findnodes('//fb:a[@xlink:href="#'.$Href.'"]'); | ||
| 350 | foreach my $NodeHrefs (@$Hrefs) { | ||
| 351 | if ($NodeHrefs->getAttribute('id')) { | ||
| 352 | $NodeHrefs->removeAttribute('xlink:href'); | ||
| 353 | $NodeHrefs->setNodeName('span'); | ||
| 354 | next; | ||
| 355 | } | ||
| 356 | NReplace($NodeHrefs); | ||
| 357 | } | ||
| 358 | delete $HrefHash{$Href}; | ||
| 359 | } | ||
| 360 | |||
| 361 | } | ||
| 362 | |||
| 363 | sub CleanNotes { | ||
| 364 | my $Node = $XPC->findnodes("/fb:fb3-body/fb:notes",$RootNode)->[0]; | ||
| 365 | return unless $Node; | ||
| 366 | my @NotesChildren = $Node->nonBlankChildNodes; | ||
| 367 | if ( scalar @NotesChildren == 0 | ||
| 368 | || (scalar @NotesChildren == 1 && $NotesChildren[0]->nodeName eq 'title') | ||
| 369 | || (scalar @NotesChildren == 2 && $NotesChildren[0]->nodeName eq 'title' && $NotesChildren[1]->nodeName eq 'epigraph') | ||
| 370 | ) { | ||
| 371 | $Node->unbindNode(); | ||
| 372 | } | ||
| 373 | } | ||
| 374 | |||
| 375 | sub CleanImages { | ||
| 376 | my $FB3 = shift; | ||
| 377 | my $FB3Body = $FB3->Body; | ||
| 378 | |||
| 379 | my %TrialImageFiles; | ||
| 380 | my @FilePathes; | ||
| 381 | |||
| 382 | # иногда так случается, что в исходном FB3 нет обложки | ||
| 383 | my $cover = eval { $FB3->Cover->PhysicalName }; | ||
| 384 | # следующая строка чтобы не удалить обложку | ||
| 385 | $TrialImageFiles{$cover} = 1 if $cover; | ||
| 386 | |||
| 387 | my @ImagesAll = $FB3Body->Relations(type => RELATION_TYPE_FB3_IMAGES); | ||
| 388 | foreach (@ImagesAll) { # эта картинка сохранилась в триалке | ||
| 389 | my $NewPart = $FB3->Part( $_->{TargetFullName} ); | ||
| 390 | my $FullPath = $NewPart->PhysicalName; | ||
| 391 | push (@FilePathes, $FullPath); | ||
| 392 | if ($ImageHash{$_->{Id}}) { | ||
| 393 | $TrialImageFiles{$FullPath} = 1; # Нужный файлик, запомним. | ||
| 394 | } else { | ||
| 395 | $FB3Body->RemoveRelations( id => $_->{Id} ); | ||
| 396 | } | ||
| 397 | } | ||
| 398 | |||
| 399 | foreach (@FilePathes) { | ||
| 400 | unlink $_ unless $TrialImageFiles{$_}; | ||
| 401 | } | ||
| 402 | |||
| 403 | return 1; | ||
| 404 | } | ||
| 405 | |||
| 406 | sub ZipFolder { | ||
| 407 | |||
| 408 | my ($SourceFileName, $TargetFileName, @first) = @_; | ||
| 409 | |||
| 410 | open my $FH, '>', "$TargetFileName"; | ||
| 411 | close $FH; | ||
| 412 | my $fn_abs = abs_path("$TargetFileName"); | ||
| 413 | unlink "$TargetFileName"; | ||
| 414 | |||
| 415 | my $old_dir = cwd(); | ||
| 416 | chdir "$SourceFileName"; | ||
| 417 | my $cmd = "zip -Xr9Dq $fn_abs " . join(' ', @first) . " *"; | ||
| 418 | my $CmdResult = `$cmd`; | ||
| 419 | warn $CmdResult if $CmdResult; | ||
| 420 | chdir $old_dir if $old_dir; | ||
| 421 | } | ||
| 422 | |||
| 423 | sub help { | ||
| 424 | |||
| 425 | print qq{ | ||
| 426 | USAGE: cutfb3.pl --in <input.file> --out <output.file> [options] | ||
| 427 | |||
| 428 | --chars|c : maximum caracters in trial fragment. default is @{[DEFAULT_TRIAL_PERCENT]}% of text. | ||
| 429 | --imagespath : /path/to/fb3/images | ||
| 430 | --type|t : type of work `trial' or `output'. default is `trial' | ||
| 431 | --help|h : print this text | ||
| 432 | |||
| 433 | }; | ||
| 434 | exit; | ||
| 435 | } | ||
| 436 | |||
| 437 | |||
| 438 | 1; |