| 1 | #!/bin/false | ||
| 2 | package App::Nrepo::Plugin::Yum; | ||
| 3 | |||
| 4 | use Moo; | ||
| 5 | use strictures 2; | ||
| 6 | |||
| 7 | use Carp; | ||
| 8 | use File::Copy qw(copy); | ||
| 9 | use File::Find qw(find); | ||
| 10 | use File::Basename qw(basename dirname); | ||
| 11 | use IO::Zlib; | ||
| 12 | use Params::Validate qw(:all); | ||
| 13 | use Data::Dumper qw(Dumper); | ||
| 14 | use Time::HiRes qw(gettimeofday tv_interval); | ||
| 15 | use XML::Twig; | ||
| 16 | use XML::LibXML; | ||
| 17 | |||
| 18 | our $VERSION = '0.1'; # VERSION | ||
| 19 | |||
| 20 | with 'App::Nrepo::Plugin::Base'; | ||
| 21 | |||
| 22 | has packages_dir => ( is => 'ro', default => 'Packages' ); | ||
| 23 | has repodata_dir => ( is => 'ro', default => 'repodata' ); | ||
| 24 | |||
| 25 | sub get_metadata { | ||
| 26 | my $self = shift; | ||
| 27 | my $arch = shift; | ||
| 28 | |||
| 29 | my $base_dir = File::Spec->catdir( $self->dir(), $arch ); | ||
| 30 | my $packages; | ||
| 31 | |||
| 32 | my @metadata_files = | ||
| 33 | ( { type => 'repomd', location => 'repodata/repomd.xml' } ); | ||
| 34 | for my $m (@metadata_files) { | ||
| 35 | my $type = $m->{'type'}; | ||
| 36 | my $location = $m->{'location'}; | ||
| 37 | my $m_url = join( '/', ( $self->url, $location ) ); | ||
| 38 | $m_url =~ s/%ARCH%/$arch/; | ||
| 39 | my $dest_file = File::Spec->catfile( $base_dir, $location ); | ||
| 40 | my $dest_dir = dirname($dest_file); | ||
| 41 | |||
| 42 | # Make sure dir exists | ||
| 43 | $self->make_dir($dest_dir); | ||
| 44 | |||
| 45 | # Check if we have the local file | ||
| 46 | my $download; | ||
| 47 | if ( $type eq 'repomd' ) { | ||
| 48 | $download++; | ||
| 49 | } | ||
| 50 | elsif ( | ||
| 51 | !$self->validate_file( | ||
| 52 | filename => $dest_file, | ||
| 53 | check => $m->{'validate'}->{'type'}, | ||
| 54 | value => $m->{'validate'}->{'value'}, | ||
| 55 | ) | ||
| 56 | ) | ||
| 57 | { | ||
| 58 | $download++; | ||
| 59 | } | ||
| 60 | |||
| 61 | # Grab the file | ||
| 62 | if ($download) { | ||
| 63 | $self->download_binary_file( url => $m_url, dest => $dest_file ); | ||
| 64 | } | ||
| 65 | else { | ||
| 66 | $self->logger->debug( | ||
| 67 | sprintf( | ||
| 68 | 'get_metadata: repo: %s arch: %s file: %s skipping as its deemed up to date', | ||
| 69 | $self->repo(), $arch, $location | ||
| 70 | ) | ||
| 71 | ); | ||
| 72 | } | ||
| 73 | |||
| 74 | # Parse the xml and retrieve the primary file location | ||
| 75 | if ( $type eq 'repomd' ) { | ||
| 76 | my $data = $self->parse_repomd($dest_file); | ||
| 77 | push @metadata_files, @{$data}; | ||
| 78 | } | ||
| 79 | |||
| 80 | # Parse the primary metadata file | ||
| 81 | if ( $type eq 'primary' ) { | ||
| 82 | $packages = $self->parse_primary($dest_file); | ||
| 83 | } | ||
| 84 | } | ||
| 85 | return $packages; | ||
| 86 | } | ||
| 87 | |||
| 88 | sub read_metadata { | ||
| 89 | my $self = shift; | ||
| 90 | my $arch = shift; | ||
| 91 | |||
| 92 | my $base_dir = File::Spec->catdir( $self->dir(), $arch ); | ||
| 93 | my $files = {}; | ||
| 94 | |||
| 95 | my @metadata_files = | ||
| 96 | ( { type => 'repomd', location => 'repodata/repomd.xml' } ); | ||
| 97 | for my $m (@metadata_files) { | ||
| 98 | my $type = $m->{'type'}; | ||
| 99 | my $location = $m->{'location'}; | ||
| 100 | my $dest_file = File::Spec->catfile( $base_dir, $location ); | ||
| 101 | my $dest_dir = dirname($dest_file); | ||
| 102 | |||
| 103 | $files->{$location}++; | ||
| 104 | |||
| 105 | if ( -f $dest_file ) { | ||
| 106 | |||
| 107 | # Parse the xml and retrieve the primary file location | ||
| 108 | if ( $type eq 'repomd' ) { | ||
| 109 | my $data = $self->parse_repomd($dest_file); | ||
| 110 | push @metadata_files, @{$data}; | ||
| 111 | } | ||
| 112 | |||
| 113 | # Parse the primary metadata file | ||
| 114 | if ( $type eq 'primary' ) { | ||
| 115 | my $contents = $self->get_gzip_contents($dest_file); | ||
| 116 | for my $file ( @{ $self->parse_primary($dest_file) } ) { | ||
| 117 | $files->{ $file->{'location'} }++; | ||
| 118 | } | ||
| 119 | } | ||
| 120 | } | ||
| 121 | } | ||
| 122 | return $files; | ||
| 123 | } | ||
| 124 | |||
| 125 | sub parse_repomd { | ||
| 126 | my $self = shift; | ||
| 127 | my $file = shift; | ||
| 128 | |||
| 129 | # XXX TODO rework this with XML::LibXML as its far faster | ||
| 130 | my $twig = XML::Twig->new( TwigRoots => { data => 1 } ); | ||
| 131 | $twig->parsefile($file); | ||
| 132 | |||
| 133 | my $root = $twig->root; | ||
| 134 | my @e = $root->children(); | ||
| 135 | my @files; | ||
| 136 | for my $e (@e) { | ||
| 137 | my $data = {}; | ||
| 138 | $data->{'type'} = $e->att('type'); | ||
| 139 | for my $c ( $e->children() ) { | ||
| 140 | if ( $c->name eq 'location' ) { | ||
| 141 | $data->{'location'} = $c->att('href'); | ||
| 142 | } | ||
| 143 | elsif ( $c->name eq 'checksum' ) { | ||
| 144 | $data->{'checksum'}->{'type'} = $c->att('type'); | ||
| 145 | $data->{'checksum'}->{'value'} = $c->text; | ||
| 146 | } | ||
| 147 | elsif ( $c->name eq 'size' ) { | ||
| 148 | $data->{'size'}->{'type'} = 'size'; | ||
| 149 | $data->{'size'}->{'value'} = $c->text; | ||
| 150 | } | ||
| 151 | } | ||
| 152 | |||
| 153 | # For some reason i have found a few repomd.xml files that do NOT | ||
| 154 | # have a size attribute ...specifically updateinfo type | ||
| 155 | # so as a work around we will try size if checksums is not enabled | ||
| 156 | # however for that file we'll revert to checksums if size is not available | ||
| 157 | if ( !$self->checksums() && $data->{'size'} ) { | ||
| 158 | $data->{'validate'} = $data->{'size'}; | ||
| 159 | } | ||
| 160 | else { | ||
| 161 | $data->{'validate'} = $data->{'checksum'}; | ||
| 162 | } | ||
| 163 | $self->logger->log_and_croak( | ||
| 164 | level => 'error', | ||
| 165 | message => "repomd xml not valid: $file" | ||
| 166 | ) unless $data->{'location'}; | ||
| 167 | push @files, $data; | ||
| 168 | } | ||
| 169 | |||
| 170 | return \@files; | ||
| 171 | } | ||
| 172 | |||
| 173 | sub parse_primary { | ||
| 174 | my $self = shift; | ||
| 175 | my $dest_file = shift; | ||
| 176 | my $io_fh = IO::Zlib->new( $dest_file, 'rb' ); | ||
| 177 | my $xml = XML::LibXML->load_xml( IO => $io_fh ); | ||
| 178 | my $t0 = [gettimeofday]; | ||
| 179 | my $packages = []; | ||
| 180 | for my $p ( $xml->getElementsByTagName('package') ) { | ||
| 181 | my ($n) = $p->getChildrenByTagName('name'); | ||
| 182 | my ($l) = $p->getChildrenByTagName('location'); | ||
| 183 | my ($s) = $p->getChildrenByTagName('size'); | ||
| 184 | my ($c) = $p->getChildrenByTagName('checksum'); | ||
| 185 | my $data = { | ||
| 186 | name => $n->textContent, | ||
| 187 | location => $l->getAttribute('href'), | ||
| 188 | size => { | ||
| 189 | type => 'size', | ||
| 190 | value => $s->getAttribute('package'), | ||
| 191 | }, | ||
| 192 | checksum => { | ||
| 193 | type => $c->getAttribute('type'), | ||
| 194 | value => $c->textContent, | ||
| 195 | }, | ||
| 196 | }; | ||
| 197 | if ( !$self->checksums() && $data->{'size'} ) { | ||
| 198 | $data->{'validate'} = $data->{'size'}; | ||
| 199 | } | ||
| 200 | else { | ||
| 201 | $data->{'validate'} = $data->{'checksum'}; | ||
| 202 | } | ||
| 203 | push @{$packages}, $data; | ||
| 204 | } | ||
| 205 | my $elapsed = tv_interval($t0); | ||
| 206 | $self->logger->debug( | ||
| 207 | sprintf( 'parse_primary: file: %s took: %s seconds', $dest_file, $elapsed ) | ||
| 208 | ); | ||
| 209 | return $packages; | ||
| 210 | } | ||
| 211 | |||
| 212 | sub get_packages { | ||
| 213 | my $self = shift; | ||
| 214 | my %o = validate( | ||
| 215 | @_, | ||
| 216 | { | ||
| 217 | arch => { type => SCALAR }, | ||
| 218 | packages => { type => ARRAYREF }, | ||
| 219 | } | ||
| 220 | ); | ||
| 221 | |||
| 222 | my $arch = $o{'arch'}; | ||
| 223 | my $base_dir = File::Spec->catdir( $self->dir(), $arch ); | ||
| 224 | |||
| 225 | for my $package ( @{ $o{'packages'} } ) { | ||
| 226 | my $name = $package->{'name'}; | ||
| 227 | my $location = $package->{'location'}; | ||
| 228 | |||
| 229 | my $p_url = join( '/', ( $self->url, $location ) ); | ||
| 230 | $p_url =~ s/%ARCH%/$arch/; | ||
| 231 | my $dest_file = File::Spec->catfile( $base_dir, $location ); | ||
| 232 | my $dest_dir = dirname($dest_file); | ||
| 233 | |||
| 234 | # Make sure dir exists | ||
| 235 | $self->make_dir($dest_dir); | ||
| 236 | |||
| 237 | # Check if we have the local file | ||
| 238 | if ( | ||
| 239 | !$self->validate_file( | ||
| 240 | filename => $dest_file, | ||
| 241 | check => $package->{'validate'}->{'type'}, | ||
| 242 | value => $package->{'validate'}->{'value'}, | ||
| 243 | ) | ||
| 244 | ) | ||
| 245 | { | ||
| 246 | $self->logger->notice( | ||
| 247 | sprintf( | ||
| 248 | 'get_packages: repo: %s arch: %s package: %s', | ||
| 249 | $self->repo(), $arch, $location | ||
| 250 | ) | ||
| 251 | ); | ||
| 252 | $self->download_binary_file( url => $p_url, dest => $dest_file ); | ||
| 253 | } | ||
| 254 | else { | ||
| 255 | $self->logger->debug( | ||
| 256 | sprintf( | ||
| 257 | 'get_packages: repo: %s arch: %s package: %s skipping as its deemed up to date', | ||
| 258 | $self->repo(), $arch, $location | ||
| 259 | ) | ||
| 260 | ); | ||
| 261 | } | ||
| 262 | } | ||
| 263 | } | ||
| 264 | |||
| 265 | sub clean_files { | ||
| 266 | my $self = shift; | ||
| 267 | my %o = validate( | ||
| 268 | @_, | ||
| 269 | { | ||
| 270 | arch => { type => SCALAR }, | ||
| 271 | files => { type => HASHREF }, | ||
| 272 | } | ||
| 273 | ); | ||
| 274 | |||
| 275 | my $arch = $o{'arch'}; | ||
| 276 | my $base_dir = File::Spec->catdir( $self->dir(), $arch ); | ||
| 277 | |||
| 278 | find( | ||
| 279 | sub { | ||
| 280 | if ( $_ !~ /^[\.]+$/ ) { | ||
| 281 | my $file = $_; | ||
| 282 | if ( -f $file ) { | ||
| 283 | my $rel = File::Spec->abs2rel( $File::Find::name, $base_dir ); | ||
| 284 | unless ( $o{'files'}->{$rel} ) { | ||
| 285 | $self->logger->info( | ||
| 286 | "clean_files: removing non referenced file: ${File::Find::name}"); | ||
| 287 | unlink $file | ||
| 288 | or $self->logger->log_and_croak( | ||
| 289 | level => 'error', | ||
| 290 | message => "Failed to remove file: ${file}: $!" | ||
| 291 | ); | ||
| 292 | } | ||
| 293 | } | ||
| 294 | } | ||
| 295 | }, | ||
| 296 | $base_dir, | ||
| 297 | ); | ||
| 298 | } | ||
| 299 | |||
| 300 | sub add_file { | ||
| 301 | my $self = shift; | ||
| 302 | my $arch = shift; | ||
| 303 | my $files = shift; | ||
| 304 | |||
| 305 | unless ( $self->validate_arch($arch) ) { | ||
| 306 | $self->logger->log_and_croak( | ||
| 307 | level => 'error', | ||
| 308 | message => sprintf 'add_file: arch: %s is not in config for repo: %s', | ||
| 309 | $arch, $self->repo() | ||
| 310 | ); | ||
| 311 | } | ||
| 312 | |||
| 313 | my $package_dir = | ||
| 314 | File::Spec->catdir( $self->dir(), $arch, $self->packages_dir() ); | ||
| 315 | $self->make_dir($package_dir) unless -d $package_dir; | ||
| 316 | |||
| 317 | for my $file ( @${files} ) { | ||
| 318 | my $filename = basename($file); | ||
| 319 | my $dest_file = | ||
| 320 | File::Spec->catfile( $self->dir(), $arch, $package_dir, $filename ); | ||
| 321 | $self->logger->debug( | ||
| 322 | sprintf 'add_file: repo: %s arch: %s file: %s dest_file: %s', | ||
| 323 | $self->repo(), $arch, $file, $dest_file ); | ||
| 324 | |||
| 325 | if ( -f $dest_file && !$self->force() ) { | ||
| 326 | $self->logger->log_and_croak( | ||
| 327 | level => 'error', | ||
| 328 | message => sprintf | ||
| 329 | 'add_file: repo: %s dest_file exists and force not enabled: %s', | ||
| 330 | $self->repo(), $dest_file | ||
| 331 | ); | ||
| 332 | } | ||
| 333 | |||
| 334 | copy( $file, $dest_file ) || $self->logger->log_and_croak( | ||
| 335 | level => 'error', | ||
| 336 | message => sprintf | ||
| 337 | 'add_file: repo: %s failed to copy file to destination: %s', | ||
| 338 | $self->repo(), $! | ||
| 339 | ); | ||
| 340 | |||
| 341 | } | ||
| 342 | |||
| 343 | $self->init_arch($arch); | ||
| 344 | |||
| 345 | } | ||
| 346 | |||
| 347 | sub del_file { | ||
| 348 | my $self = shift; | ||
| 349 | my $arch = shift; | ||
| 350 | my $files = shift; | ||
| 351 | |||
| 352 | unless ( $self->validate_arch($arch) ) { | ||
| 353 | $self->logger->log_and_croak( | ||
| 354 | level => 'error', | ||
| 355 | message => sprintf 'del_file: arch: %s is not in config for repo: %s', | ||
| 356 | $arch, $self->repo() | ||
| 357 | ); | ||
| 358 | } | ||
| 359 | |||
| 360 | for my $file ( @${files} ) { | ||
| 361 | my $filename = basename($file); | ||
| 362 | my $dest_file = | ||
| 363 | File::Spec->catfile( $self->dir(), $arch, $self->packages_dir(), | ||
| 364 | $filename ); | ||
| 365 | $self->logger->debug( sprintf 'del_file: repo: %s arch: %s dest_file: %s', | ||
| 366 | $self->repo(), $arch, $dest_file ); | ||
| 367 | |||
| 368 | unless ( -f $dest_file ) { | ||
| 369 | $self->logger->log_and_croak( | ||
| 370 | level => 'error', | ||
| 371 | message => sprintf 'del_file: repo: %s dest_file: %s does not exist', | ||
| 372 | $self->repo(), $dest_file | ||
| 373 | ); | ||
| 374 | } | ||
| 375 | |||
| 376 | unlink $dest_file || $self->logger->log_and_croak( | ||
| 377 | level => 'error', | ||
| 378 | message => sprintf | ||
| 379 | 'del_file: repo: %s failed to del file: %s from destination: %s', | ||
| 380 | $self->repo(), $dest_file, $! | ||
| 381 | ); | ||
| 382 | |||
| 383 | } | ||
| 384 | |||
| 385 | $self->init_arch($arch); | ||
| 386 | } | ||
| 387 | |||
| 388 | sub init_arch { | ||
| 389 | my $self = shift; | ||
| 390 | my $arch = shift; | ||
| 391 | |||
| 392 | my $dir = File::Spec->catdir( $self->dir, $arch ); | ||
| 393 | |||
| 394 | $self->logger->debug( sprintf 'init_arch: repo: %s arch: %s dir: %s', | ||
| 395 | $self->repo(), $arch, $dir ); | ||
| 396 | |||
| 397 | $self->make_dir($dir); | ||
| 398 | $self->make_dir( File::Spec->catdir( $dir, $self->packages_dir() ) ); | ||
| 399 | |||
| 400 | #XXX add gpg | ||
| 401 | |||
| 402 | #TODO perhaps replace createrepo with pure perl version at some stage | ||
| 403 | my $createrepo_bin = $self->find_command_path('createrepo'); | ||
| 404 | |||
| 405 | unless ( $createrepo_bin and -x $createrepo_bin ) { | ||
| 406 | $self->logger->log_and_croak( | ||
| 407 | level => 'error', | ||
| 408 | message => sprintf( | ||
| 409 | 'init_arch: repo: %s arch: %s unable to find createrepo program in path', | ||
| 410 | $self->repo(), $arch | ||
| 411 | ), | ||
| 412 | ); | ||
| 413 | } | ||
| 414 | |||
| 415 | my @cmd = ( | ||
| 416 | $createrepo_bin, '--basedir', $dir, '--outputdir', $dir, | ||
| 417 | $self->packages_dir() | ||
| 418 | ); | ||
| 419 | |||
| 420 | # --update will reuse the existing metadata if the file is already defined and size/mtime matches | ||
| 421 | # dont do this if we're forcing or the repomd.xml doesnt exist | ||
| 422 | unless ( $self->force() ) { | ||
| 423 | if ( -f File::Spec->catfile( $dir, $self->repodata_dir(), 'repomd.xml' ) ) { | ||
| 424 | splice @cmd, 1, 0, '--update'; | ||
| 425 | } | ||
| 426 | } | ||
| 427 | |||
| 428 | $self->logger->debug( sprintf 'init_arch: running command: %s', | ||
| 429 | join( ' ', @cmd ) ); | ||
| 430 | unless ( system(@cmd) == 0 ) { | ||
| 431 | $self->logger->log_and_croak( | ||
| 432 | level => 'error', | ||
| 433 | message => sprintf( | ||
| 434 | 'init_arch: repo: %s failed to run command: %s with exit code: %s', | ||
| 435 | $self->repo(), join( ' ', @cmd ), $?, | ||
| 436 | ) | ||
| 437 | ); | ||
| 438 | } | ||
| 439 | |||
| 440 | } | ||
| 441 | |||
| 442 | sub type { | ||
| 443 | |||
| 444 | #my $self = shift; | ||
| 445 | return 'Yum'; | ||
| 446 | } | ||
| 447 | |||
| 448 | 1; |