| 1 | #!/bin/false | ||
| 2 | package App::Nrepo::Plugin::Base; | ||
| 3 | |||
| 4 | use Moo::Role; | ||
| 5 | use strictures 2; | ||
| 6 | use namespace::clean; | ||
| 7 | |||
| 8 | use App::Nrepo::Logger; | ||
| 9 | use Carp; | ||
| 10 | use Data::Dumper; | ||
| 11 | use Digest::SHA; | ||
| 12 | use File::Find qw(find); | ||
| 13 | use File::Path qw(make_path remove_tree); | ||
| 14 | use File::Spec; | ||
| 15 | use HTTP::Tiny; | ||
| 16 | use IO::Zlib; | ||
| 17 | use Module::Path qw[ module_path ]; | ||
| 18 | use Module::Runtime qw[ compose_module_name ]; | ||
| 19 | use Params::Validate qw(:all); | ||
| 20 | use Time::HiRes qw(gettimeofday tv_interval); | ||
| 21 | |||
| 22 | our $VERSION = '0.1'; # VERSION | ||
| 23 | |||
| 24 | has logger => ( is => 'ro', default => sub { App::Nrepo::Logger->new() } ); | ||
| 25 | has repo => ( is => 'ro', required => 1 ); | ||
| 26 | has dir => ( is => 'ro', required => 1 ); | ||
| 27 | has url => ( is => 'ro', optional => 1 ); | ||
| 28 | has checksums => ( is => 'ro', optional => 1 ); | ||
| 29 | has force => ( is => 'ro', optional => 1 ); | ||
| 30 | has arches => ( is => 'ro', required => 1 ); | ||
| 31 | has http => ( is => 'lazy' ); | ||
| 32 | has ssl_ca => ( is => 'ro', optional => 1 ); | ||
| 33 | has ssl_cert => ( is => 'ro', optional => 1 ); | ||
| 34 | has ssl_key => ( is => 'ro', optional => 1 ); | ||
| 35 | |||
| 36 | sub _build_http { | ||
| 37 | my $self = shift; | ||
| 38 | |||
| 39 | my %o; | ||
| 40 | $o{SSL_options}->{'SSL_ca_file'} = $self->ssl_ca() if $self->can('ssl_ca'); | ||
| 41 | $o{SSL_options}->{'SSL_cert_file'} = $self->ssl_cert() | ||
| 42 | if $self->can('ssl_cert'); | ||
| 43 | $o{SSL_options}->{'SSL_key_file'} = $self->ssl_key() if $self->can('ssl_key'); | ||
| 44 | |||
| 45 | return HTTP::Tiny->new(%o); | ||
| 46 | } | ||
| 47 | |||
| 48 | sub get_gzip_contents { | ||
| 49 | my $self = shift; | ||
| 50 | my $file = shift; | ||
| 51 | |||
| 52 | if ( -f $file ) { | ||
| 53 | { | ||
| 54 | my $fh = IO::Zlib->new( $file, 'rb' ); | ||
| 55 | |||
| 56 | #XXX for some reason this does not work | ||
| 57 | #local $/ = undef; | ||
| 58 | #my $contents = <$fh>; | ||
| 59 | #return $contents; | ||
| 60 | |||
| 61 | my @contents = <$fh>; | ||
| 62 | $fh->close; | ||
| 63 | return join( '', @contents ); | ||
| 64 | } | ||
| 65 | } | ||
| 66 | } | ||
| 67 | |||
| 68 | sub find_command_path { | ||
| 69 | my $self = shift; | ||
| 70 | my $command = shift || return; | ||
| 71 | |||
| 72 | my @path = File::Spec->path(); | ||
| 73 | for my $p (@path) { | ||
| 74 | my $command_path = File::Spec->catfile( $p, $command ); | ||
| 75 | return $command_path if -x $command_path; | ||
| 76 | } | ||
| 77 | return; | ||
| 78 | } | ||
| 79 | |||
| 80 | sub make_dir { | ||
| 81 | my $self = shift; | ||
| 82 | my $dir = shift; | ||
| 83 | if ( !-d $dir ) { | ||
| 84 | my $dirs = make_path($dir); | ||
| 85 | $self->logger->log_and_croak( | ||
| 86 | level => 'error', | ||
| 87 | message => "Failed to create path: ${dir}" | ||
| 88 | ) unless -d $dir; | ||
| 89 | $self->logger->debug("Created path: ${dir}"); | ||
| 90 | return 1; | ||
| 91 | } | ||
| 92 | return 0; | ||
| 93 | } | ||
| 94 | |||
| 95 | sub remove_dir { | ||
| 96 | my $self = shift; | ||
| 97 | my $dir = shift; | ||
| 98 | if ( -d $dir ) { | ||
| 99 | my $dirs = remove_tree($dir); | ||
| 100 | $self->logger->log_and_croak( | ||
| 101 | level => 'error', | ||
| 102 | message => "Failed to remove path: ${dir}" | ||
| 103 | ) if -d $dir; | ||
| 104 | $self->logger->debug("removed path: ${dir}"); | ||
| 105 | return 1; | ||
| 106 | } | ||
| 107 | return 0; | ||
| 108 | } | ||
| 109 | |||
| 110 | sub validate_arch { | ||
| 111 | my $self = shift; | ||
| 112 | my $arch = shift; | ||
| 113 | |||
| 114 | my $matched; | ||
| 115 | for my $a ( @{ $self->arches() } ) { | ||
| 116 | $matched++ if $a eq $arch; | ||
| 117 | } | ||
| 118 | |||
| 119 | return $matched; | ||
| 120 | } | ||
| 121 | |||
| 122 | sub validate_file { | ||
| 123 | my $self = shift; | ||
| 124 | my %o = validate( | ||
| 125 | @_, | ||
| 126 | { | ||
| 127 | filename => { type => SCALAR }, | ||
| 128 | check => { type => SCALAR }, | ||
| 129 | value => { type => SCALAR }, | ||
| 130 | } | ||
| 131 | ); | ||
| 132 | |||
| 133 | # If theres no file, its not valid | ||
| 134 | return 0 unless -f $o{'filename'}; | ||
| 135 | |||
| 136 | # If force is enabled, its not valid | ||
| 137 | return 0 if $self->force(); | ||
| 138 | |||
| 139 | # Check against size | ||
| 140 | if ( $o{'check'} eq 'size' ) { | ||
| 141 | return $self->_validate_file_size( $o{'filename'}, $o{'value'} ); | ||
| 142 | } | ||
| 143 | |||
| 144 | # Check against sha | ||
| 145 | elsif ( $o{'check'} eq 'sha' ) { | ||
| 146 | return $self->_validate_file_sha( $o{'filename'}, $o{'value'} ); | ||
| 147 | } | ||
| 148 | |||
| 149 | # Check against sha256 | ||
| 150 | elsif ( $o{'check'} eq 'sha256' ) { | ||
| 151 | return $self->_validate_file_sha256( $o{'filename'}, $o{'value'} ); | ||
| 152 | } | ||
| 153 | else { | ||
| 154 | $self->logger->log_and_croak( | ||
| 155 | level => 'error', | ||
| 156 | message => "unknown validation check type: $o{'check'}" | ||
| 157 | ); | ||
| 158 | } | ||
| 159 | } | ||
| 160 | |||
| 161 | sub _validate_file_size { | ||
| 162 | my $self = shift; | ||
| 163 | my $file = shift; | ||
| 164 | my $size = shift; | ||
| 165 | |||
| 166 | my @stats = stat($file); | ||
| 167 | my $file_size = $stats[7]; | ||
| 168 | |||
| 169 | return $file_size eq $size ? 1 : undef; | ||
| 170 | } | ||
| 171 | |||
| 172 | sub _validate_file_sha { | ||
| 173 | my $self = shift; | ||
| 174 | my $file = shift; | ||
| 175 | my $checksum = shift; | ||
| 176 | |||
| 177 | my $sha = Digest::SHA->new('sha1'); | ||
| 178 | $sha->addfile($file); | ||
| 179 | return $sha->hexdigest eq $checksum ? 1 : undef; | ||
| 180 | } | ||
| 181 | |||
| 182 | sub _validate_file_sha256 { | ||
| 183 | my $self = shift; | ||
| 184 | my $file = shift; | ||
| 185 | my $checksum = shift; | ||
| 186 | |||
| 187 | my $sha = Digest::SHA->new('sha256'); | ||
| 188 | $sha->addfile($file); | ||
| 189 | return $sha->hexdigest eq $checksum ? 1 : undef; | ||
| 190 | } | ||
| 191 | |||
| 192 | sub mirror { | ||
| 193 | my $self = shift; | ||
| 194 | |||
| 195 | for my $arch ( @{ $self->arches() } ) { | ||
| 196 | $self->logger->info( | ||
| 197 | sprintf( | ||
| 198 | "mirror: starting repo: %s arch: %s from url: %s to dir: %s", | ||
| 199 | $self->repo, $arch, $self->url, $self->dir | ||
| 200 | ) | ||
| 201 | ); | ||
| 202 | my $packages = $self->get_metadata($arch); | ||
| 203 | $self->get_packages( arch => $arch, packages => $packages ); | ||
| 204 | } | ||
| 205 | |||
| 206 | } | ||
| 207 | |||
| 208 | sub clean { | ||
| 209 | my $self = shift; | ||
| 210 | |||
| 211 | $self->logger->info( | ||
| 212 | sprintf( "clean: starting repo: %s in dir: %s", $self->repo, $self->dir ) ); | ||
| 213 | for my $arch ( @{ $self->arches() } ) { | ||
| 214 | my $files = $self->read_metadata($arch); | ||
| 215 | $self->clean_files( arch => $arch, files => $files ); | ||
| 216 | } | ||
| 217 | |||
| 218 | } | ||
| 219 | |||
| 220 | sub init { | ||
| 221 | my $self = shift; | ||
| 222 | my $arch = shift; | ||
| 223 | $self->logger->info( sprintf 'init: repo: %s dir: %s', | ||
| 224 | $self->repo(), $self->dir() ); | ||
| 225 | if ($arch) { | ||
| 226 | $self->init_arch($arch); | ||
| 227 | } | ||
| 228 | else { | ||
| 229 | for my $a ( @{ $self->arches() } ) { | ||
| 230 | $self->init_arch($a); | ||
| 231 | } | ||
| 232 | } | ||
| 233 | |||
| 234 | } | ||
| 235 | |||
| 236 | sub tag { | ||
| 237 | my $self = shift; | ||
| 238 | my %o = validate( | ||
| 239 | @_, | ||
| 240 | { | ||
| 241 | src_tag => { type => SCALAR }, | ||
| 242 | src_dir => { type => SCALAR }, | ||
| 243 | dest_tag => { type => SCALAR }, | ||
| 244 | dest_dir => { type => SCALAR }, | ||
| 245 | symlink => { type => BOOLEAN, default => 0 }, | ||
| 246 | hard_tag_regex => { type => SCALAR, optional => 1 }, | ||
| 247 | } | ||
| 248 | ); | ||
| 249 | |||
| 250 | $self->logger->debug( | ||
| 251 | sprintf( | ||
| 252 | 'tag: repo: %s tagging: %s -> %s', | ||
| 253 | $self->repo(), $o{'src_dir'}, $o{'dest_dir'} | ||
| 254 | ) | ||
| 255 | ); | ||
| 256 | |||
| 257 | # Make sure intended tag matches what we want if present | ||
| 258 | if ( $o{'hard_tag_regex'} && !$o{'symlink'} ) { | ||
| 259 | $self->logger->log_and_die( | ||
| 260 | level => 'error', | ||
| 261 | message => sprintf( | ||
| 262 | "tag: repo: %s dest_tag: %s does not match hard_tag_regex: %s", | ||
| 263 | $self->repo(), $o{'dest_tag'}, $o{'hard_tag_regex'} | ||
| 264 | ), | ||
| 265 | ) unless $o{'dest_tag'} =~ m#$o{'hard_tag_regex'}#; | ||
| 266 | } | ||
| 267 | |||
| 268 | # When src_dir does not exist do not continue | ||
| 269 | $self->logger->log_and_die( | ||
| 270 | level => 'error', | ||
| 271 | message => sprintf( | ||
| 272 | "tag: repo: %s src_dir: %s does not exist", | ||
| 273 | $self->repo(), $o{'src_dir'} | ||
| 274 | ), | ||
| 275 | ) unless -d $o{'src_dir'}; | ||
| 276 | |||
| 277 | # When dest_dir exists and force is not set do not continue | ||
| 278 | # Handle symbolic link destinations | ||
| 279 | if ( -l $o{'dest_dir'} ) { | ||
| 280 | if ( $self->force() ) { | ||
| 281 | unlink $o{'dest_dir'}; | ||
| 282 | } | ||
| 283 | else { | ||
| 284 | $self->logger->log_and_die( | ||
| 285 | level => 'error', | ||
| 286 | message => sprintf( | ||
| 287 | "tag: repo: %s dest_dir: %s exists and force not enabled.", | ||
| 288 | $self->repo(), $o{'dest_dir'} | ||
| 289 | ), | ||
| 290 | ); | ||
| 291 | } | ||
| 292 | } | ||
| 293 | |||
| 294 | # Handle hard linked destinations | ||
| 295 | elsif ( -d $o{'dest_dir'} ) { | ||
| 296 | if ( $self->force() ) { | ||
| 297 | $self->remove_dir( $o{'dest_dir'} ); | ||
| 298 | } | ||
| 299 | else { | ||
| 300 | $self->logger->log_and_die( | ||
| 301 | level => 'error', | ||
| 302 | message => sprintf( | ||
| 303 | "tag: repo: %s dest_dir: %s exists and force not enabled.", | ||
| 304 | $self->repo(), $o{'dest_dir'} | ||
| 305 | ), | ||
| 306 | ); | ||
| 307 | } | ||
| 308 | } | ||
| 309 | |||
| 310 | # Setup the new destination | ||
| 311 | |||
| 312 | # handle hardlinked destination | ||
| 313 | unless ( $o{'symlink'} ) { | ||
| 314 | $self->make_dir( $o{'dest_dir'} ); | ||
| 315 | $self->logger->debug( | ||
| 316 | sprintf( | ||
| 317 | 'tag: repo: %s hardlink src_dir: %s dest_dir: %s', | ||
| 318 | $self->repo(), $o{'src_dir'}, $o{'dest_dir'} | ||
| 319 | ) | ||
| 320 | ); | ||
| 321 | find( | ||
| 322 | sub { | ||
| 323 | if ( $_ !~ /^[\.]+$/ ) { | ||
| 324 | my $src_path = $File::Find::name; | ||
| 325 | my $path = File::Spec->abs2rel( $File::Find::name, $o{'src_dir'} ); | ||
| 326 | if ( -d $src_path ) { | ||
| 327 | $self->make_dir( File::Spec->catdir( $o{'dest_dir'}, $path ) ); | ||
| 328 | } | ||
| 329 | elsif ( -f $src_path ) { | ||
| 330 | my $dest_path = File::Spec->catfile( $o{'dest_dir'}, $path ); | ||
| 331 | |||
| 332 | if ( link $src_path, $dest_path ) { | ||
| 333 | $self->logger->debug( | ||
| 334 | sprintf( 'tag: repo: %s hardlink: %s', $self->repo(), $path ) ); | ||
| 335 | } | ||
| 336 | else { | ||
| 337 | $self->logger->log_and_croak( | ||
| 338 | level => 'error', | ||
| 339 | message => sprintf( | ||
| 340 | 'tag: repo: %s failed to hardlink: %s to %s', | ||
| 341 | $self->repo(), $src_path, $dest_path, | ||
| 342 | ), | ||
| 343 | ); | ||
| 344 | } | ||
| 345 | } | ||
| 346 | } | ||
| 347 | }, | ||
| 348 | $o{'src_dir'}, | ||
| 349 | ); | ||
| 350 | |||
| 351 | } | ||
| 352 | |||
| 353 | # handle symlink destination | ||
| 354 | else { | ||
| 355 | if ( symlink $o{'src_dir'}, $o{'dest_dir'} ) { | ||
| 356 | $self->logger->debug( | ||
| 357 | sprintf( | ||
| 358 | 'tag: repo: %s symlink src_dir: %s dest_dir: %s', | ||
| 359 | $self->repo(), $o{'src_dir'}, $o{'dest_dir'} | ||
| 360 | ) | ||
| 361 | ); | ||
| 362 | } | ||
| 363 | else { | ||
| 364 | $self->logger->log_and_die( | ||
| 365 | level => 'error', | ||
| 366 | message => sprintf( | ||
| 367 | "tag: repo: %s couldnt link src_dir: %s to dst_dir: %s: $!", | ||
| 368 | $self->repo(), $o{'src_dir'}, $o{'dest_dir'} | ||
| 369 | ), | ||
| 370 | ); | ||
| 371 | } | ||
| 372 | } | ||
| 373 | } | ||
| 374 | |||
| 375 | sub download_binary_file { | ||
| 376 | my $self = shift; | ||
| 377 | my %o = validate( | ||
| 378 | @_, | ||
| 379 | { | ||
| 380 | url => { type => SCALAR }, | ||
| 381 | dest => { type => SCALAR }, | ||
| 382 | retry_limit => { type => SCALAR, default => 3 }, | ||
| 383 | } | ||
| 384 | ); | ||
| 385 | |||
| 386 | $self->logger->debug( | ||
| 387 | sprintf( | ||
| 388 | 'download_binary_file: repo: %s url: %s dest: %s', | ||
| 389 | $self->repo(), $o{url}, $o{dest}, | ||
| 390 | ) | ||
| 391 | ); | ||
| 392 | |||
| 393 | # HTTP::Tiny's mirror function does not seem to validate the file if its locally present in any way | ||
| 394 | unlink $o{dest} if -f $o{dest}; | ||
| 395 | |||
| 396 | my $retry_count = 0; | ||
| 397 | my $retry_limit = $o{retry_limit}; | ||
| 398 | my $success; | ||
| 399 | |||
| 400 | while ( !$success && $retry_count <= $retry_limit ) { | ||
| 401 | my $t0 = [gettimeofday]; | ||
| 402 | my $res = $self->http->mirror( $o{'url'}, $o{'dest'} ); | ||
| 403 | my $elapsed = tv_interval($t0); | ||
| 404 | |||
| 405 | $self->logger->debug( | ||
| 406 | sprintf( | ||
| 407 | 'download_binary_file: repo: %s url: %s took: %s', | ||
| 408 | $self->repo(), $o{url}, $elapsed, | ||
| 409 | ) | ||
| 410 | ); | ||
| 411 | |||
| 412 | if ( $res->{'success'} ) { | ||
| 413 | return 1; | ||
| 414 | } | ||
| 415 | else { | ||
| 416 | $self->logger->debug( | ||
| 417 | sprintf( | ||
| 418 | 'download_binary_file: repo: %s url: %s failed with status: %s reason: %s', | ||
| 419 | $self->repo(), $o{url}, $res->{'status'}, $res->{'reason'}, | ||
| 420 | ) | ||
| 421 | ); | ||
| 422 | $retry_count++; | ||
| 423 | if ( $retry_count <= $retry_limit ) { | ||
| 424 | $self->logger->debug( | ||
| 425 | sprintf( | ||
| 426 | 'download_binary_file: repo: %s url: %s retrying', | ||
| 427 | $self->repo(), $o{url}, | ||
| 428 | ) | ||
| 429 | ) if $retry_count; | ||
| 430 | } | ||
| 431 | else { | ||
| 432 | $self->logger->log_and_croak( | ||
| 433 | level => 'error', | ||
| 434 | message => sprintf( | ||
| 435 | 'download_binary_file: repo: %s url: %s failed and exhausted all retries', | ||
| 436 | $self->repo(), $o{url}, | ||
| 437 | ) | ||
| 438 | ); | ||
| 439 | } | ||
| 440 | } | ||
| 441 | } | ||
| 442 | } | ||
| 443 | |||
| 444 | 1; |