Nrepo/Plugin/Yum.pm untested

filter: Statementstmt 0.0% 0 / 212 Branchbran 0.0% 0 / 54 Conditioncond 0.0% 0 / 21 MC/DCmcdc 0.0% 0 / 14 Subroutinesub 0.0% 0 / 22 Podpod 0.0% 0 / 10 totaltotal 0.0% 0 / 333 CC 28 SCAR 30.0
CC
28
Cov
0%
get_metadata
42.0
parse_repomd
42.0
init_arch
30.0
CRAP 20.0
1 #!/bin/false
2 package App::Nrepo::Plugin::Yum;
3  
4 use Moo;
5 use strictures 2;
6  
7 use Carp;
8 use File::Copy qw(copy);
9 use File::Find qw(find);
10 use File::Basename qw(basename dirname);
11 use IO::Zlib;
12 use Params::Validate qw(:all);
13 use Data::Dumper qw(Dumper);
14 use Time::HiRes qw(gettimeofday tv_interval);
15 use XML::Twig;
16 use XML::LibXML;
17  
18 our $VERSION = '0.1'; # VERSION
19  
20 with 'App::Nrepo::Plugin::Base';
21  
22 has packages_dir => ( is => 'ro', default => 'Packages' );
23 has repodata_dir => ( is => 'ro', default => 'repodata' );
24  
25 sub get_metadata {
26 my $self = shift;
27 my $arch = shift;
28  
29 my $base_dir = File::Spec->catdir( $self->dir(), $arch );
30 my $packages;
31  
32 my @metadata_files =
33 ( { type => 'repomd', location => 'repodata/repomd.xml' } );
34 for my $m (@metadata_files) {
35 my $type = $m->{'type'};
36 my $location = $m->{'location'};
37 my $m_url = join( '/', ( $self->url, $location ) );
38 $m_url =~ s/%ARCH%/$arch/;
39 my $dest_file = File::Spec->catfile( $base_dir, $location );
40 my $dest_dir = dirname($dest_file);
41  
42 # Make sure dir exists
43 $self->make_dir($dest_dir);
44  
45 # Check if we have the local file
46 my $download;
47 if ( $type eq 'repomd' ) {
48 $download++;
49 }
50 elsif (
51 !$self->validate_file(
52 filename => $dest_file,
53 check => $m->{'validate'}->{'type'},
54 value => $m->{'validate'}->{'value'},
55 )
56 )
57 {
58 $download++;
59 }
60  
61 # Grab the file
62 if ($download) {
63 $self->download_binary_file( url => $m_url, dest => $dest_file );
64 }
65 else {
66 $self->logger->debug(
67 sprintf(
68 'get_metadata: repo: %s arch: %s file: %s skipping as its deemed up to date',
69 $self->repo(), $arch, $location
70 )
71 );
72 }
73  
74 # Parse the xml and retrieve the primary file location
75 if ( $type eq 'repomd' ) {
76 my $data = $self->parse_repomd($dest_file);
77 push @metadata_files, @{$data};
78 }
79  
80 # Parse the primary metadata file
81 if ( $type eq 'primary' ) {
82 $packages = $self->parse_primary($dest_file);
83 }
84 }
85 return $packages;
86 }
87  
88 sub read_metadata {
89 my $self = shift;
90 my $arch = shift;
91  
92 my $base_dir = File::Spec->catdir( $self->dir(), $arch );
93 my $files = {};
94  
95 my @metadata_files =
96 ( { type => 'repomd', location => 'repodata/repomd.xml' } );
97 for my $m (@metadata_files) {
98 my $type = $m->{'type'};
99 my $location = $m->{'location'};
100 my $dest_file = File::Spec->catfile( $base_dir, $location );
101 my $dest_dir = dirname($dest_file);
102  
103 $files->{$location}++;
104  
105 if ( -f $dest_file ) {
106  
107 # Parse the xml and retrieve the primary file location
108 if ( $type eq 'repomd' ) {
109 my $data = $self->parse_repomd($dest_file);
110 push @metadata_files, @{$data};
111 }
112  
113 # Parse the primary metadata file
114 if ( $type eq 'primary' ) {
115 my $contents = $self->get_gzip_contents($dest_file);
116 for my $file ( @{ $self->parse_primary($dest_file) } ) {
117 $files->{ $file->{'location'} }++;
118 }
119 }
120 }
121 }
122 return $files;
123 }
124  
125 sub parse_repomd {
126 my $self = shift;
127 my $file = shift;
128  
129 # XXX TODO rework this with XML::LibXML as its far faster
130 my $twig = XML::Twig->new( TwigRoots => { data => 1 } );
131 $twig->parsefile($file);
132  
133 my $root = $twig->root;
134 my @e = $root->children();
135 my @files;
136 for my $e (@e) {
137 my $data = {};
138 $data->{'type'} = $e->att('type');
139 for my $c ( $e->children() ) {
140 if ( $c->name eq 'location' ) {
141 $data->{'location'} = $c->att('href');
142 }
143 elsif ( $c->name eq 'checksum' ) {
144 $data->{'checksum'}->{'type'} = $c->att('type');
145 $data->{'checksum'}->{'value'} = $c->text;
146 }
147 elsif ( $c->name eq 'size' ) {
148 $data->{'size'}->{'type'} = 'size';
149 $data->{'size'}->{'value'} = $c->text;
150 }
151 }
152  
153 # For some reason i have found a few repomd.xml files that do NOT
154 # have a size attribute ...specifically updateinfo type
155 # so as a work around we will try size if checksums is not enabled
156 # however for that file we'll revert to checksums if size is not available
157 if ( !$self->checksums() && $data->{'size'} ) {
158 $data->{'validate'} = $data->{'size'};
159 }
160 else {
161 $data->{'validate'} = $data->{'checksum'};
162 }
163 $self->logger->log_and_croak(
164 level => 'error',
165 message => "repomd xml not valid: $file"
166 ) unless $data->{'location'};
167 push @files, $data;
168 }
169  
170 return \@files;
171 }
172  
173 sub parse_primary {
174 my $self = shift;
175 my $dest_file = shift;
176 my $io_fh = IO::Zlib->new( $dest_file, 'rb' );
177 my $xml = XML::LibXML->load_xml( IO => $io_fh );
178 my $t0 = [gettimeofday];
179 my $packages = [];
180 for my $p ( $xml->getElementsByTagName('package') ) {
181 my ($n) = $p->getChildrenByTagName('name');
182 my ($l) = $p->getChildrenByTagName('location');
183 my ($s) = $p->getChildrenByTagName('size');
184 my ($c) = $p->getChildrenByTagName('checksum');
185 my $data = {
186 name => $n->textContent,
187 location => $l->getAttribute('href'),
188 size => {
189 type => 'size',
190 value => $s->getAttribute('package'),
191 },
192 checksum => {
193 type => $c->getAttribute('type'),
194 value => $c->textContent,
195 },
196 };
197 if ( !$self->checksums() && $data->{'size'} ) {
198 $data->{'validate'} = $data->{'size'};
199 }
200 else {
201 $data->{'validate'} = $data->{'checksum'};
202 }
203 push @{$packages}, $data;
204 }
205 my $elapsed = tv_interval($t0);
206 $self->logger->debug(
207 sprintf( 'parse_primary: file: %s took: %s seconds', $dest_file, $elapsed )
208 );
209 return $packages;
210 }
211  
212 sub get_packages {
213 my $self = shift;
214 my %o = validate(
215 @_,
216 {
217 arch => { type => SCALAR },
218 packages => { type => ARRAYREF },
219 }
220 );
221  
222 my $arch = $o{'arch'};
223 my $base_dir = File::Spec->catdir( $self->dir(), $arch );
224  
225 for my $package ( @{ $o{'packages'} } ) {
226 my $name = $package->{'name'};
227 my $location = $package->{'location'};
228  
229 my $p_url = join( '/', ( $self->url, $location ) );
230 $p_url =~ s/%ARCH%/$arch/;
231 my $dest_file = File::Spec->catfile( $base_dir, $location );
232 my $dest_dir = dirname($dest_file);
233  
234 # Make sure dir exists
235 $self->make_dir($dest_dir);
236  
237 # Check if we have the local file
238 if (
239 !$self->validate_file(
240 filename => $dest_file,
241 check => $package->{'validate'}->{'type'},
242 value => $package->{'validate'}->{'value'},
243 )
244 )
245 {
246 $self->logger->notice(
247 sprintf(
248 'get_packages: repo: %s arch: %s package: %s',
249 $self->repo(), $arch, $location
250 )
251 );
252 $self->download_binary_file( url => $p_url, dest => $dest_file );
253 }
254 else {
255 $self->logger->debug(
256 sprintf(
257 'get_packages: repo: %s arch: %s package: %s skipping as its deemed up to date',
258 $self->repo(), $arch, $location
259 )
260 );
261 }
262 }
263 }
264  
265 sub clean_files {
266 my $self = shift;
267 my %o = validate(
268 @_,
269 {
270 arch => { type => SCALAR },
271 files => { type => HASHREF },
272 }
273 );
274  
275 my $arch = $o{'arch'};
276 my $base_dir = File::Spec->catdir( $self->dir(), $arch );
277  
278 find(
279 sub {
280 if ( $_ !~ /^[\.]+$/ ) {
281 my $file = $_;
282 if ( -f $file ) {
283 my $rel = File::Spec->abs2rel( $File::Find::name, $base_dir );
284 unless ( $o{'files'}->{$rel} ) {
285 $self->logger->info(
286 "clean_files: removing non referenced file: ${File::Find::name}");
287 unlink $file
288 or $self->logger->log_and_croak(
289 level => 'error',
290 message => "Failed to remove file: ${file}: $!"
291 );
292 }
293 }
294 }
295 },
296 $base_dir,
297 );
298 }
299  
300 sub add_file {
301 my $self = shift;
302 my $arch = shift;
303 my $files = shift;
304  
305 unless ( $self->validate_arch($arch) ) {
306 $self->logger->log_and_croak(
307 level => 'error',
308 message => sprintf 'add_file: arch: %s is not in config for repo: %s',
309 $arch, $self->repo()
310 );
311 }
312  
313 my $package_dir =
314 File::Spec->catdir( $self->dir(), $arch, $self->packages_dir() );
315 $self->make_dir($package_dir) unless -d $package_dir;
316  
317 for my $file ( @${files} ) {
318 my $filename = basename($file);
319 my $dest_file =
320 File::Spec->catfile( $self->dir(), $arch, $package_dir, $filename );
321 $self->logger->debug(
322 sprintf 'add_file: repo: %s arch: %s file: %s dest_file: %s',
323 $self->repo(), $arch, $file, $dest_file );
324  
325 if ( -f $dest_file && !$self->force() ) {
326 $self->logger->log_and_croak(
327 level => 'error',
328 message => sprintf
329 'add_file: repo: %s dest_file exists and force not enabled: %s',
330 $self->repo(), $dest_file
331 );
332 }
333  
334 copy( $file, $dest_file ) || $self->logger->log_and_croak(
335 level => 'error',
336 message => sprintf
337 'add_file: repo: %s failed to copy file to destination: %s',
338 $self->repo(), $!
339 );
340  
341 }
342  
343 $self->init_arch($arch);
344  
345 }
346  
347 sub del_file {
348 my $self = shift;
349 my $arch = shift;
350 my $files = shift;
351  
352 unless ( $self->validate_arch($arch) ) {
353 $self->logger->log_and_croak(
354 level => 'error',
355 message => sprintf 'del_file: arch: %s is not in config for repo: %s',
356 $arch, $self->repo()
357 );
358 }
359  
360 for my $file ( @${files} ) {
361 my $filename = basename($file);
362 my $dest_file =
363 File::Spec->catfile( $self->dir(), $arch, $self->packages_dir(),
364 $filename );
365 $self->logger->debug( sprintf 'del_file: repo: %s arch: %s dest_file: %s',
366 $self->repo(), $arch, $dest_file );
367  
368 unless ( -f $dest_file ) {
369 $self->logger->log_and_croak(
370 level => 'error',
371 message => sprintf 'del_file: repo: %s dest_file: %s does not exist',
372 $self->repo(), $dest_file
373 );
374 }
375  
376 unlink $dest_file || $self->logger->log_and_croak(
377 level => 'error',
378 message => sprintf
379 'del_file: repo: %s failed to del file: %s from destination: %s',
380 $self->repo(), $dest_file, $!
381 );
382  
383 }
384  
385 $self->init_arch($arch);
386 }
387  
388 sub init_arch {
389 my $self = shift;
390 my $arch = shift;
391  
392 my $dir = File::Spec->catdir( $self->dir, $arch );
393  
394 $self->logger->debug( sprintf 'init_arch: repo: %s arch: %s dir: %s',
395 $self->repo(), $arch, $dir );
396  
397 $self->make_dir($dir);
398 $self->make_dir( File::Spec->catdir( $dir, $self->packages_dir() ) );
399  
400 #XXX add gpg
401  
402 #TODO perhaps replace createrepo with pure perl version at some stage
403 my $createrepo_bin = $self->find_command_path('createrepo');
404  
405 unless ( $createrepo_bin and -x $createrepo_bin ) {
406 $self->logger->log_and_croak(
407 level => 'error',
408 message => sprintf(
409 'init_arch: repo: %s arch: %s unable to find createrepo program in path',
410 $self->repo(), $arch
411 ),
412 );
413 }
414  
415 my @cmd = (
416 $createrepo_bin, '--basedir', $dir, '--outputdir', $dir,
417 $self->packages_dir()
418 );
419  
420 # --update will reuse the existing metadata if the file is already defined and size/mtime matches
421 # dont do this if we're forcing or the repomd.xml doesnt exist
422 unless ( $self->force() ) {
423 if ( -f File::Spec->catfile( $dir, $self->repodata_dir(), 'repomd.xml' ) ) {
424 splice @cmd, 1, 0, '--update';
425 }
426 }
427  
428 $self->logger->debug( sprintf 'init_arch: running command: %s',
429 join( ' ', @cmd ) );
430 unless ( system(@cmd) == 0 ) {
431 $self->logger->log_and_croak(
432 level => 'error',
433 message => sprintf(
434 'init_arch: repo: %s failed to run command: %s with exit code: %s',
435 $self->repo(), join( ' ', @cmd ), $?,
436 )
437 );
438 }
439  
440 }
441  
442 sub type {
443  
444 #my $self = shift;
445 return 'Yum';
446 }
447  
448 1;