Amazon/S3/CLI.pm untested

filter: Statementstmt 0.0% 0 / 383 Branchbran 0.0% 0 / 126 Conditioncond 0.0% 0 / 112 MC/DCmcdc 0.0% 0 / 70 Subroutinesub 0.0% 0 / 39 Podpod 0.0% 0 / 22 totaltotal 0.0% 0 / 752 CC 64 SCAR 38.2
CC
64
Cov
0%
cmd_get_key
182.0
cmd_add_key
72.0
init
72.0
CRAP 45.5
1 package Amazon::S3::CLI;
2 # modulino proving CLI access to S3 operations
3  
4 use strict;
5 use warnings;
6  
7 use Amazon::S3;
8 use CLI::Simple::Constants qw(:booleans :chars);
9 use CLI::Simple::Utils qw(choose);
10 use Carp;
11 use Cwd qw(abs_path);
12 use Data::Dumper;
13 use English qw(-no_match_vars);
14 use File::Basename qw(basename);
15 use List::Util qw(none);
16 use locale; # for proper sorting
17 use JSON::PP;
18  
19 __PACKAGE__->use_log4perl( level => 'info' );
20  
21 use parent qw(CLI::Simple);
22  
23 caller or exit __PACKAGE__->main;
24  
25 ########################################################################
26 sub get_bucket {
27 ########################################################################
28 my ( $self, $bucket_name ) = @_;
29  
30 $bucket_name //= $self->get_bucket_name;
31  
32 my $bucket = $self->get_s3->bucket(
33 { bucket => $bucket_name,
34 verify_region => $TRUE,
35 }
36 );
37  
38 return $bucket;
39 }
40  
41 ########################################################################
42 sub cmd_empty_bucket {
43 ########################################################################
44 my ($self) = @_;
45  
46 my ($bucket_name) = $self->get_args;
47 $bucket_name //= $self->get_bucket_name;
48  
49 die "ERROR: usage: empty-bucket bucket-name\n"
50 if !$bucket_name;
51  
52 if ( !$self->get_force ) {
53 return $FAILURE
54 if !$self->confirm('This command will remove all objects and versions. Proceed');
55 }
56  
57 my $rsp = $self->get_s3->empty_bucket($bucket_name);
58  
59 if ( $self->get_format eq 'json' ) {
60 print {*STDOUT} JSON::PP->new->pretty->encode($rsp);
61 return $SUCCESS;
62 }
63 elsif ( $self->get_format eq 'table' && $self->get_ascii_table ) {
64 my $t = Text::ASCIITable->new( { headingText => sprintf "Deletion Report\nBucket: %s", $bucket_name } );
65  
66 $t->setCols( 'Versions', 'Delete Markers', 'Multipart Uploads Aborted', 'Total' );
67  
68 $t->addRow( @{$rsp}{qw(versions_deleted delete_markers_deleted multipart_uploads_aborted total)} );
69  
70 print {*STDOUT} $t;
71 return $SUCCESS;
72 }
73  
74 foreach (qw(versions_deleted delete_markers_deleted multipart_uploads_aborted total)) {
75 print {*STDOUT} sprintf "%30s %s\n", $rsp->{$_};
76 }
77  
78 return $SUCCESS;
79 }
80  
81 ########################################################################
82 sub cmd_add_key {
83 ########################################################################
84 my ($self) = @_;
85  
86 my (@args) = $self->get_args;
87  
88 my ( $bucket_name, $filename, $object_name ) = choose {
89 # bucket-name filename object-name
90 return @args
91 if @args == 3;
92  
93 # --bucket-name bucket-name filename
94 return ( @args, $args[1] )
95 if @args == 2 && !$self->get_bucket_name;
96  
97 # --bucket-name bucket-name filename object-name
98 return ( $self->get_bucket_name, @args )
99 if $self->get_bucket_name && @args == 2;
100  
101 # --bucket-name bucket-name --key key-name filename
102 return ( $self->get_bucket_name, $args[0], $self->get_key )
103 if $self->get_bucket_name && $self->get_key && @args == 1;
104  
105 return ( $self->get_bucket_name, $args[0], $args[0] )
106 if $self->get_bucket_name && @args == 1;
107  
108 die "ERROR: usage add-key --bucket-name bucket-name filename object-name\n";
109 };
110  
111 die sprintf "ERROR: file %s does not exist\n", $filename
112 if !-e $filename;
113  
114 if ( $object_name =~ m{\A[/.]}xsm ) {
115 $object_name =~ s{\A[/.]+(.*)$}{$1}xsm;
116 }
117  
118 $self->get_logger->debug(
119 sub {
120 return Dumper(
121 [ bucket_name => $bucket_name,
122 filename => $filename,
123 object_name => $object_name,
124 ]
125 );
126 }
127 );
128  
129 my $content_type = $self->get_content_type // guess_content_type($filename);
130  
131 my $bucket = $self->get_bucket($bucket_name);
132  
133 $self->get_logger->debug( sub { return Dumper( [ $bucket->head_key($object_name) ] ); } );
134  
135 $self->get_logger->debug( sub { return Dumper( [ $bucket, $self->get_s3->last_response ] ); } );
136  
137 $bucket->add_key_filename( $object_name, $filename, { content_type => $content_type } );
138  
139 return $SUCCESS;
140 }
141  
142 ########################################################################
143 sub cmd_list_directory_buckets {
144 ########################################################################
145 my ($self) = @_;
146  
147 my $buckets = $self->get_s3->list_directory_buckets();
148  
149 return $self->_list_buckets($buckets);
150 }
151  
152 ########################################################################
153 sub cmd_create_bucket {
154 ########################################################################
155 my ($self) = @_;
156  
157 my ($bucket_name) = $self->get_args;
158 $bucket_name //= $self->get_bucket_name;
159  
160 if ( $self->get_availability_zone ) {
161 $self->get_s3->use_express_one_zone;
162 }
163  
164 my $response = $self->get_s3->add_bucket(
165 { bucket => $bucket_name,
166 availability_zone => $self->get_availability_zone,
167 region => $self->get_region,
168 }
169 );
170  
171 return $SUCCESS;
172 }
173  
174 ########################################################################
175 sub cmd_copy_key {
176 ########################################################################
177 my ($self) = @_;
178  
179 my ( $bucket_name, $key, $new_key ) = $self->get_args;
180  
181 if ( $self->get_bucket_name ) {
182 if ( $self->get_key ) {
183 $key = $self->get_key;
184 $new_key = $bucket_name;
185 }
186 else {
187 $new_key = $key;
188 $key = $bucket_name;
189 $bucket_name = $self->get_bucket_name;
190 }
191 }
192  
193 die "ERROR: usage: copy-key bucket-name key new-key\n"
194 if !$key || !$bucket_name || !$new_key;
195  
196 my $bucket = $self->get_bucket($bucket_name);
197  
198 $bucket->copy_object(
199 source => $key,
200 key => $new_key,
201 );
202  
203 return $SUCCESS;
204 }
205  
206 ########################################################################
207 sub cmd_get_key {
208 ########################################################################
209 my ($self) = @_;
210  
211 my ( $bucket_name, $key ) = $self->get_args;
212  
213 if ( !$key && $self->get_bucket_name ) {
214 if ( $self->get_key ) {
215 $key = $self->get_key;
216 }
217 else {
218 $key = $bucket_name;
219 $bucket_name = $self->get_bucket_name;
220 }
221 }
222  
223 die "ERROR: usage: get-key bucket-name key\n"
224 if !$key || !$bucket_name;
225  
226 my $bucket = $self->get_s3->bucket($bucket_name);
227  
228 my $modified_since = $self->get_modified_since;
229 my $range = $self->get_range;
230 my $version_id = $self->get_version_id;
231  
232 my %headers
233 = ( $range ? ( Range => 'bytes=' . $range ) : (), $modified_since ? ( 'If-Modified-Since' => $modified_since ) : (), );
234  
235 my %uri_params = ( $version_id ? ( versionId => $version_id ) : (), );
236  
237 my $object = $bucket->get_key(
238 { key => $key,
239 headers => \%headers,
240 uri_params => \%uri_params,
241 }
242 );
243  
244 if ( !$object ) {
245 my $version_id = $self->get_version_id;
246  
247 die sprintf "ERROR: object %s%s not found\n", $key, $version_id ? " version $version_id" : $EMPTY;
248 }
249  
250 my $ofh = choose {
251 my $output = $self->get_output;
252  
253 return *STDOUT
254 if $output && $output eq q{-};
255  
256 if ( !$output ) {
257 if ( $self->get_format ne 'json' ) {
258 $output = basename($key);
259 }
260 else {
261 return *STDOUT;
262 }
263 }
264  
265 open my $fh, '>', $output
266 or die "ERROR: could not open $output for writing\n$OS_ERROR";
267  
268 binmode $fh;
269  
270 return $fh;
271 };
272  
273 my $content = choose {
274 return JSON::PP->new->pretty->encode($object)
275 if $self->get_format eq 'json';
276  
277 return $object->{value};
278 };
279  
280 print {$ofh} $content;
281  
282 close $ofh;
283  
284 return $SUCCESS;
285 }
286  
287 ########################################################################
288 sub cmd_delete_key {
289 ########################################################################
290 my ($self) = @_;
291  
292 my ( $bucket_name, $key ) = $self->get_args;
293  
294 if ( !$key && $self->get_bucket_name ) {
295 if ( $self->get_key ) {
296 $key = $self->get_key;
297 }
298 else {
299 $key = $bucket_name;
300 $bucket_name = $self->get_bucket_name;
301 }
302 }
303  
304 die "ERROR: usage: delete-key bucket-name key\n"
305 if !$key || !$bucket_name;
306  
307 my $bucket = $self->get_bucket($bucket_name);
308  
309 $bucket->delete_key( $key, $self->get_version_id );
310  
311 return $SUCCESS;
312 }
313  
314 ########################################################################
315 sub get_bucket_v2 {
316 ########################################################################
317 my ( $self, $bucket_name ) = @_;
318  
319 $bucket_name //= $self->get_bucket_name;
320  
321 my $bucket = $self->get_s3->bucketv2(
322 { bucket => $bucket_name,
323 verify_region => $TRUE,
324 }
325 );
326  
327 return $bucket;
328 }
329  
330 ########################################################################
331 sub cmd_get_bucket_policy {
332 ########################################################################
333 my ($self) = @_;
334  
335 my ($bucket_name) = $self->get_args;
336 $bucket_name //= $self->get_bucket_name;
337  
338 die "ERROR: usage: get-bucket-policy bucket-name\n"
339 if !$bucket_name;
340  
341 my $policy = $self->get_bucket_v2($bucket_name)->GetBucketPolicy();
342  
343 return $FAILURE
344 if !$policy;
345  
346 print {*STDOUT} JSON::PP->new->pretty->canonical->encode($policy);
347  
348 return $SUCCESS;
349 }
350  
351 ########################################################################
352 sub cmd_get_bucket_policy_status {
353 ########################################################################
354 my ($self) = @_;
355  
356 my ($bucket_name) = $self->get_args;
357 $bucket_name //= $self->get_bucket_name;
358  
359 die "ERROR: usage: get-bucket-policy-status bucket-name\n"
360 if !$bucket_name;
361  
362 my $status = $self->get_bucket_v2($bucket_name)->GetBucketPolicyStatus();
363  
364 return $FAILURE
365 if !$status;
366  
367 print {*STDOUT} JSON::PP->new->pretty->canonical->encode($status);
368  
369 return $SUCCESS;
370 }
371  
372 ########################################################################
373 sub cmd_get_bucket_acl {
374 ########################################################################
375 my ($self) = @_;
376  
377 my ($bucket_name) = $self->get_args;
378 $bucket_name //= $self->get_bucket_name;
379  
380 die "ERROR: usage: get-bucket-acl bucket-name\n"
381 if !$bucket_name;
382  
383 my $acl = $self->get_bucket_v2($bucket_name)->GetBucketAcl();
384  
385 return $FAILURE
386 if !$acl;
387  
388 print {*STDOUT} JSON::PP->new->pretty->canonical->encode($acl);
389  
390 return $SUCCESS;
391 }
392  
393 ########################################################################
394 sub cmd_remove_bucket {
395 ########################################################################
396 my ($self) = @_;
397  
398 my ($bucket_name) = $self->get_args;
399 $bucket_name //= $self->get_bucket_name;
400  
401 die "ERROR: usage: remove-bucket bucket-name\n"
402 if !$bucket_name;
403  
404 $self->get_s3->delete_bucket( { bucket => $bucket_name } );
405  
406 return $SUCCESS;
407 }
408  
409 ########################################################################
410 sub cmd_list_object_versions {
411 ########################################################################
412 my ($self) = @_;
413  
414 my ( $bucket_name, $prefix ) = $self->get_args;
415 $bucket_name //= $self->get_bucket_name;
416 $prefix //= $self->get_prefix;
417  
418 if ( defined $prefix ) {
419 $prefix =~ s/^\///xsm;
420 }
421  
422 my $response = $self->get_s3->list_object_versions( { bucket => $bucket_name, prefix => $prefix } );
423  
424 if ( $self->get_format eq 'json' ) {
425 print {*STDOUT} JSON::PP->new->pretty->encode($response);
426 return $SUCCESS;
427 }
428 elsif ( $self->get_ascii_table && $self->get_format eq 'table' ) {
429 my $t = Text::ASCIITable->new( { headingText => sprintf "Bucket: %s", $bucket_name } );
430  
431 $t->setCols( 'Key', 'Size', 'Last Modified', 'Latest', 'Version ID' );
432  
433 foreach my $k ( @{ $response->{Version} } ) {
434 $t->addRow( @{$k}{qw(Key Size LastModified IsLatest VersionId)} );
435 }
436  
437 print {*STDOUT} $t;
438 return $SUCCESS;
439 }
440  
441 print {*STDOUT} "Key,Size,LastModified,IsLatest,VersionId\n";
442  
443 foreach my $k ( @{ $response->{Version} } ) {
444 print {*STDOUT} sprintf qq{"%s","%s","%s","%s","%s"\n}, @{$k}{qw(Key Size LastModified IsLatest VersionId)};
445 }
446  
447 return $SUCCESS;
448 }
449  
450 ########################################################################
451 sub cmd_list_bucket_keys {
452 ########################################################################
453 my ($self) = @_;
454  
455 my ( $bucket_name, $prefix ) = $self->get_args;
456 $bucket_name //= $self->get_bucket_name;
457 $prefix //= $self->get_prefix;
458  
459 if ( defined $prefix ) {
460 $prefix =~ s/^\///xsm;
461 }
462  
463 die "ERROR: usage: list-bucket-keys bucket-name [prefix]\n"
464 if !$bucket_name;
465  
466 my $response
467 = eval { return $self->get_s3->list_bucket_all_v2( { bucket => $bucket_name, $prefix ? ( prefix => $prefix ) : () } ); };
468  
469 return $self->_list_keys( $bucket_name, $prefix, $response );
470 }
471  
472 ########################################################################
473 sub _list_keys {
474 ########################################################################
475 my ( $self, $bucket_name, $prefix, $response ) = @_;
476  
477 if ( !$response ) {
478 return $SUCCESS
479 if $self->get_format ne 'json';
480  
481 print {*STDOUT} "{}\n";
482 return $SUCCESS;
483 }
484  
485 if ( $self->get_format eq 'json' ) {
486 my $result = { map { $_->{key} => $_ } @{ $response->{keys} // [] } };
487 print JSON::PP->new->pretty->encode($result);
488 return $SUCCESS;
489 }
490  
491 my $data = [ reverse sort { $a->{key} cmp $b->{key} } @{ $response->{keys} } ];
492  
493 my $cols = [qw(key size last_modified etag)];
494  
495 my $heading = $response->{bucket};
496  
497 if ( $self->get_prefix ) {
498 $heading = sprintf '%s/%s', $heading, $self->get_prefix;
499 }
500  
501 if ( $self->get_ascii_table && $self->get_format eq 'table' ) {
502 my $t = Text::ASCIITable->new( { headingText => sprintf "Bucket: %s\nPrefix: %s\n", $bucket_name, $prefix // q{/} } );
503  
504 $t->setCols( 'Key', 'Size', 'Last Modified', 'ETag' );
505  
506 foreach my $k ( @{$data} ) {
507 $t->addRow( @{$k}{qw(key size last_modified etag)} );
508 }
509  
510 print {*STDOUT} $t;
511  
512 return $SUCCESS;
513 }
514  
515 print {*STDOUT} "Key,Size,LastModified,ETag\n";
516  
517 foreach my $k ( @{$data} ) {
518 print {*STDOUT} sprintf qq{"%s","%s","%s","%s"\n}, @{$k}{qw(key size last_modified etag)};
519 }
520  
521 return $SUCCESS;
522 }
523  
524 ########################################################################
525 sub cmd_list_buckets {
526 ########################################################################
527 my ($self) = @_;
528  
529 my $buckets = $self->get_s3->buckets();
530  
531 if ( !$buckets || !@{ $buckets->{buckets} // [] } ) {
532 print {*STDOUT} "no buckets\n";
533  
534 return $SUCCESS;
535 }
536  
537 return $self->_list_buckets($buckets);
538 }
539  
540 ########################################################################
541 sub _list_buckets {
542 ########################################################################
543 my ( $self, $buckets ) = @_;
544  
545 if ( $self->get_format eq 'json' ) {
546 my $data = {
547 owner_id => $buckets->{owner_id},
548 buckets =>
549 { map { $_->bucket => { region => $_->region, creation_date => $_->creation_date, } } @{ $buckets->{buckets} } },
550 };
551  
552 print {*STDOUT} JSON::PP->new->pretty->allow_blessed->convert_blessed->encode($data);
553  
554 return $SUCCESS;
555 }
556  
557 my $data = [ sort { $a->bucket cmp $b->bucket } @{ $buckets->{buckets} } ];
558  
559 if ( $self->get_ascii_table && $self->get_format eq 'table' ) {
560 my $t = Text::ASCIITable->new( { headingText => 'Bucket Listing', } );
561  
562 $t->setCols( qw(Bucket Region), 'Creation Date' );
563  
564 foreach my $b ( @{$data} ) {
565 $t->addRow( $b->bucket, $b->region, $b->creation_date, );
566 }
567  
568 print {*STDOUT} $t;
569  
570 return $SUCCESS;
571 }
572  
573 print {*STDOUT} "Bucket,Region,CreationDate\n";
574  
575 foreach my $b ( @{$data} ) {
576 print {*STDOUT} sprintf qq{"%s","%s","%s"\n}, $b->bucket, $b->region, $b->creation_date;
577 }
578  
579 return $SUCCESS;
580 }
581  
582 ########################################################################
583 sub init {
584 ########################################################################
585 my ($self) = @_;
586  
587 my %endpoint = choose {
588 return ( host => $self->get_host )
589 if defined $self->get_host;
590  
591 return ( endpoint_url => $self->get_endpoint_url )
592 if $self->get_endpoint_url;
593  
594 return ();
595 };
596  
597 $self->set_ascii_table(
598 eval {
599 require Text::ASCIITable;
600 1;
601 }
602 );
603  
604 die "ERROR: --format must be one of json, text or table\n"
605 if none { $self->get_format eq $_ } qw(json text table);
606  
607 my %credentials;
608  
609 my $has_amazon_credentials = eval {
610 require Amazon::Credentials;
611 1;
612 };
613  
614 if ($has_amazon_credentials) {
615 $credentials{credentials} = Amazon::Credentials->new(
616 { $self->get_profile
617 ? ( profile => $self->get_profile )
618 : (),
619 }
620 );
621 }
622 else {
623 die "ERROR: --profile requires Amazon::Credentials\n"
624 if $self->get_profile;
625  
626 $credentials{aws_access_key_id} = $ENV{AWS_ACCESS_KEY_ID};
627 $credentials{aws_secret_access_key} = $ENV{AWS_SECRET_ACCESS_KEY};
628 $credentials{token} = $ENV{AWS_SESSION_TOKEN};
629 }
630  
631 my $s3 = Amazon::S3->new(
632 { %credentials,
633 debug => $ENV{DEBUG},
634 raise_error => $TRUE,
635 region => $self->get_region,
636 logger => $self->get_logger,
637 dns_bucket_names => $self->get_dns_bucket_names // $FALSE,
638 %endpoint,
639 defined $self->get_secure ? ( secure => $self->get_secure ) : (),
640 }
641 );
642  
643 $self->set_s3($s3);
644  
645 return;
646 }
647  
648 ########################################################################
649 sub confirm {
650 ########################################################################
651 my ( $self, $prompt ) = @_;
652  
653 print "$prompt [y/N] ";
654  
655 my $answer = <STDIN>;
656 return $FALSE
657 if !defined $answer;
658  
659 chomp $answer;
660  
661 return $answer =~ /\Ay(?:es)?\z/i ? $TRUE : $FALSE;
662 }
663  
664 ########################################################################
665 sub guess_content_type {
666 ########################################################################
667 my ($filename) = @_;
668  
669 my $has_mmagic = eval { require File::MimeInfo::Magic; 1; };
670  
671 return 'application/octet-stream'
672 if !$has_mmagic || !$filename || !-f $filename;
673  
674 my $content_type = eval { return File::MimeInfo::Magic::mimetype($filename); };
675  
676 return $content_type
677 if !$EVAL_ERROR && $content_type;
678  
679 return 'application/octet-stream';
680 }
681  
682 ########################################################################
683 sub main {
684 ########################################################################
685  
686 my %default_options = (
687 output => $EMPTY,
688 region => 'us-east-1',
689 format => 'text',
690 );
691  
692 my @option_specs = qw(
693 availability-zone=s
694 bucket-name|b=s
695 content-type=s
696 debug
697 dns-bucket-names
698 endpoint-url|u=s
699 force|f
700 format|F=s
701 help|h
702 host|H=s
703 key|k=s
704 modified-since|m=s
705 output|o=s
706 prefix=s
707 profile|p=s
708 range|R=s
709 region|r=s
710 secure|s
711 version-id=s
712 );
713  
714 my %commands = (
715 'add-key' => \&cmd_add_key,
716 'copy-key' => \&cmd_copy_key,
717 'create-bucket' => \&cmd_create_bucket,
718 'delete-key' => \&cmd_delete_key,
719 'empty-bucket' => \&cmd_empty_bucket,
720 'get-key' => \&cmd_get_key,
721 'get-bucket-policy' => \&cmd_get_bucket_policy,
722 'get-bucket-acl' => \&cmd_get_bucket_acl,
723 'get-bucket-policy-status' => \&cmd_get_bucket_policy_status,
724 'list-bucket-keys' => \&cmd_list_bucket_keys,
725 'list-directory-buckets' => \&cmd_list_directory_buckets,
726 'list-keys' => 'list-bucket-keys',
727 'list-object-versions' => \&cmd_list_object_versions,
728 'remove-bucket' => \&cmd_remove_bucket,
729 'list-buckets' => \&cmd_list_buckets,
730 );
731  
732 return __PACKAGE__->new(
733 commands => \%commands,
734 option_specs => \@option_specs,
735 default_options => \%default_options,
736 abbreviations => $TRUE,
737 extra_options => [qw(s3 secure ascii_table)],
738 )->run;
739 }
740  
741 1;
742  
743 __END__