Indexer/Storage/S3.pm untested

filter: Statementstmt 0.0% 0 / 89 Branchbran 0.0% 0 / 18 Conditioncond 0.0% 0 / 27 MC/DCmcdc 0.0% 0 / 13 Subroutinesub 0.0% 0 / 20 Podpod 0.0% 0 / 9 totaltotal 0.0% 0 / 176 CC 10 SCAR 27.6
CC
10
Cov
0%
list_distributions
56.0
new
6.0
fetch_object
6.0
CRAP 15.9
1 package DarkPAN::Indexer::Storage::S3;
2  
3 # S3 storage engine for DarkPAN::Indexer.
4 #
5 # Consumes the DarkPAN::Indexer::Storage role (the contract). Wraps the
6 # four S3 primitives the old roles called directly:
7 # list_objects_v2 -> list_distributions
8 # get_object -> fetch_object
9 # head_object -> has_object
10 # put_object -> save_object
11 #
12 use strict;
13 use warnings;
14  
15 use Amazon::Credentials;
16 use Amazon::S3::Lite;
17 use Amazon::S3::Lite::Lock;
18 use Data::Dumper;
19 use Carp qw(croak);
20 use English qw(-no_match_vars);
21  
22 use Role::Tiny::With;
23 with 'DarkPAN::Indexer::Storage'; # requires: list_distributions fetch_object
24 # save_object has_object lock base_url
25  
26 caller or exit __PACKAGE__->main();
27  
28 ########################################################################
29 sub new {
30 ########################################################################
31 my ( $class, $config ) = @_;
32  
33 # prefer storage keys over AWS
34 my %storage = ( %{ $config->{AWS} // {} }, %{ $config->{storage} // {} } );
35 $storage{region} //= $ENV{AWS_REGION} // 'us-east-1';
36  
37 $config->{storage} = \%storage;
38  
39 croak "S3 storage: bucket is required\n"
40 if !defined $storage{bucket} || $storage{bucket} eq q{};
41  
42 my $self = bless {
43 config => $config,
44 bucket => $storage{bucket},
45 prefix => $storage{prefix},
46 region => $storage{region},
47 profile => $storage{profile},
48 base_url => $config->{url}, # public (CloudFront) base
49 }, $class;
50  
51 return $self;
52 }
53  
54 ########################################################################
55 sub _s3 {
56 ########################################################################
57 my ($self) = @_;
58  
59 return $self->{s3} //= Amazon::S3::Lite->new(
60 { region => $self->{region},
61 credentials => Amazon::Credentials->new( profile => $self->{profile} ),
62 }
63 );
64 }
65  
66 ########################################################################
67 sub base_url { return $_[0]->{base_url} }
68 ########################################################################
69  
70 ########################################################################
71 # list_distributions() -> list of keys (relative to the bucket) that
72 # look like distribution tarballs under the configured prefix. Paginates
73 # via continuation token. The .tar.gz filter lives here because "what
74 # counts as a distribution" is the one bit of domain knowledge this
75 # seam legitimately owns.
76 ########################################################################
77 sub list_distributions {
78 ########################################################################
79 my ($self) = @_;
80  
81 my $s3 = $self->_s3;
82 my @keys;
83 my $token;
84  
85 while (1) {
86 my %options = ( defined $self->{prefix} ? ( prefix => $self->{prefix} ) : () );
87  
88 if ( defined $token ) {
89 $options{continuation_token} = $token;
90 }
91  
92 my $result = $s3->list_objects_v2( $self->{bucket}, %options );
93  
94 for my $object ( @{ $result->{objects} || [] } ) {
95 my $key = $object->{key};
96 next if !defined $key;
97 next if $key !~ m{[.]tar[.]gz\z}xsm;
98 push @keys, $key;
99 }
100  
101 last
102 if !$result->{is_truncated};
103  
104 $token = $result->{next_continuation_token};
105 last if !$token;
106 }
107  
108 return @keys;
109 }
110  
111 ########################################################################
112 # fetch_object($key) -> raw bytes, or undef if genuinely absent.
113 # A real I/O failure throws (fail-fast contract).
114 ########################################################################
115 sub fetch_object {
116 my ( $self, $key ) = @_;
117  
118 my $object = $self->_s3->get_object( $self->{bucket}, $key );
119  
120 return
121 if !defined $object; # not found -> undef
122  
123 return $object->{content};
124 }
125  
126 ########################################################################
127 # has_object($key) -> boolean. Wraps head_object.
128 ########################################################################
129 sub has_object {
130 my ( $self, $key ) = @_;
131  
132 return $self->_s3->head_object( $self->{bucket}, $key ) ? 1 : 0;
133 }
134  
135 ########################################################################
136 # save_object($key, $bytes, %opts) -> writes. %opts may carry
137 # content_type; caller (Utils) already gzips, so this moves bytes only.
138 ########################################################################
139 sub save_object {
140 my ( $self, $key, $content, %opts ) = @_;
141  
142 my $content_type = $opts{content_type} // 'application/octet-stream';
143  
144 return $self->_s3->put_object( $self->{bucket}, $key, $content, content_type => $content_type, );
145 }
146  
147 ########################################################################
148 # lock($key, %opts) -> a GUARD object on success, false on failure.
149 #
150 # CONTRACT (both engines must match): lock() acquires and returns a
151 # guard. The guard is true on success and releases the lock in its
152 # DESTROY when it goes out of scope — including on die. The caller MUST
153 # bind it to a lexical for the whole critical section:
154 #
155 # my $guard = $storage->lock($key, ttl => 300, wait => 60)
156 # or croak 'could not acquire packages index lock';
157 # ... fetch -> mutate -> save ...
158 # # $guard drops here -> release, even if the block died
159 #
160 # Discarding the return (void context, or letting it fall off the end of
161 # a block) releases the lock immediately and defeats the mutex. Hold it.
162 #
163 # S3 uses the conditional-PUT lock (Amazon::S3::Lite::Lock); its guard
164 # releases with If-Match on its own etag so it never clobbers a lock
165 # that was stolen after a TTL lapse. The Filesystem engine returns an
166 # flock-based guard instead — same contract, different mechanism.
167  
168 ########################################################################
169 sub lock {
170 ########################################################################
171 my ( $self, $key, %opts ) = @_;
172  
173 require Amazon::S3::Lite::Lock;
174  
175 # acquire() returns the guard on success, false if the lock is held
176 # and could not be (stolen as stale / waited out).
177 return Amazon::S3::Lite::Lock->new(
178 s3 => $self->_s3,
179 bucket => $self->{bucket},
180 key => $key,
181 ttl => $opts{ttl} // 300,
182 wait => $opts{wait} // 0,
183 )->acquire;
184 }
185  
186 ########################################################################
187 sub main {
188 ########################################################################
189 my $config_file = shift @ARGV;
190  
191 require JSON;
192  
193 open my $fh, '<', $config_file
194 or die "ERROR: could not open $config_file for reading\n$OS_ERROR";
195  
196 my $config;
197  
198 {
199 local $RS = undef;
200 $config = JSON->new->decode(<$fh>);
201 close $fh;
202 }
203  
204 my $s3_storage = __PACKAGE__->new( $config->{bedrock} );
205  
206 print Dumper( [ distributions => $s3_storage->list_distributions ] );
207  
208 return 0;
209 }
210  
211 1;