| 1 | package DarkPAN::Indexer::Storage::S3; | ||
| 2 | |||
| 3 | # S3 storage engine for DarkPAN::Indexer. | ||
| 4 | # | ||
| 5 | # Consumes the DarkPAN::Indexer::Storage role (the contract). Wraps the | ||
| 6 | # four S3 primitives the old roles called directly: | ||
| 7 | # list_objects_v2 -> list_distributions | ||
| 8 | # get_object -> fetch_object | ||
| 9 | # head_object -> has_object | ||
| 10 | # put_object -> save_object | ||
| 11 | # | ||
| 12 | use strict; | ||
| 13 | use warnings; | ||
| 14 | |||
| 15 | use Amazon::Credentials; | ||
| 16 | use Amazon::S3::Lite; | ||
| 17 | use Amazon::S3::Lite::Lock; | ||
| 18 | use Data::Dumper; | ||
| 19 | use Carp qw(croak); | ||
| 20 | use English qw(-no_match_vars); | ||
| 21 | |||
| 22 | use Role::Tiny::With; | ||
| 23 | with 'DarkPAN::Indexer::Storage'; # requires: list_distributions fetch_object | ||
| 24 | # save_object has_object lock base_url | ||
| 25 | |||
| 26 | caller or exit __PACKAGE__->main(); | ||
| 27 | |||
| 28 | ######################################################################## | ||
| 29 | sub new { | ||
| 30 | ######################################################################## | ||
| 31 | my ( $class, $config ) = @_; | ||
| 32 | |||
| 33 | # prefer storage keys over AWS | ||
| 34 | my %storage = ( %{ $config->{AWS} // {} }, %{ $config->{storage} // {} } ); | ||
| 35 | $storage{region} //= $ENV{AWS_REGION} // 'us-east-1'; | ||
| 36 | |||
| 37 | $config->{storage} = \%storage; | ||
| 38 | |||
| 39 | croak "S3 storage: bucket is required\n" | ||
| 40 | if !defined $storage{bucket} || $storage{bucket} eq q{}; | ||
| 41 | |||
| 42 | my $self = bless { | ||
| 43 | config => $config, | ||
| 44 | bucket => $storage{bucket}, | ||
| 45 | prefix => $storage{prefix}, | ||
| 46 | region => $storage{region}, | ||
| 47 | profile => $storage{profile}, | ||
| 48 | base_url => $config->{url}, # public (CloudFront) base | ||
| 49 | }, $class; | ||
| 50 | |||
| 51 | return $self; | ||
| 52 | } | ||
| 53 | |||
| 54 | ######################################################################## | ||
| 55 | sub _s3 { | ||
| 56 | ######################################################################## | ||
| 57 | my ($self) = @_; | ||
| 58 | |||
| 59 | return $self->{s3} //= Amazon::S3::Lite->new( | ||
| 60 | { region => $self->{region}, | ||
| 61 | credentials => Amazon::Credentials->new( profile => $self->{profile} ), | ||
| 62 | } | ||
| 63 | ); | ||
| 64 | } | ||
| 65 | |||
| 66 | ######################################################################## | ||
| 67 | sub base_url { return $_[0]->{base_url} } | ||
| 68 | ######################################################################## | ||
| 69 | |||
| 70 | ######################################################################## | ||
| 71 | # list_distributions() -> list of keys (relative to the bucket) that | ||
| 72 | # look like distribution tarballs under the configured prefix. Paginates | ||
| 73 | # via continuation token. The .tar.gz filter lives here because "what | ||
| 74 | # counts as a distribution" is the one bit of domain knowledge this | ||
| 75 | # seam legitimately owns. | ||
| 76 | ######################################################################## | ||
| 77 | sub list_distributions { | ||
| 78 | ######################################################################## | ||
| 79 | my ($self) = @_; | ||
| 80 | |||
| 81 | my $s3 = $self->_s3; | ||
| 82 | my @keys; | ||
| 83 | my $token; | ||
| 84 | |||
| 85 | while (1) { | ||
| 86 | my %options = ( defined $self->{prefix} ? ( prefix => $self->{prefix} ) : () ); | ||
| 87 | |||
| 88 | if ( defined $token ) { | ||
| 89 | $options{continuation_token} = $token; | ||
| 90 | } | ||
| 91 | |||
| 92 | my $result = $s3->list_objects_v2( $self->{bucket}, %options ); | ||
| 93 | |||
| 94 | for my $object ( @{ $result->{objects} || [] } ) { | ||
| 95 | my $key = $object->{key}; | ||
| 96 | next if !defined $key; | ||
| 97 | next if $key !~ m{[.]tar[.]gz\z}xsm; | ||
| 98 | push @keys, $key; | ||
| 99 | } | ||
| 100 | |||
| 101 | last | ||
| 102 | if !$result->{is_truncated}; | ||
| 103 | |||
| 104 | $token = $result->{next_continuation_token}; | ||
| 105 | last if !$token; | ||
| 106 | } | ||
| 107 | |||
| 108 | return @keys; | ||
| 109 | } | ||
| 110 | |||
| 111 | ######################################################################## | ||
| 112 | # fetch_object($key) -> raw bytes, or undef if genuinely absent. | ||
| 113 | # A real I/O failure throws (fail-fast contract). | ||
| 114 | ######################################################################## | ||
| 115 | sub fetch_object { | ||
| 116 | my ( $self, $key ) = @_; | ||
| 117 | |||
| 118 | my $object = $self->_s3->get_object( $self->{bucket}, $key ); | ||
| 119 | |||
| 120 | return | ||
| 121 | if !defined $object; # not found -> undef | ||
| 122 | |||
| 123 | return $object->{content}; | ||
| 124 | } | ||
| 125 | |||
| 126 | ######################################################################## | ||
| 127 | # has_object($key) -> boolean. Wraps head_object. | ||
| 128 | ######################################################################## | ||
| 129 | sub has_object { | ||
| 130 | my ( $self, $key ) = @_; | ||
| 131 | |||
| 132 | return $self->_s3->head_object( $self->{bucket}, $key ) ? 1 : 0; | ||
| 133 | } | ||
| 134 | |||
| 135 | ######################################################################## | ||
| 136 | # save_object($key, $bytes, %opts) -> writes. %opts may carry | ||
| 137 | # content_type; caller (Utils) already gzips, so this moves bytes only. | ||
| 138 | ######################################################################## | ||
| 139 | sub save_object { | ||
| 140 | my ( $self, $key, $content, %opts ) = @_; | ||
| 141 | |||
| 142 | my $content_type = $opts{content_type} // 'application/octet-stream'; | ||
| 143 | |||
| 144 | return $self->_s3->put_object( $self->{bucket}, $key, $content, content_type => $content_type, ); | ||
| 145 | } | ||
| 146 | |||
| 147 | ######################################################################## | ||
| 148 | # lock($key, %opts) -> a GUARD object on success, false on failure. | ||
| 149 | # | ||
| 150 | # CONTRACT (both engines must match): lock() acquires and returns a | ||
| 151 | # guard. The guard is true on success and releases the lock in its | ||
| 152 | # DESTROY when it goes out of scope â including on die. The caller MUST | ||
| 153 | # bind it to a lexical for the whole critical section: | ||
| 154 | # | ||
| 155 | # my $guard = $storage->lock($key, ttl => 300, wait => 60) | ||
| 156 | # or croak 'could not acquire packages index lock'; | ||
| 157 | # ... fetch -> mutate -> save ... | ||
| 158 | # # $guard drops here -> release, even if the block died | ||
| 159 | # | ||
| 160 | # Discarding the return (void context, or letting it fall off the end of | ||
| 161 | # a block) releases the lock immediately and defeats the mutex. Hold it. | ||
| 162 | # | ||
| 163 | # S3 uses the conditional-PUT lock (Amazon::S3::Lite::Lock); its guard | ||
| 164 | # releases with If-Match on its own etag so it never clobbers a lock | ||
| 165 | # that was stolen after a TTL lapse. The Filesystem engine returns an | ||
| 166 | # flock-based guard instead â same contract, different mechanism. | ||
| 167 | |||
| 168 | ######################################################################## | ||
| 169 | sub lock { | ||
| 170 | ######################################################################## | ||
| 171 | my ( $self, $key, %opts ) = @_; | ||
| 172 | |||
| 173 | require Amazon::S3::Lite::Lock; | ||
| 174 | |||
| 175 | # acquire() returns the guard on success, false if the lock is held | ||
| 176 | # and could not be (stolen as stale / waited out). | ||
| 177 | return Amazon::S3::Lite::Lock->new( | ||
| 178 | s3 => $self->_s3, | ||
| 179 | bucket => $self->{bucket}, | ||
| 180 | key => $key, | ||
| 181 | ttl => $opts{ttl} // 300, | ||
| 182 | wait => $opts{wait} // 0, | ||
| 183 | )->acquire; | ||
| 184 | } | ||
| 185 | |||
| 186 | ######################################################################## | ||
| 187 | sub main { | ||
| 188 | ######################################################################## | ||
| 189 | my $config_file = shift @ARGV; | ||
| 190 | |||
| 191 | require JSON; | ||
| 192 | |||
| 193 | open my $fh, '<', $config_file | ||
| 194 | or die "ERROR: could not open $config_file for reading\n$OS_ERROR"; | ||
| 195 | |||
| 196 | my $config; | ||
| 197 | |||
| 198 | { | ||
| 199 | local $RS = undef; | ||
| 200 | $config = JSON->new->decode(<$fh>); | ||
| 201 | close $fh; | ||
| 202 | } | ||
| 203 | |||
| 204 | my $s3_storage = __PACKAGE__->new( $config->{bedrock} ); | ||
| 205 | |||
| 206 | print Dumper( [ distributions => $s3_storage->list_distributions ] ); | ||
| 207 | |||
| 208 | return 0; | ||
| 209 | } | ||
| 210 | |||
| 211 | 1; |