| 1 | package DarkPAN::Indexer::Storage; | ||||||||||
| 2 | |||||||||||
| 3 | # The storage contract. This is an INTERFACE role, not a behavior mixin: | ||||||||||
| 4 | # it provides almost nothing and instead `requires` the six methods every | ||||||||||
| 5 | # storage engine must implement. A class becomes a usable backend by: | ||||||||||
| 6 | # | ||||||||||
| 7 | # package DarkPAN::Indexer::Storage::Foo; | ||||||||||
| 8 | # use Role::Tiny::With; | ||||||||||
| 9 | # with 'DarkPAN::Indexer::Storage'; # dies at compile time if a | ||||||||||
| 10 | # # required method is missing | ||||||||||
| 11 | # | ||||||||||
| 12 | # The loader resolves a config `type` to DarkPAN::Indexer::Storage::<type> | ||||||||||
| 13 | # (or a +Fully::Qualified name verbatim), requires it, and asserts | ||||||||||
| 14 | # $class->DOES('DarkPAN::Indexer::Storage') | ||||||||||
| 15 | # before constructing via new_from_config. So `requires` guarantees the | ||||||||||
| 16 | # methods exist; DOES guarantees the class actually consumed this role. | ||||||||||
| 17 | # | ||||||||||
| 18 | # TWO LAYERS, don't confuse them. These are the generic, backend-specific | ||||||||||
| 19 | # primitives (byte movers + enumeration + lock). The DOMAIN layer | ||||||||||
| 20 | # (Role::Indexer, Role::Utils) speaks intent -- fetch_packages_version_index, | ||||||||||
| 21 | # save_packages_version_index -- and calls DOWN to these. Domain names | ||||||||||
| 22 | # never appear here; a backend does not know what an "index" is, only how | ||||||||||
| 23 | # to move bytes for a key. | ||||||||||
| 24 | |||||||||||
| 25 | 5 | ▶ | use strict; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 26 | 5 | ▶ | use warnings; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 27 | |||||||||||
| 28 | 5 | ▶ | use Carp qw(croak); | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 29 | 5 | ▶ | use English qw(-no_match_vars); | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 30 | 5 | ▶ | use File::Temp qw(tempfile); | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 31 | |||||||||||
| 32 | 5 | ▶ | use Role::Tiny; | ||||||||
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||||||||||
| 33 | |||||||||||
| 34 | # --- construction ----------------------------------------------------- | ||||||||||
| 35 | # | ||||||||||
| 36 | # new($class, $entry) | ||||||||||
| 37 | # Build an engine from a merged config entry (see the config schema). | ||||||||||
| 38 | # Each engine reads its own settings from $entry->{storage}; the S3 | ||||||||||
| 39 | # engine additionally honors a legacy $entry->{AWS} block, storage | ||||||||||
| 40 | # winning on collision. Other engines are greenfield -- storage only. | ||||||||||
| 41 | 5 | requires 'new'; | |||||||||
| 42 | |||||||||||
| 43 | # --- enumeration (domain-aware, backend-specific) --------------------- | ||||||||||
| 44 | # | ||||||||||
| 45 | # list_distributions() -> list of keys (relative to the storage root) | ||||||||||
| 46 | # that are distribution tarballs. This is the ONE seam method that | ||||||||||
| 47 | # legitimately understands the tree: it decides what counts as a | ||||||||||
| 48 | # distribution (e.g. the .tar.gz filter) and where to scan. Everything | ||||||||||
| 49 | # else here is dumb byte movement. | ||||||||||
| 50 | 5 | requires 'list_distributions'; | |||||||||
| 51 | |||||||||||
| 52 | # --- object primitives (generic, key-in / bytes-out) ------------------ | ||||||||||
| 53 | # | ||||||||||
| 54 | # fetch_object($key) -> raw bytes, or undef if the key is genuinely | ||||||||||
| 55 | # absent. A real I/O error throws (fail-fast). Keys are always | ||||||||||
| 56 | # relative to the storage root; the engine resolves them (bucket+key, | ||||||||||
| 57 | # root+path, etc). | ||||||||||
| 58 | 5 | requires 'fetch_object'; | |||||||||
| 59 | |||||||||||
| 60 | # save_object($key, $bytes, %opts) -> writes bytes at $key. %opts may | ||||||||||
| 61 | # carry content_type. Content is already in final form (the domain | ||||||||||
| 62 | # layer gzips before calling); this moves bytes only. Throws on failure. | ||||||||||
| 63 | 5 | requires 'save_object'; | |||||||||
| 64 | |||||||||||
| 65 | # has_object($key) -> boolean. Existence check (the head-object case). | ||||||||||
| 66 | 5 | requires 'has_object'; | |||||||||
| 67 | |||||||||||
| 68 | # --- mutual exclusion ------------------------------------------------- | ||||||||||
| 69 | # | ||||||||||
| 70 | # lock($key, %opts) -> a GUARD object on success, false on failure. | ||||||||||
| 71 | # The guard is true, and releases the lock in DESTROY when it leaves | ||||||||||
| 72 | # scope (including on die). Callers MUST bind it to a lexical for the | ||||||||||
| 73 | # whole critical section: | ||||||||||
| 74 | # | ||||||||||
| 75 | # my $guard = $storage->lock($key, ttl => 300, wait => 60) | ||||||||||
| 76 | # or croak 'could not acquire lock'; | ||||||||||
| 77 | # ... fetch -> mutate -> save ... | ||||||||||
| 78 | # # $guard drops here -> release | ||||||||||
| 79 | # | ||||||||||
| 80 | # Discarding the return releases immediately and defeats the mutex. | ||||||||||
| 81 | # %opts: ttl (seconds a held lock stays fresh), wait (seconds to block | ||||||||||
| 82 | # trying to acquire; 0 = don't wait). Mechanism is engine-specific | ||||||||||
| 83 | # (S3 conditional-PUT vs flock); the contract above is not. | ||||||||||
| 84 | 5 | requires 'lock'; | |||||||||
| 85 | |||||||||||
| 86 | # --- read path -------------------------------------------------------- | ||||||||||
| 87 | # | ||||||||||
| 88 | # base_url() -> the public base URL from which a read-only consumer (the | ||||||||||
| 89 | # resolver) reaches this DarkPAN, or undef if it has none (e.g. a | ||||||||||
| 90 | # purely local store with no web front). The resolver fetches | ||||||||||
| 91 | # "{base_url}/{index path}". This is derived from config (`url`), not | ||||||||||
| 92 | # from the storage backend's own endpoint -- an S3 store fronted by | ||||||||||
| 93 | # CloudFront reads from the CloudFront domain, not the bucket. | ||||||||||
| 94 | 5 | requires 'base_url'; | |||||||||
| 95 | |||||||||||
| 96 | ######################################################################## | ||||||||||
| 97 | 6 | ▶ | sub _slurp { | ||||||||
|
Subroutine
_slurp: called CC=2 SCAR=7.0
| |||||||||||
| 98 | ######################################################################## | ||||||||||
| 99 | 6 | my ( $self, $file ) = @_; | |||||||||
| 100 | |||||||||||
| 101 | 6 | local $RS = undef; | |||||||||
| 102 | |||||||||||
| 103 | 6 | my $content; | |||||||||
| 104 | |||||||||||
| 105 | 6 | ▶ | open my $fh, '<', $file | ||||||||
|
Branch: 1 branch
| |||||||||||
| 106 | or croak "ERROR: could not open $file for reading\n$OS_ERROR"; | ||||||||||
| 107 | |||||||||||
| 108 | 6 | binmode $fh; | |||||||||
| 109 | |||||||||||
| 110 | 6 | $content = <$fh>; | |||||||||
| 111 | 6 | close $fh; | |||||||||
| 112 | |||||||||||
| 113 | 6 | return $content; | |||||||||
| 114 | } | ||||||||||
| 115 | |||||||||||
| 116 | ######################################################################## | ||||||||||
| 117 | 6 | ▶ | sub publish_index { | ||||||||
|
Subroutine
publish_index: called CC=4 SCAR=14.5
Pod
undocumented
| |||||||||||
| 118 | ######################################################################## | ||||||||||
| 119 | 6 | my ( $self, $temp_index, $index ) = @_; | |||||||||
| 120 | |||||||||||
| 121 | 6 | my $uncompressed_content = $self->_slurp($temp_index); | |||||||||
| 122 | 6 | my $content = q{}; | |||||||||
| 123 | |||||||||||
| 124 | 6 | my $is_zipped = $index =~ /[.]gz$/xsm; | |||||||||
| 125 | |||||||||||
| 126 | 6 | ▶ | if ($is_zipped) { | ||||||||
|
Branch: 1 branch
| |||||||||||
| 127 | 6 | require IO::Compress::Gzip; | |||||||||
| 128 | |||||||||||
| 129 | 6 | ▶ | IO::Compress::Gzip::gzip( \$uncompressed_content, \$content ) | ||||||||
|
Branch: 1 branch
| |||||||||||
| 130 | or croak "ERROR: could not gzip $temp_index\n"; | ||||||||||
| 131 | } | ||||||||||
| 132 | else { | ||||||||||
| 133 | 0 | $content = $uncompressed_content; | |||||||||
| 134 | } | ||||||||||
| 135 | |||||||||||
| 136 | 6 | ▶ | $self->save_object( $index, $content, content_type => $is_zipped ? 'application/gzip' : 'application/octet-stream' ); | ||||||||
|
Branch: 1 branch
| |||||||||||
| 137 | |||||||||||
| 138 | 6 | return; | |||||||||
| 139 | } | ||||||||||
| 140 | |||||||||||
| 141 | ######################################################################## | ||||||||||
| 142 | 4 | ▶ | sub retrieve_index { | ||||||||
|
Subroutine
retrieve_index: called CC=3 SCAR=11.1
Pod
undocumented
| |||||||||||
| 143 | ######################################################################## | ||||||||||
| 144 | 4 | my ( $self, $index ) = @_; | |||||||||
| 145 | |||||||||||
| 146 | 4 | my $is_zipped = $index =~ /[.]gz$/xsm; | |||||||||
| 147 | |||||||||||
| 148 | 4 | my $compressed_content = $self->fetch_object($index); | |||||||||
| 149 | 4 | my $content = q{}; | |||||||||
| 150 | |||||||||||
| 151 | 4 | ▶ | if ($is_zipped) { | ||||||||
|
Branch: 1 branch
| |||||||||||
| 152 | 4 | require IO::Uncompress::Gunzip; | |||||||||
| 153 | |||||||||||
| 154 | 4 | ▶ | IO::Uncompress::Gunzip::gunzip( \$compressed_content, \$content ) | ||||||||
|
Branch: 1 branch
| |||||||||||
| 155 | or croak "ERROR: could not unzip $index\n"; | ||||||||||
| 156 | } | ||||||||||
| 157 | else { | ||||||||||
| 158 | 0 | $content = $compressed_content; | |||||||||
| 159 | } | ||||||||||
| 160 | |||||||||||
| 161 | 4 | my ( $fh, $temp_index ) = tempfile( DIR => '/tmp', UNLINK => 0 ); | |||||||||
| 162 | 4 | binmode $fh; | |||||||||
| 163 | |||||||||||
| 164 | 4 | print {$fh} $content; | |||||||||
| 165 | |||||||||||
| 166 | 4 | close $fh; | |||||||||
| 167 | |||||||||||
| 168 | 4 | return $temp_index; | |||||||||
| 169 | } | ||||||||||
| 170 | |||||||||||
| 171 | 5 | 1; | |||||||||