Indexer/Storage/Filesystem.pm

filter: Statementstmt 87.1% 108 / 124 Branchbran 52.0% 26 / 50 Conditioncond 40.0% 4 / 10 MC/DCmcdc 0.0% 0 / 6 Subroutinesub 88.8% 24 / 27 Podpod 0.0% 0 / 7 totaltotal 72.3% 162 / 224 CC 30 SCAR 17.1
CC
30
Cov
75%
lock
17.4
save_object
7.3
has_object
6.0
CRAP 5.5
« Indexer.pm ↑ Summary Indexer/CLI.pm »
1 package DarkPAN::Indexer::Storage::Filesystem;
2  
3 # Filesystem storage engine for DarkPAN::Indexer.
4 #
5 # Implements the DarkPAN::Indexer::Storage contract against a local
6 # directory tree. Keys are paths RELATIVE TO the configured root, exactly
7 # as S3 keys are relative to the bucket -- so the orchestrator and format
8 # are identical across backends.
9 #
10 # retrieve_index / publish_index / _slurp are PROVIDED by the role (they
11 # call fetch_object / save_object), so this engine implements only the
12 # seven required primitives:
13 # new list_distributions fetch_object save_object has_object lock base_url
14 #
15 # Config shape (in the DarkPAN entry):
16 # storage => { type => 'Filesystem', root => '/srv/darkpan' }
17 # url => 'http://localhost:8080' # optional public base for reads
18 #
19 # This is the credential-free, network-free backend: it doubles as the
20 # test harness for the whole indexer and as the laptop-runnable demo.
21  
22 5 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 5 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
24  
25 5 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
26 5 ▶use Data::Dumper;
Subroutine BEGIN: called CC=1 SCAR=0.0
27 5 ▶use English qw(-no_match_vars);
Subroutine BEGIN: called CC=1 SCAR=0.0
28 5 ▶use File::Basename qw(dirname);
Subroutine BEGIN: called CC=1 SCAR=0.0
29 5 ▶use File::Find qw(find);
Subroutine BEGIN: called CC=1 SCAR=0.0
30 5 ▶use File::Path qw(make_path);
Subroutine BEGIN: called CC=1 SCAR=0.0
31 5 ▶use File::Spec;
Subroutine BEGIN: called CC=1 SCAR=0.0
32 5 ▶use Fcntl qw(:flock);
Subroutine BEGIN: called CC=1 SCAR=0.0
33  
34 5 ▶use Role::Tiny::With;
Subroutine BEGIN: called CC=1 SCAR=0.0
35 5 with 'DarkPAN::Indexer::Storage';
36  
37 ########################################################################
38 5 ▶sub new {
Subroutine new: called CC=5 SCAR=17.3
Pod undocumented
39 ########################################################################
40 5 my ( $class, $config ) = @_;
41  
42 5 ▶ my %storage = %{ $config->{storage} // {} };
Condition50%1/2
$config->{"storage"} // {}
l!l
50
MC/DC0%0/1
$config->{"storage"} // {}
$config->{"storage"}
Truth table50%1/2
A
A= $config->{"storage"}
Aresult
11
00
43  
44 5 my $root = $storage{root};
45  
46 5 ▶ croak "Filesystem storage: root is required\n"
Branch: 1 branch
BranchTrueFalseTotal
if not defined $root or $root eq "" 0 5 5
Condition33%1/3
not defined $root or $root eq ""
l!l&&r!l&&!r
005
MC/DC0%0/2
not defined $root or $root eq ""
not defined $root $root eq ""
Truth table33%1/3
not(A) or B
A= not defined $rootB= $root eq ""
ABresult
1X1
011
000
47 if !defined $root || $root eq q{};
48  
49 5 ▶ croak "Filesystem storage: root '$root' is not a directory\n"
Branch: 1 branch
BranchTrueFalseTotal
unless -d $root 0 5 5
50 if !-d $root;
51  
52 my $self = bless {
53 config => $config,
54 root => $root,
55 prefix => $storage{prefix}, # optional scan sub-root under root
56 base_url => $config->{url}, # optional public base for the resolver
57 5 }, $class;
58  
59 5 return $self;
60 }
61  
62 ########################################################################
63 0 ▶sub base_url { return $_[0]->{base_url} }
Subroutine base_url: not called CC=1 SCAR=6.9
Pod undocumented
64 ########################################################################
65  
66 ########################################################################
67 # _abs($key) -> absolute path for a root-relative key.
68 ########################################################################
69 28 ▶sub _abs {
Subroutine _abs: called CC=1 SCAR=0.0
70 ########################################################################
71 28 my ( $self, $key ) = @_;
72  
73 28 return File::Spec->catfile( $self->{root}, $key );
74 }
75  
76 ########################################################################
77 # list_distributions() -> root-relative keys of .tar.gz files under the
78 # configured prefix (or the whole root if no prefix). Same domain filter
79 # as the S3 engine: this seam owns "what counts as a distribution".
80 ########################################################################
81 4 ▶sub list_distributions {
Subroutine list_distributions: called CC=3 SCAR=11.0
Pod undocumented
82 ########################################################################
83 4 my ($self) = @_;
84  
85 my $scan_root
86 = defined $self->{prefix}
87 ? File::Spec->catdir( $self->{root}, $self->{prefix} )
88 4 ▶ : $self->{root};
Branch: 1 branch
BranchTrueFalseTotal
defined $self->{'prefix'} ? : 0 4 4
89  
90 return
91 4 ▶ if !-d $scan_root;
Branch: 1 branch
BranchTrueFalseTotal
unless -d $scan_root 0 4 4
92  
93 4 my @keys;
94  
95 find(
96 { no_chdir => 1,
97 29 ▶ wanted => sub {
Subroutine __ANON__: called CC=3 SCAR=11.0
98 29 my $path = $File::Find::name;
99 29 ▶ return if !-f $path;
Branch: 1 branch
BranchTrueFalseTotal
unless -f $path 19 10 29
100 10 ▶ return if $path !~ m{[.]tar[.]gz\z}xsm;
Branch: 1 branch
BranchTrueFalseTotal
unless $path =~ /[.]tar[.]gz\z/msx 4 6 10
101  
102 # convert absolute path back to a root-relative key
103 6 my $key = File::Spec->abs2rel( $path, $self->{root} );
104 6 push @keys, $key;
105 },
106 },
107 4 $scan_root,
108 );
109  
110 4 return @keys;
111 }
112  
113 ########################################################################
114 # fetch_object($key) -> raw bytes, or undef if genuinely absent.
115 # A real read error (exists but unreadable) throws.
116 ########################################################################
117 12 ▶sub fetch_object {
Subroutine fetch_object: called CC=4 SCAR=14.0
Pod undocumented
118 ########################################################################
119 12 my ( $self, $key ) = @_;
120  
121 12 my $path = $self->_abs($key);
122  
123 return
124 12 ▶ if !-e $path; # not found -> undef (matches S3 semantics)
Branch: 1 branch
BranchTrueFalseTotal
unless -e $path 2 10 12
125  
126 10 ▶ open my $fh, '<', $path
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $path or croak("Filesystem storage: cannot read '${path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 10 0 10
127 or croak "Filesystem storage: cannot read '$path': $OS_ERROR\n";
128 10 binmode $fh;
129  
130 10 local $RS = undef;
131 10 my $content = <$fh>;
132  
133 10 ▶ close $fh
Branch: 1 branch
BranchTrueFalseTotal
close $fh or croak("Filesystem storage: cannot close '${path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 10 0 10
134 or croak "Filesystem storage: cannot close '$path': $OS_ERROR\n";
135  
136 10 return $content;
137 }
138  
139 ########################################################################
140 # has_object($key) -> boolean.
141 ########################################################################
142 0 ▶sub has_object {
Subroutine has_object: not called CC=2 SCAR=17.9
Pod undocumented
143 ########################################################################
144 0 my ( $self, $key ) = @_;
145  
146 0 ▶ return -e $self->_abs($key) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
-e $self->_abs($key) ? : 0 0 0
147 }
148  
149 ########################################################################
150 # save_object($key, $bytes, %opts) -> writes bytes at $key, creating
151 # intermediate directories. content_type is accepted for contract
152 # symmetry with S3 but has no meaning on a filesystem.
153 ########################################################################
154 6 ▶sub save_object {
Subroutine save_object: called CC=6 SCAR=19.9
Pod undocumented
155 ########################################################################
156 6 my ( $self, $key, $content, %opts ) = @_;
157  
158 6 my $path = $self->_abs($key);
159 6 my $dir = dirname($path);
160  
161 6 ▶ if ( !-d $dir ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (-d $dir) 0 6 6
162 0 ▶ make_path($dir)
Branch: 1 branch
BranchTrueFalseTotal
make_path($dir) or croak("Filesystem storage: cannot create '${dir}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 0 0 0
163 or croak "Filesystem storage: cannot create '$dir': $OS_ERROR\n";
164 }
165  
166 6 ▶ open my $fh, '>', $path
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $path or croak("Filesystem storage: cannot write '${path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 6 0 6
167 or croak "Filesystem storage: cannot write '$path': $OS_ERROR\n";
168 6 binmode $fh;
169  
170 6 ▶ print {$fh} $content
Branch: 1 branch
BranchTrueFalseTotal
print {$fh;} $content or croak("Filesystem storage: write failed for '${path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 6 0 6
171 or croak "Filesystem storage: write failed for '$path': $OS_ERROR\n";
172  
173 6 ▶ close $fh
Branch: 1 branch
BranchTrueFalseTotal
close $fh or croak("Filesystem storage: cannot close '${path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 6 0 6
174 or croak "Filesystem storage: cannot close '$path': $OS_ERROR\n";
175  
176 6 return 1;
177 }
178  
179 ########################################################################
180 # lock($key, %opts) -> a GUARD on success, false on failure.
181 #
182 # Same contract as the S3 engine: bind the return to a lexical; the guard
183 # releases (unlocks + closes) in DESTROY when it leaves scope, including
184 # on die. Mechanism here is flock on a "<key>.lock" file under the root.
185 #
186 # %opts: wait (seconds to block for the lock; 0 = non-blocking, fail fast).
187 # ttl is accepted for contract symmetry but is not meaningful for flock --
188 # the OS releases the lock if the holder dies, so there is no stale lock
189 # to time out.
190 ########################################################################
191 10 ▶sub lock {
Subroutine lock: called CC=8 SCAR=28.6
Pod undocumented
192 ########################################################################
193 10 my ( $self, $key, %opts ) = @_;
194  
195 10 ▶ my $wait = $opts{wait} // 0;
Condition50%1/2
$opts{"wait"} // 0
l!l
010
MC/DC0%0/1
$opts{"wait"} // 0
$opts{"wait"}
Truth table50%1/2
A
A= $opts{"wait"}
Aresult
11
00
196  
197 10 my $lock_path = $self->_abs("$key.lock");
198 10 my $dir = dirname($lock_path);
199  
200 10 ▶ make_path($dir) if !-d $dir;
Branch: 1 branch
BranchTrueFalseTotal
unless -d $dir 5 5 10
201  
202 10 ▶ open my $fh, '>', $lock_path
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, ">", $lock_path or croak("Filesystem storage: cannot open lock '${lock_path}': $DarkPAN::Indexer::Storage::Filesystem::OS_ERROR\n") 10 0 10
203 or croak "Filesystem storage: cannot open lock '$lock_path': $OS_ERROR\n";
204  
205 10 ▶ my $mode = $wait ? LOCK_EX : ( LOCK_EX | LOCK_NB );
Branch: 1 branch
BranchTrueFalseTotal
$wait ? : 0 10 10
206  
207 10 ▶ if ($wait) {
Branch: 1 branch
BranchTrueFalseTotal
if ($wait) { } 0 10 10
208 # block up to $wait seconds using an alarm, so we don't hang forever
209 0 my $got = eval {
210 0 ▶ local $SIG{ALRM} = sub { die "timeout\n" };
Subroutine __ANON__: not called CC=1 SCAR=6.9
211 0 alarm $wait;
212 0 my $ok = flock $fh, LOCK_EX;
213 0 alarm 0;
214 0 return $ok;
215 };
216 0 ▶ if ( !$got ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($got) 0 0 0
217 0 close $fh;
218 0 return; # could not acquire within wait
219 }
220 }
221 else {
222 10 ▶ if ( !flock $fh, LOCK_EX | LOCK_NB ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (flock $fh, 6) 0 10 10
223 0 close $fh;
224 0 return; # held by someone else, non-blocking
225 }
226 }
227  
228 10 return DarkPAN::Indexer::Storage::Filesystem::Guard->new(
229 fh => $fh,
230 path => $lock_path,
231 );
232 }
233  
234 ########################################################################
235 package DarkPAN::Indexer::Storage::Filesystem::Guard;
236 ########################################################################
237 # RAII flock guard. True while held; releases (unlock, close, unlink) on
238 # DESTROY. flock is advisory and OS-released on process exit, so unlike
239 # the S3 guard there is no stolen-lock hazard -- a dead holder's lock is
240 # freed by the kernel.
241  
242 5 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
243 5 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
244  
245 5 ▶use Fcntl qw(:flock);
Subroutine BEGIN: called CC=1 SCAR=0.0
246  
247 10 ▶sub new {
Subroutine new: called CC=1 SCAR=0.0
248 10 my ( $class, %args ) = @_;
249 10 return bless { %args, released => 0 }, $class;
250 }
251  
252 10 ▶sub release {
Subroutine release: called CC=5 SCAR=17.3
253 10 my ($self) = @_;
254  
255 10 ▶ return if $self->{released};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"released"} 0 10 10
256 10 $self->{released} = 1;
257  
258 10 ▶ if ( $self->{fh} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->{"fh"}) 10 0 10
259 10 flock $self->{fh}, LOCK_UN;
260 10 close $self->{fh};
261 }
262  
263 # best-effort cleanup of the lockfile; harmless if another waiter
264 # recreated it.
265 10 ▶ unlink $self->{path} if $self->{path} && -e $self->{path};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"path"} and -e $self->{"path"} 10 0 10
Condition33%1/3
$self->{"path"} and -e $self->{"path"}
!ll&&!rl&&r
0010
MC/DC0%0/2
$self->{"path"} and -e $self->{"path"}
$self->{"path"} -e $self->{"path"}
Truth table33%1/3
A and B
A= $self->{"path"}B= -e $self->{"path"}
ABresult
0X0
100
111
266  
267 10 return;
268 }
269  
270 10 ▶sub DESTROY {
Subroutine DESTROY: called CC=2 SCAR=7.0
271 10 my ($self) = @_;
272 10 ▶ return if ${^GLOBAL_PHASE} eq 'DESTRUCT';
Branch: 1 branch
BranchTrueFalseTotal
if ${^GLOBAL_PHASE} eq "DESTRUCT" 0 10 10
273 10 $self->release;
274 10 return;
275 }
276  
277 5 1;
« Indexer.pm ↑ Summary Indexer/CLI.pm »