blib/lib/App/FileSync/Scan.pm

filter: Statementstmt 96.0% 120 / 125 Branchbran 72.5% 29 / 40 Conditioncond 66.6% 22 / 33 MC/DCmcdc 40.0% 6 / 15 Subroutinesub 100% 22 / 22 Podpod 100% 2 / 2 totaltotal 84.8% 201 / 237 CC 35 SCAR 22.4
CC
35
Cov
86%
clean_stale_temps
17.1
_walk
13.2
__ANON__
11.1
CRAP 9.4
« blib/lib/App/FileSync/Verify.pm ↑ Summary blib/lib/App/FileSync/Client.pm »
1 package App::FileSync::Scan;
2  
3 # Walk a tree and describe it: one manifest entry per regular file (with its
4 # SHA-256) and per empty directory. A previous manifest serves as a cache, so
5 # a file whose size and mtime are unchanged is not read again.
6  
7 329 ▶use App::FileSync::Platform;
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 329 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 329 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 329 ▶use autodie qw(:all);
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 329 ▶use Digest::SHA ();
Subroutine BEGIN: called CC=1 SCAR=0.0
14 329 ▶use File::Spec;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 329 ▶use Readonly;
Subroutine BEGIN: called CC=1 SCAR=0.0
16 329 ▶use Sub::Private { mode => 'enforce' };
Subroutine BEGIN: called CC=1 SCAR=0.0
17 329 ▶use Time::HiRes ();
Subroutine BEGIN: called CC=1 SCAR=0.0
18  
19 329 ▶use App::FileSync::Args qw(named);
Subroutine BEGIN: called CC=1 SCAR=0.0
20 329 ▶use App::FileSync::Constants qw($MODE_MASK $TEMP_PREFIX $STALE_TEMP_SECONDS $TYPE_FILE $TYPE_DIR $STAT_MODE $STAT_SIZE $STAT_MTIME $SHA_BITS);
Subroutine BEGIN: called CC=1 SCAR=0.0
21 329 ▶use App::FileSync::I18N qw(message os_error);
Subroutine BEGIN: called CC=1 SCAR=0.0
22 329 ▶use App::FileSync::IO qw(list_dir);
Subroutine BEGIN: called CC=1 SCAR=0.0
23  
24 329 ▶use version; our $VERSION = version->declare('0.002.1');
Subroutine BEGIN: called CC=1 SCAR=0.0
25  
26 # A cached mtime within this much of the file's is the same mtime: JSON
27 # round-trips the fractional part with float noise.
28 329 Readonly::Scalar my $MTIME_EPSILON => 0.001;
29  
30 # Matches a temporary file left by an interrupted run.
31 329 Readonly my $TEMP_NAME => qr/\A\Q$TEMP_PREFIX\E/;
32  
33 # Describe one regular file, reusing the cached hash when size and mtime
34 # match. Lexical rather than a named private sub because it runs once per
35 # file in the tree; see App::FileSync/LIMITATIONS.
36 #
37 # Entry: $context is the scan state; $abs/$rel name the file; $stat is lstat.
38 # Exit: returns the manifest entry, or undef after recording the file as
39 # skipped (it could not be read).
40 1197 ▶my $file_entry = sub {
Subroutine __ANON__: called CC=10 SCAR=24.0
41 1197 my ($context, $abs, $rel, $stat) = @_;
42  
43 1197 my ($size, $mtime) = @{$stat}[$STAT_SIZE, $STAT_MTIME];
44 1197 my $cached = $context->{cache}{$rel};
45  
46 1197 my $hash;
47 1197 ▶ if ($cached
Branch: 2 branches
BranchTrueFalseTotal
if ($cached and ($cached->{'t'} || '') eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{'h'} and defined $cached->{'s'} and $cached->{'s'} == $size and abs((defined $cached->{'m'} ? $cached->{'m'} : -1) - $mtime) < $MTIME_EPSILON) { } 851 346 1197
defined $cached->{'m'} ? : 854 0 854
Condition76%13/17
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{"h"} and defined $cached->{"s"} and $cached->{"s"} == $size and abs((defined $cached->{"m"} ? $cached->{"m"} : -1) - $mtime) < $MTIME_EPSILON
!ll&&!rl&&r
3433851
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{"h"} and defined $cached->{"s"} and $cached->{"s"} == $size
!ll&&!rl&&r
29845854
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{"h"} and defined $cached->{"s"}
!ll&&!rl&&r
2980899
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{"h"}
!ll&&!rl&&r
2980899
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE
!ll&&!rl&&r
2980899
$cached->{'t'} || ''
l!l
8990
MC/DC50%3/6
$cached and ($cached->{"t"} || "") eq $App::FileSync::Scan::TYPE_FILE and defined $cached->{"h"} and defined $cached->{"s"} and $cached->{"s"} == $size and abs((defined $cached->{"m"} ? $cached->{"m"} : -1) - $mtime) < $MTIME_EPSILON
$cached $cached->{'t'} defined $cached->{"h"} defined $cached->{"s"} $cached->{"s"} == $size abs((defined $cached->{"m"} ? $cached->{"m"} : -1) - $mtime) < $MTIME_EPSILON
Truth table57%4/7
A and B and C and D and E and F
A= $cachedB= $cached->{'t'}C= defined $cached->{"h"}D= defined $cached->{"s"}E= $cached->{"s"} == $sizeF= abs((defined $cached->{"m"} ? $cached->{"m"} : -1) - $mtime) < $MTIME_EPSILON
ABCDEFresult
0XXXXX0
10XXXX0
110XXX0
1110XX0
11110X0
1111100
1111111
48 && ($cached->{t} || '') eq $TYPE_FILE
49 && defined $cached->{h}
50 && defined $cached->{s} && $cached->{s} == $size
51 && abs((defined $cached->{m} ? $cached->{m} : -1) - $mtime) < $MTIME_EPSILON) {
52 851 $hash = $cached->{h};
53 851 $context->{stats}{cached}++;
54 } else {
55 346 my $sha = Digest::SHA->new($SHA_BITS);
56 346 ▶ unless (eval { $sha->addfile($abs, 'b'); 1 }) {
Branch: 1 branch
BranchTrueFalseTotal
unless (eval { do { $sha->addfile($abs, "b"); 1 } }) 0 346 346
57 0 $context->{skipped}{$rel} = message('scan.unreadable', { error => $@ =~ s/\s+\z//r });
58 0 $context->{stats}{skipped}++;
59 0 return undef; ## no critic (Subroutines::ProhibitExplicitReturnUndef)
60 }
61 346 $hash = $sha->hexdigest;
62 346 $context->{stats}{hashed}++;
63 }
64  
65 return {
66 1197 t => $TYPE_FILE,
67 s => $size,
68 m => $mtime,
69 h => $hash,
70 mode => $stat->[$STAT_MODE] & $MODE_MASK,
71 };
72 329 };
73  
74 =encoding utf8
75  
76 =head1 NAME
77  
78 App::FileSync::Scan - describe a directory tree as a manifest
79  
80 =head1 SYNOPSIS
81  
82 my $scan = App::FileSync::Scan::scan(root => $root, ignore => $ignore, cache => $last);
83 for my $path (sort keys %{ $scan->{manifest} }) { ... }
84 printf "%d hashed, %d reused\n", @{ $scan->{stats} }{qw(hashed cached)};
85  
86 =head1 FUNCTIONS
87  
88 =head2 scan
89  
90 Purpose: walk a tree and describe every regular file and empty directory.
91  
92 Args: C<root> (required), C<ignore> (an L<App::FileSync::Ignore>), C<cache>
93 (a previous manifest whose entries are reused when size and mtime still match).
94  
95 Returns: a hashref with three hashrefs:
96  
97 manifest path => { t => 'f', s => size, m => mtime, h => sha256, mode => bits }
98 path => { t => 'd' } (only directories empty after ignores)
99 stats { files, dirs, bytes, hashed, cached, skipped }
100 skipped path => reason, for symlinks, special files and unreadable entries
101  
102 Side Effects: reads every file that is not cached. Never writes.
103  
104 Usage:
105  
106 my $scan = App::FileSync::Scan::scan(root => $master, ignore => $ignore);
107  
108 =head3 EXAMPLE
109  
110 my $scan = App::FileSync::Scan::scan(root => '/home/foo/Documents');
111 print scalar(keys %{ $scan->{manifest} }), " entries\n";
112  
113 =head3 API SPECIFICATION
114  
115 =head4 INPUT
116  
117 {
118 root => { type => 'string', min => 1 },
119 ignore => { type => 'object', can => 'match', optional => 1 },
120 cache => { type => 'hashref', optional => 1 },
121 }
122  
123 =head4 OUTPUT
124  
125 {
126 type => 'hashref',
127 schema => {
128 manifest => { type => 'hashref' },
129 stats => { type => 'hashref' },
130 skipped => { type => 'hashref' },
131 },
132 }
133  
134 =head3 MESSAGES
135  
136 Skip reasons (in C<skipped>, not raised):
137  
138 Key Meaning
139 scan.unreadable a file could not be read
140 scan.cannot_stat an entry vanished or cannot be examined
141 scan.cannot_list a directory cannot be read
142 scan.symlink symbolic links are not synced
143 scan.special devices, sockets and fifos are not synced
144  
145 Key Meaning Resolution
146 args.invalid no root was given pass root
147  
148 =head3 FORMAL SPECIFICATION
149  
150 Manifest == PATH ⇸ ENTRY
151 scan(r, ign, c) = (M, S, K) where
152 dom M = { p | p ∈ files(r) ∪ emptydirs(r) ∧ ¬ ign(p) ∧ ¬ unreadable(p) }
153 ∀ p ∈ dom M • kind p = file ⇒ M(p).h = sha256(FS(r/p))
154 dom K ∩ dom M = ∅
155  
156 =head3 PSEUDOCODE
157  
158 for each entry under root, depth first, sorted:
159 skip ignored paths (and everything below an ignored directory)
160 directory: recurse; record it only if nothing inside was recorded
161 regular file: reuse the cached hash if size and mtime match, else hash it
162 anything else: record why it was skipped
163  
164 =cut
165  
166 386 ▶sub scan {
Subroutine scan: called CC=2 SCAR=6.9
Pod documented
167 386 my $args = named(\@_, {
168 root => { type => 'string', min => 1 },
169 ignore => { type => 'object', optional => 1 },
170 cache => { type => 'hashref', optional => 1 },
171 }, 'Scan::scan');
172  
173 my $context = {
174 ignore => $args->{ignore},
175 cache => $args->{cache} || {},
176 manifest => {},
177 skipped => {},
178 386 ▶ stats => { map { $_ => 0 } qw(files dirs bytes hashed cached skipped) },
Condition100%2/2
$args->{'cache'} || {}
l!l
37016
MC/DC100%1/1
$args->{'cache'} || {}
$args->{'cache'}
Truth table100%2/2
A
A= $args->{'cache'}
Aresult
11
00
179 };
180  
181 386 _walk($context, $args->{root}, '');
182  
183 return {
184 manifest => $context->{manifest},
185 stats => $context->{stats},
186 skipped => $context->{skipped},
187 386 };
188 }
189  
190 =head2 clean_stale_temps
191  
192 Purpose: remove temporary files left behind by an interrupted run.
193  
194 Args: C<root>, optional C<max_age> in seconds (default 3600).
195  
196 Returns: the number of files removed.
197  
198 Side Effects: deletes C<.filesync-tmp-*> files older than C<max_age> anywhere
199 under C<root>. Younger ones may belong to a run in progress and are left
200 alone. A file that cannot be removed is skipped.
201  
202 Usage:
203  
204 App::FileSync::Scan::clean_stale_temps(root => $root);
205  
206 =head3 EXAMPLE
207  
208 my $removed = App::FileSync::Scan::clean_stale_temps(root => '/srv/filesync/master/docs', max_age => 600);
209  
210 =head3 API SPECIFICATION
211  
212 =head4 INPUT
213  
214 {
215 root => { type => 'string', optional => 1 },
216 max_age => { type => 'integer', min => 0, optional => 1 },
217 }
218  
219 =head4 OUTPUT
220  
221 { type => 'integer', min => 0 }
222  
223 =head3 MESSAGES
224  
225 None; failures to remove are not errors.
226  
227 =head3 FORMAL SPECIFICATION
228  
229 ΔFS
230 Stale == { p : dom FS | p under root ∧ prefix(TEMP, basename p) ∧ now − mtime p ≥ age }
231 dom FS' = dom FS \ Removable(Stale)
232 result = #(dom FS \ dom FS')
233  
234 =cut
235  
236 335 ▶sub clean_stale_temps {
Subroutine clean_stale_temps: called CC=13 SCAR=28.4
Pod documented
237 335 my $args = named(\@_, {
238 root => { type => 'string', optional => 1 },
239 max_age => { type => 'integer', min => 0, optional => 1 },
240 }, 'Scan::clean_stale_temps');
241  
242 335 my $root = $args->{root};
243 335 ▶ return 0 unless defined $root && length $root && -d $root;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $root and length $root and -d $root 0 335 335
Condition33%2/6
defined $root and length $root and -d $root
!ll&&!rl&&r
00335
defined $root and length $root
!ll&&!rl&&r
00335
MC/DC0%0/3
defined $root and length $root and -d $root
defined $root length $root -d $root
Truth table25%1/4
A and B and C
A= defined $rootB= length $rootC= -d $root
ABCresult
0XX0
10X0
1100
1111
244  
245 335 ▶ my $max_age = defined $args->{max_age} ? $args->{max_age} : $STALE_TEMP_SECONDS;
Branch: 1 branch
BranchTrueFalseTotal
defined $args->{'max_age'} ? : 174 161 335
246 335 my $now = time;
247 335 my $removed = 0;
248  
249 # Breadth first with an explicit queue: no recursion depth limit, and
250 # symlinked directories are never followed because lstat says what they are.
251 335 my @queue = ($root);
252 335 ▶ while (@queue) {
Branch: 1 branch
BranchTrueFalseTotal
if (@queue) 441 335 776
253 441 my $dir = shift @queue;
254 441 ▶ for my $name (@{ list_dir($dir) || [] }) {
Condition50%1/2
list_dir($dir) || []
l!l
4410
MC/DC0%0/1
list_dir($dir) || []
list_dir($dir)
Truth table50%1/2
A
A= list_dir($dir)
Aresult
11
00
255 1400 my $path = File::Spec->catfile($dir, $name);
256 1400 ▶ my @stat = Time::HiRes::lstat($path) or next;
Branch: 1 branch
BranchTrueFalseTotal
my(@stat) = &Time::HiRes::lstat($path) or next 1400 0 1400
257 1400 ▶ if (-d _) {
Branch: 2 branches
BranchTrueFalseTotal
if (-d _) { } 106 1294 1400
elsif ($name =~ /$TEMP_NAME/ and $now - $stat[$App::FileSync::Scan::STAT_MTIME] >= $max_age) { } 0 2588 2588
258 106 push @queue, $path;
259 ▶ } elsif ($name =~ $TEMP_NAME && $now - $stat[$STAT_MTIME] >= $max_age) {
Condition33%1/3
$name =~ /$TEMP_NAME/ and $now - $stat[$App::FileSync::Scan::STAT_MTIME] >= $max_age
!ll&&!rl&&r
129400
MC/DC0%0/2
$name =~ /$TEMP_NAME/ and $now - $stat[$App::FileSync::Scan::STAT_MTIME] >= $max_age
$name =~ /$TEMP_NAME/ $now - $stat[$App::FileSync::Scan::STAT_MTIME] >= $max_age
Truth table33%1/3
A and B
A= $name =~ /$TEMP_NAME/B= $now - $stat[$App::FileSync::Scan::STAT_MTIME] >= $max_age
ABresult
0X0
100
111
260 329 ▶ no autodie qw(unlink);
Subroutine BEGIN: called CC=1 SCAR=0.0
261 0 ▶ $removed++ if unlink $path;
Branch: 1 branch
BranchTrueFalseTotal
if unlink $path 0 0 0
262 }
263 }
264 }
265  
266 335 return $removed;
267 }
268  
269 # _walk
270 # Purpose: scan one directory, recursing into subdirectories.
271 # Entry: $context is the scan state; $abs/$rel name the directory
272 # ($rel is '' at the root).
273 # Exit: returns how many entries inside were recorded, so the caller
274 # knows whether to record this directory as empty.
275 # Side effects: fills $context->{manifest}, {skipped} and {stats}.
276 527 ▶sub _walk :Private {
Subroutine _walk: called CC=13 SCAR=25.8
277 527 my ($context, $abs, $rel) = @_;
278  
279 527 my $names = list_dir($abs);
280 527 ▶ unless ($names) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($names) 1 526 527
281 1 ▶ _skip($context, length $rel ? $rel : '.', 'scan.cannot_list', os_error());
Branch: 1 branch
BranchTrueFalseTotal
length $rel ? : 0 1 1
282 1 return 0;
283 }
284  
285 526 my $entries = 0;
286 526 for my $name (@{$names}) {
287 1531 ▶ my $child_rel = length $rel ? "$rel/$name" : $name;
Branch: 1 branch
BranchTrueFalseTotal
length $rel ? : 95 1436 1531
288 1531 ▶ next if $context->{ignore} && $context->{ignore}->match($child_rel);
Branch: 1 branch
BranchTrueFalseTotal
if $context->{"ignore"} and $context->{"ignore"}->match($child_rel) 188 1343 1531
Condition100%3/3
$context->{"ignore"} and $context->{"ignore"}->match($child_rel)
!ll&&!rl&&r
41339188
MC/DC100%2/2
$context->{"ignore"} and $context->{"ignore"}->match($child_rel)
$context->{"ignore"} $context->{"ignore"}->match($child_rel)
Truth table100%3/3
A and B
A= $context->{"ignore"}B= $context->{"ignore"}->match($child_rel)
ABresult
0X0
100
111
289  
290 1343 my $child_abs = File::Spec->catfile($abs, $name);
291 1343 my @stat = Time::HiRes::lstat($child_abs);
292 1343 ▶ if (!@stat) {
Branch: 3 branches
BranchTrueFalseTotal
if (not @stat) { } 0 1343 1343
elsif (-d _) { } 141 1202 1343
elsif (-f _) { } 1197 5 1202
293 0 _skip($context, $child_rel, 'scan.cannot_stat', os_error());
294 } elsif (-d _) {
295 # An empty directory is worth syncing; a non-empty one is implied
296 # by the files inside it.
297 141 $entries++;
298 141 ▶ if (_walk($context, $child_abs, $child_rel) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (_walk($context, $child_abs, $child_rel) == 0) 77 64 141
299 77 $context->{manifest}{$child_rel} = { t => $TYPE_DIR };
300 77 $context->{stats}{dirs}++;
301 }
302 } elsif (-f _) {
303 1197 ▶ my $entry = $file_entry->($context, $child_abs, $child_rel, \@stat) or next;
Branch: 1 branch
BranchTrueFalseTotal
my $entry = &$file_entry($context, $child_abs, $child_rel, \@stat) or next 1197 0 1197
304 1197 $context->{manifest}{$child_rel} = $entry;
305 1197 $context->{stats}{files}++;
306 1197 $context->{stats}{bytes} += $entry->{s};
307 1197 $entries++;
308 } else {
309 5 ▶ _skip($context, $child_rel, -l _ ? 'scan.symlink' : 'scan.special');
Branch: 1 branch
BranchTrueFalseTotal
-l _ ? : 5 0 5
310 }
311 }
312  
313 526 return $entries;
314 329 ▶}
Subroutine BEGIN: called CC=1 SCAR=0.0
315  
316 # _skip
317 # Purpose: record a path the manifest leaves out, and why.
318 # Entry: $key names the reason in the catalogue; $error is the OS
319 # reason where there is one.
320 # Exit: returns 1.
321 # Side effects: updates $context->{skipped} and the skipped count.
322 6 ▶sub _skip :Private {
Subroutine _skip: called CC=1 SCAR=0.0
323 6 my ($context, $rel, $key, $error) = @_;
324  
325 6 $context->{skipped}{$rel} = message($key, { error => $error });
326 6 $context->{stats}{skipped}++;
327 6 return 1;
328 329 ▶}
Subroutine BEGIN: called CC=1 SCAR=0.0
329  
330 329 1;
331  
332 __END__
« blib/lib/App/FileSync/Verify.pm ↑ Summary blib/lib/App/FileSync/Client.pm »