lib/App/dupfind/App.pm

filter: Statementstmt 91.6% 22 / 24 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 8 / 8 Podpod n/a totaltotal 93.7% 30 / 32 CC 1 SCAR 0.4
CC
1
Cov
92%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« blib/lib/App/dupfind/Threaded/Overrides.pm ↑ Summary lib/App/dupfind.pm »
1 # ABSTRACT: This is the application that gets run() by $bin/dupfind
2  
3 7 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 7 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 package App::dupfind::App;
7 {
8 $App::dupfind::App::VERSION = '0.140230'; # TRIAL
9 }
10  
11 7 ▶BEGIN { select STDERR; $|++; select STDOUT; $|++; }
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 7 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
14  
15 7 ▶use Getopt::Long;
Subroutine BEGIN: called CC=1 SCAR=0.0
16 7 ▶use Benchmark ':hireswallclock';
Subroutine BEGIN: called CC=1 SCAR=0.0
17  
18 7 ▶use lib 'lib';
Subroutine BEGIN: called CC=1 SCAR=0.0
19  
20 0 ▶use App::dupfind;
Subroutine BEGIN: called CC=1 SCAR=2.6
21 use App::dupfind::Threaded;
22  
23 use Moo;
24  
25 exit run() unless caller;
26  
27 has opts =>
28 (
29 is => 'rw',
30 required => 1,
31 builder => '_build_opts'
32 );
33  
34 has metrics =>
35 (
36 is => 'rw',
37 required => 1,
38 builder => sub
39 {
40 {
41 scan_count => 0,
42 size_dup_count => 0,
43 real_dup_count => 0,
44 }
45 }
46 );
47  
48 has benchmarks =>
49 (
50 is => 'rw',
51 required => 1,
52 builder => sub
53 {
54 {
55 scanfs => { },
56 prune => { },
57 weed => { },
58 digest => { },
59 remove => { },
60 run => { },
61 }
62 }
63 );
64  
65 has deduper =>
66 (
67 is => 'ro',
68 required => 1,
69 lazy => 1,
70 builder => sub
71 {
72 my $self = shift;
73 my $ddclass = $self->opts->{threads}
74 ? 'App::dupfind::Threaded'
75 : 'App::dupfind';
76  
77 $ddclass->new( opts => $self->opts );
78 }
79 );
80  
81  
82 sub BUILD
83 {
84 my $self = shift;
85 my $opts = $self->opts;
86  
87 $opts->{threads} //= 0;
88  
89 $opts->{progress} = 1 if $opts->{verbose};
90  
91 $opts->{remove} = 1 if $opts->{prompt};
92  
93 $opts->{weed} = 0 if $opts->{weed} =~ /^(n|no)$/i;
94  
95 $opts->{weed} = 1 if !! @{ $opts->{wpass} };
96  
97 $opts->{wpass} = [ 'first_middle_last' ]
98 if $opts->{weed} && ! @{ $opts->{wpass} };
99  
100 die 'Bogus cache parameters provided! Just use the defaults buddy.'
101 if $opts->{cachestop} < 0 || $opts->{ramcache} < 0;
102  
103 die 'Invalid cachestop size; must be a small fraction of total ramcache.'
104 if $opts->{ramcache} && $opts->{cachestop} > ( $opts->{ramcache} / 100 );
105  
106 $opts->{cachesize} = int ( $opts->{ramcache} / $opts->{cachestop} );
107 }
108  
109 sub _build_opts
110 {
111 my $self = shift;
112  
113 my $opts =
114 {
115 bytes => 1024 ** 3, # 1 GB max read
116 cachestop => ( 1024 ** 2 ) * 2, # 2 MB file size limit on cache candidacy
117 dir => undef,
118 format => 'human', # options are "human" or "robot"
119 help => undef,
120 links => 0,
121 maxdepth => 50,
122 progress => 0,
123 prompt => 0,
124 quiet => 0,
125 ramcache => ( 1024 ** 2 ) * 300, # 300 MB default cache size
126 remove => 0,
127 threads => 0,
128 verbose => 0,
129 weed => 1,
130 wpass => [ ],
131 wpsize => 32,
132 };
133  
134 GetOptions
135 (
136 'bytes|b=i' => \$opts->{bytes},
137 'cachestop=i' => \$opts->{cachestop},
138 'dir|d=s' => \$opts->{dir},
139 'format|f=s' => \$opts->{format},
140 'help|h|?' => \$opts->{help},
141 'links|l' => \$opts->{links},
142 'maxdepth|m=i' => \$opts->{maxdepth},
143 'progress' => \$opts->{progress},
144 'prompt|p' => \$opts->{prompt},
145 'quiet|q' => \$opts->{quiet},
146 'ramcache=i' => \$opts->{ramcache},
147 'remove|x' => \$opts->{remove},
148 'threads|t=i' => \$opts->{threads},
149 'verbose|v' => \$opts->{verbose},
150 'weedout|w=s' => \$opts->{weed},
151 'wpass=s' => $opts->{wpass},
152 'wpsize=i' => \$opts->{wpsize},
153 ) or exit _usage();
154  
155 exit _usage() unless defined $opts->{dir};
156  
157 return $opts;
158 }
159  
160 sub _usage
161 {
162 # This is just the help message:
163  
164 require Pod::Usage;
165  
166 Pod::Usage::pod2usage( { -exitval => 1, -verbose => 2 } )
167 }
168  
169 sub _bench_this
170 {
171 my ( $self, $mark, $event ) = @_;
172  
173 $self->benchmarks->{ $mark }->{ $event } = Benchmark->new();
174 }
175  
176 sub _calculate_bench_times
177 {
178 my $self = shift;
179 my $benchmarks = $self->benchmarks;
180  
181 for my $mark ( keys %$benchmarks )
182 {
183 next unless $benchmarks->{ $mark }->{start};
184  
185 $self->benchmarks->{ $mark }->{result} =
186 timestr timediff
187 (
188 $benchmarks->{ $mark }->{end},
189 $benchmarks->{ $mark }->{start}
190 );
191 }
192  
193 $self->benchmarks->{weed}->{result} ||= 'did not weed';
194 $self->benchmarks->{remove}->{result} ||= 'no deletions';
195 }
196  
197 sub _run_summary
198 {
199 my $self = shift;
200 my $opts = $self->opts;
201 my $benchmarks = $self->benchmarks;
202 my $metrics = $self->metrics;
203  
204 my ( $cache_hits, $cache_misses ) = $self->deduper->cache_stats;
205  
206 $self->_stderr( <<__SUMMARY__ );
207 ------------------------------
208 ** THREADS...............$opts->{threads}
209 ** RAM CACHE.............$opts->{ramcache} bytes
210 ** CACHE HITS/MISSES.....$cache_hits/$cache_misses
211 ** TOTAL FILES SCANNED...$metrics->{scan_count}
212 ** TOTAL SAME SIZE.......$metrics->{size_dup_count}
213 ** TOTAL ACTUAL DUPES....$metrics->{real_dup_count}
214 -- TIMES --
215 ** TREE SCAN TIME........$benchmarks->{scanfs}->{result}
216 ** HARDLINK PRUNE TIME...$benchmarks->{prune}->{result}
217 ** WEED-OUT TIME.........$benchmarks->{weed}->{result}
218 ** CRYPTO-HASHING TIME...$benchmarks->{digest}->{result}
219 ** DELETION TIME.........$benchmarks->{remove}->{result}
220 ** TOTAL RUN TIME........$benchmarks->{run}->{result}
221 __SUMMARY__
222 }
223  
224 sub run
225 {
226 my $self = __PACKAGE__->new();
227  
228 $self->_bench_this( run => 'start' );
229  
230 my( $size_dups, $pruned_dups, $weeded_dups, $digest_dups );
231  
232 $size_dups = $self->scanfs;
233  
234 $pruned_dups = $self->prune( $size_dups );
235  
236 $weeded_dups = $self->weed( $pruned_dups ) if $self->opts->{weed};
237  
238 $digest_dups = $self->digest( $weeded_dups // $pruned_dups );
239  
240 $self->metrics->{real_dup_count} = $self->deduper->count_dups( $digest_dups );
241  
242 $self->_stderr( '** DISPLAYING OUTPUT', '-' x 30 );
243  
244 $self->deduper->show_dups( $digest_dups );
245  
246 $self->remove( $digest_dups ) if $self->opts->{remove};
247  
248 $self->_bench_this( run => 'end' );
249  
250 $self->_calculate_bench_times;
251  
252 $self->_run_summary;
253 }
254  
255 sub scanfs
256 {
257 my $self = shift;
258  
259 $self->_stderr( '** SCANNING ALL FILES FOR SIZE DUPLICATES' );
260  
261 $self->_bench_this( scanfs => 'start' );
262  
263 my ( $size_dups, $scan_ct, $size_dup_ct ) = $self->deduper->get_size_dups();
264  
265 $self->metrics->{scan_count} = $scan_ct;
266  
267 $self->metrics->{size_dup_count} = $size_dup_ct;
268  
269 $self->_bench_this( scanfs => 'end' );
270  
271 say '** NO DUPLICATES FOUND' and exit unless keys %$size_dups;
272  
273 return $size_dups;
274 }
275  
276 sub prune
277 {
278 my ( $self, $size_dups ) = @_;
279  
280 $self->_stderr( '** PRUNING HARD LINKS' );
281  
282 $self->_bench_this( prune => 'start' );
283  
284 $size_dups = $self->deduper->toss_out_hardlinks( $size_dups );
285  
286 $self->_bench_this( prune => 'end' );
287  
288 say '** NO DUPLICATES FOUND' and exit unless keys %$size_dups;
289  
290 return $size_dups;
291 }
292  
293 sub weed
294 {
295 my ( $self, $size_dups ) = @_;
296  
297 $self->_stderr( '** WEEDING-OUT FILES THAT ARE OBVIOUSLY DIFFERENT' );
298  
299 $self->_bench_this( weed => 'start' );
300  
301 my $weeded_dups = $self->deduper->weed_dups( $size_dups );
302  
303 $self->_bench_this( weed => 'end' );
304  
305 say '** NO DUPLICATES FOUND' and exit unless keys %$weeded_dups;
306  
307 return $weeded_dups;
308 }
309  
310 sub digest
311 {
312 my ( $self, $size_dups ) = @_;
313  
314 $self->_stderr( '** CHECKSUMMING SIZE DUPLICATES' );
315  
316 $self->_bench_this( digest => 'start' );
317  
318 my $digest_dups = $self->deduper->digest_dups( $size_dups );
319  
320 $self->_bench_this( digest => 'end' );
321  
322 say '** NO DUPLICATES FOUND' and exit unless keys %$digest_dups;
323  
324 return $digest_dups;
325 }
326  
327 sub remove
328 {
329 my ( $self, $digests ) = @_;
330  
331 $self->_bench_this( remove => 'start' );
332  
333 $self->deduper->delete_dups( $digests );
334  
335 $self->_bench_this( remove => 'end' );
336 }
337  
338 sub _stderr { return if shift->opts->{quiet}; warn "$_\n" for @_ };
339  
340 __PACKAGE__->meta->make_immutable;
341  
342 1;
343  
344 __END__
« blib/lib/App/dupfind/Threaded/Overrides.pm ↑ Summary lib/App/dupfind.pm »