blib/lib/App/dupfind/Threaded/MapReduce/Digest.pm

filter: Statementstmt 81.8% 9 / 11 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 4 / 4 Podpod n/a totaltotal 86.6% 13 / 15 CC 1 SCAR 0.7
CC
1
Cov
82%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« blib/lib/App/dupfind/Threaded/MapReduce.pm ↑ Summary blib/lib/App/dupfind/Threaded/MapReduce/Weed.pm »
1 # ABSTRACT: Map-reduce version of digest_dups method and worker thread for it
2  
3 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 package App::dupfind::Threaded::MapReduce::Digest;
7 {
8 $App::dupfind::Threaded::MapReduce::Digest::VERSION = '0.140230'; # TRIAL
9 }
10  
11 1 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 0 ▶use threads;
Subroutine BEGIN: called CC=1 SCAR=2.6
14 use threads::shared;
15  
16 use Moo::Role;
17  
18 use Time::HiRes 'usleep';
19 use Digest::xxHash 'xxhash_hex';
20  
21 requires 'opts';
22  
23 my $read_semaphore :shared;
24  
25 sub digest_dups
26 {
27 my ( $self, $size_dups ) = @_;
28  
29 # you have to do this for this threaded version of dupfind, and it has
30 # to happen after you've already pruned out the hardlinks
31  
32 # don't bother to hash zero-size files
33  
34 my $zero_digest = xxhash_hex '', 0;
35  
36 my $zero_files = delete $size_dups->{0};
37  
38 my $map_code = sub { $self->_digest_worker( @_ ) };
39  
40 my $reduced = $self->map_reduce( $size_dups => $map_code );
41  
42 $reduced->{ $zero_digest } = $zero_files if ref $zero_files;
43  
44 return $reduced;
45 }
46  
47 # !! The code to be mapped MUST increment $self->counter for each item it sees
48 sub _digest_worker
49 {
50 my $self = shift;
51  
52 my $digest_cache = {};
53 my $cache_stop = $self->opts->{cachestop};
54 my $max_cache = $self->opts->{cachesize};
55 my $ram_caching = !! $self->opts->{ramcache};
56 my $cache_size = 0;
57 my $cache_hits = 0;
58 my $cache_misses = 0;
59  
60 local $/;
61  
62 WORKER: while
63 (
64 ! $self->term_flag
65 && defined ( my $grouping = $self->work_queue->dequeue )
66 )
67 {
68 my $data;
69  
70 GROUPING: for my $file ( @$grouping )
71 {
72 my $size = -s $grouping->[0];
73 my $digest;
74  
75 open my $fh, '<', $file or do
76 {
77 $self->increment_counter;
78  
79 next GROUPING;
80 };
81  
82 READ_LOCK:
83 {
84 lock $read_semaphore;
85  
86 $data = <$fh>;
87 }
88  
89 close $fh;
90  
91 if ( $ram_caching )
92 {
93 if ( $digest = $digest_cache->{ $data } )
94 {
95 $cache_hits++;
96 }
97 else
98 {
99 if ( $cache_size < $max_cache && $size <= $cache_stop )
100 {
101 $digest_cache->{ $data } = $digest = xxhash_hex $data, 0;
102  
103 $cache_size++;
104  
105 $cache_misses++;
106 }
107 else
108 {
109 $digest = xxhash_hex $data, 0;
110 }
111 }
112 }
113 else
114 {
115 $digest = xxhash_hex $data, 0;
116 }
117  
118 $self->push_mapped( $digest => $file );
119  
120 $self->increment_counter;
121 }
122  
123 $digest_cache = {}; # it's only worthwhile per-size-grouping
124 $cache_size = 0;
125  
126 $self->add_stats( cache_hits => $cache_hits );
127 $self->add_stats( cache_misses => $cache_misses );
128 }
129 }
130  
131 1;
132  
133 __END__
« blib/lib/App/dupfind/Threaded/MapReduce.pm ↑ Summary blib/lib/App/dupfind/Threaded/MapReduce/Weed.pm »