blib/lib/App/dupfind/Threaded/MapReduce/Weed.pm

filter: Statementstmt 81.8% 9 / 11 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 4 / 4 Podpod n/a totaltotal 86.6% 13 / 15 CC 1 SCAR 0.7
CC
1
Cov
82%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« blib/lib/App/dupfind/Threaded/MapReduce/Digest.pm ↑ Summary blib/lib/App/dupfind/Threaded/Overrides.pm »
1 # ABSTRACT: Map-reduce version of weed_dups, and the worker thread for it
2  
3 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 package App::dupfind::Threaded::MapReduce::Weed;
7 {
8 $App::dupfind::Threaded::MapReduce::Weed::VERSION = '0.140230'; # TRIAL
9 }
10  
11 1 ▶use 5.010;
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 0 ▶use threads;
Subroutine BEGIN: called CC=1 SCAR=2.6
14 use threads::shared;
15  
16 use Moo::Role;
17  
18 use Time::HiRes 'usleep';
19  
20 requires 'opts';
21  
22 sub weed_dups
23 {
24 my ( $self, $size_dups ) = @_;
25  
26 # you have to do this for this threaded version of dupfind, and it has
27 # to happen after you've already pruned out the hardlinks
28  
29 my $zero_files = delete $size_dups->{0};
30  
31 my $dup_count = $self->count_dups( $size_dups );
32  
33 my ( $map_code, $pass_count, $new_count, $diff, $len );
34  
35 $len = $self->opts->{wpsize} || 32;
36  
37 for my $planned_pass ( $self->_plan_weed_passes )
38 {
39 $pass_count++;
40  
41 $self->say_stderr( "** $dup_count POTENTIAL DUPLICATES" );
42  
43 $map_code = sub { $self->_weed_worker( $planned_pass, $len ) };
44  
45 $size_dups = $self->map_reduce( $size_dups => $map_code );
46  
47 $new_count = $self->count_dups( $size_dups );
48  
49 $diff = $dup_count - $new_count;
50  
51 $dup_count = $new_count;
52  
53 $self->say_stderr( " ...ELIMINATED $diff NON-DUPS IN PASS $pass_count" );
54 $self->say_stderr( " ...$new_count POTENTIAL DUPS REMAIN" );
55 }
56  
57 $size_dups->{0} = &shared_clone( $zero_files ) if ref $zero_files;
58  
59 return $size_dups;
60 }
61  
62 sub _weed_worker
63 {
64 my ( $self, $weeder, $len ) = @_;
65  
66 WORKER: while
67 (
68 ! $self->term_flag
69 && defined ( my $grouping = $self->work_queue->dequeue )
70 )
71 {
72 my $same_bytes = {};
73 my $weed_failed = [];
74  
75 next unless !! @$grouping; # why?
76  
77 my $file_size = -s $grouping->[0];
78  
79 GROUPING: for my $file ( @$grouping )
80 {
81 my $bytes_read = $self->$weeder( $file, $len, $file_size );
82  
83 $self->increment_counter;
84  
85 push @{ $same_bytes->{ $bytes_read } }, $file
86 if defined $bytes_read;
87  
88 push @$weed_failed, $file unless defined $bytes_read;
89 }
90  
91 # delete obvious non-dupe files from the group of same-size files
92 # by virtue of the fact that they will be a single length arrayref
93  
94 delete $same_bytes->{ $_ }
95 for grep { @{ $same_bytes->{ $_ } } == 1 }
96 keys %$same_bytes;
97  
98 # recompose the arrayref of filenames for the same-size file grouping
99 # but leave out the files we just weeded out from the group
100  
101 my @group = map { @{ $same_bytes->{ $_ } } } keys %$same_bytes;
102  
103 push @group, @$weed_failed if @$weed_failed;
104  
105 $self->push_mapped( $file_size => @group );
106 }
107 }
108  
109 1;
110  
111 __END__
« blib/lib/App/dupfind/Threaded/MapReduce/Digest.pm ↑ Summary blib/lib/App/dupfind/Threaded/Overrides.pm »