« blib/lib/App/dupfind/Threaded/MapReduce/Digest.pm
↑ Summary
blib/lib/App/dupfind/Threaded/Overrides.pm »
| 1 | # ABSTRACT: Map-reduce version of weed_dups, and the worker thread for it | ||
| 2 | |||
| 3 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | |||
| 6 | package App::dupfind::Threaded::MapReduce::Weed; | ||
| 7 | { | ||
| 8 | $App::dupfind::Threaded::MapReduce::Weed::VERSION = '0.140230'; # TRIAL | ||
| 9 | } | ||
| 10 | |||
| 11 | 1 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | |||
| 13 | 0 | ▶ | use threads; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 14 | use threads::shared; | ||
| 15 | |||
| 16 | use Moo::Role; | ||
| 17 | |||
| 18 | use Time::HiRes 'usleep'; | ||
| 19 | |||
| 20 | requires 'opts'; | ||
| 21 | |||
| 22 | sub weed_dups | ||
| 23 | { | ||
| 24 | my ( $self, $size_dups ) = @_; | ||
| 25 | |||
| 26 | # you have to do this for this threaded version of dupfind, and it has | ||
| 27 | # to happen after you've already pruned out the hardlinks | ||
| 28 | |||
| 29 | my $zero_files = delete $size_dups->{0}; | ||
| 30 | |||
| 31 | my $dup_count = $self->count_dups( $size_dups ); | ||
| 32 | |||
| 33 | my ( $map_code, $pass_count, $new_count, $diff, $len ); | ||
| 34 | |||
| 35 | $len = $self->opts->{wpsize} || 32; | ||
| 36 | |||
| 37 | for my $planned_pass ( $self->_plan_weed_passes ) | ||
| 38 | { | ||
| 39 | $pass_count++; | ||
| 40 | |||
| 41 | $self->say_stderr( "** $dup_count POTENTIAL DUPLICATES" ); | ||
| 42 | |||
| 43 | $map_code = sub { $self->_weed_worker( $planned_pass, $len ) }; | ||
| 44 | |||
| 45 | $size_dups = $self->map_reduce( $size_dups => $map_code ); | ||
| 46 | |||
| 47 | $new_count = $self->count_dups( $size_dups ); | ||
| 48 | |||
| 49 | $diff = $dup_count - $new_count; | ||
| 50 | |||
| 51 | $dup_count = $new_count; | ||
| 52 | |||
| 53 | $self->say_stderr( " ...ELIMINATED $diff NON-DUPS IN PASS $pass_count" ); | ||
| 54 | $self->say_stderr( " ...$new_count POTENTIAL DUPS REMAIN" ); | ||
| 55 | } | ||
| 56 | |||
| 57 | $size_dups->{0} = &shared_clone( $zero_files ) if ref $zero_files; | ||
| 58 | |||
| 59 | return $size_dups; | ||
| 60 | } | ||
| 61 | |||
| 62 | sub _weed_worker | ||
| 63 | { | ||
| 64 | my ( $self, $weeder, $len ) = @_; | ||
| 65 | |||
| 66 | WORKER: while | ||
| 67 | ( | ||
| 68 | ! $self->term_flag | ||
| 69 | && defined ( my $grouping = $self->work_queue->dequeue ) | ||
| 70 | ) | ||
| 71 | { | ||
| 72 | my $same_bytes = {}; | ||
| 73 | my $weed_failed = []; | ||
| 74 | |||
| 75 | next unless !! @$grouping; # why? | ||
| 76 | |||
| 77 | my $file_size = -s $grouping->[0]; | ||
| 78 | |||
| 79 | GROUPING: for my $file ( @$grouping ) | ||
| 80 | { | ||
| 81 | my $bytes_read = $self->$weeder( $file, $len, $file_size ); | ||
| 82 | |||
| 83 | $self->increment_counter; | ||
| 84 | |||
| 85 | push @{ $same_bytes->{ $bytes_read } }, $file | ||
| 86 | if defined $bytes_read; | ||
| 87 | |||
| 88 | push @$weed_failed, $file unless defined $bytes_read; | ||
| 89 | } | ||
| 90 | |||
| 91 | # delete obvious non-dupe files from the group of same-size files | ||
| 92 | # by virtue of the fact that they will be a single length arrayref | ||
| 93 | |||
| 94 | delete $same_bytes->{ $_ } | ||
| 95 | for grep { @{ $same_bytes->{ $_ } } == 1 } | ||
| 96 | keys %$same_bytes; | ||
| 97 | |||
| 98 | # recompose the arrayref of filenames for the same-size file grouping | ||
| 99 | # but leave out the files we just weeded out from the group | ||
| 100 | |||
| 101 | my @group = map { @{ $same_bytes->{ $_ } } } keys %$same_bytes; | ||
| 102 | |||
| 103 | push @group, @$weed_failed if @$weed_failed; | ||
| 104 | |||
| 105 | $self->push_mapped( $file_size => @group ); | ||
| 106 | } | ||
| 107 | } | ||
| 108 | |||
| 109 | 1; | ||
| 110 | |||
| 111 | __END__ | ||