« lib/App/dupfind/Threaded/ThreadManagement.pm
↑ Summary
blib/lib/App/dupfind/Threaded/MapReduce/Digest.pm »
| 1 | # ABSTRACT: Basic, abstracted implementation of map-reduce for threaded tasks | ||
| 2 | |||
| 3 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | |||
| 6 | package App::dupfind::Threaded::MapReduce; | ||
| 7 | { | ||
| 8 | $App::dupfind::Threaded::MapReduce::VERSION = '0.140230'; # TRIAL | ||
| 9 | } | ||
| 10 | |||
| 11 | 1 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | |||
| 13 | 0 | ▶ | use threads; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 14 | use threads::shared; | ||
| 15 | |||
| 16 | use Moo; | ||
| 17 | use Time::HiRes 'usleep'; | ||
| 18 | |||
| 19 | use lib 'lib'; | ||
| 20 | |||
| 21 | extends 'App::dupfind::Threaded::ThreadManagement'; | ||
| 22 | |||
| 23 | sub map_reduce | ||
| 24 | { | ||
| 25 | my $self = shift; | ||
| 26 | |||
| 27 | $self->reset_all; | ||
| 28 | |||
| 29 | $self->mapper( @_ ); | ||
| 30 | |||
| 31 | return $self->reducer; | ||
| 32 | } | ||
| 33 | |||
| 34 | sub mapper | ||
| 35 | { | ||
| 36 | my ( $self, $size_dups, $map ) = @_; | ||
| 37 | my $dup_count = 0; | ||
| 38 | |||
| 39 | $dup_count = $self->count_dups( $size_dups ); | ||
| 40 | |||
| 41 | # creates thread pool, passing in as an argument the number of files | ||
| 42 | # that the pool needs to digest. this is NOT equivalent to the number | ||
| 43 | # of threads to be created; that is determined in the options ($opts) | ||
| 44 | |||
| 45 | $self->create_thread_pool( $map => $dup_count ); | ||
| 46 | |||
| 47 | for my $size ( keys %$size_dups ) | ||
| 48 | { | ||
| 49 | my $group = $size_dups->{ $size }; | ||
| 50 | |||
| 51 | $self->work_queue->enqueue( $group ) if !$self->term_flag; | ||
| 52 | } | ||
| 53 | |||
| 54 | # wait for threads to finish; depends on the coderef in $map | ||
| 55 | # doing its job of updating the counter for ever item processed! | ||
| 56 | |||
| 57 | while ( $self->counter < $dup_count ) | ||
| 58 | { | ||
| 59 | usleep 1000; # sleep for 1 millisecond | ||
| 60 | } | ||
| 61 | |||
| 62 | # ...tell the threads to exit | ||
| 63 | $self->end_wait_thread_pool; | ||
| 64 | |||
| 65 | return; | ||
| 66 | } | ||
| 67 | |||
| 68 | sub reducer | ||
| 69 | { | ||
| 70 | my $self = shift @_; | ||
| 71 | |||
| 72 | # get rid of non-dupes | ||
| 73 | $self->delete_mapped( $_ ) | ||
| 74 | for grep { @{ $self->mapped->{ $_ } } == 1 } | ||
| 75 | keys %{ $self->mapped }; | ||
| 76 | |||
| 77 | return $self->mapped; | ||
| 78 | } | ||
| 79 | |||
| 80 | __PACKAGE__->meta->make_immutable; | ||
| 81 | |||
| 82 | 1; | ||
| 83 | |||
| 84 | __END__ | ||