| 1 | # ABSTRACT: Public methods for the App::dupfind deduplication engine | ||
| 2 | |||
| 3 | 11 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 11 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | |||
| 6 | package App::dupfind::Common; | ||
| 7 | { | ||
| 8 | $App::dupfind::Common::VERSION = '0.140230'; # TRIAL | ||
| 9 | } | ||
| 10 | |||
| 11 | 11 | ▶ | use 5.010; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | |||
| 13 | 11 | ▶ | use Moo; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 14 | 11 | ▶ | use Digest::xxHash 'xxhash_hex'; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 15 | |||
| 16 | use lib 'lib'; | ||
| 17 | |||
| 18 | has opts => ( is => 'ro', required => 1 ); | ||
| 19 | |||
| 20 | with 'App::dupfind::Guts'; | ||
| 21 | |||
| 22 | before [ qw/ weed_dups digest_dups / ] => sub | ||
| 23 | { | ||
| 24 | require Term::ProgressBar if shift->opts->{progress} | ||
| 25 | }; | ||
| 26 | |||
| 27 | before delete_dups => sub { require Term::Prompt }; | ||
| 28 | |||
| 29 | |||
| 30 | sub count_dups | ||
| 31 | { | ||
| 32 | my ( $self, $dups ) = @_; | ||
| 33 | |||
| 34 | my $count = 0; | ||
| 35 | |||
| 36 | $count += @$_ for map { $dups->{ $_ } } keys %$dups; | ||
| 37 | |||
| 38 | return $count; | ||
| 39 | } | ||
| 40 | |||
| 41 | sub get_size_dups | ||
| 42 | { | ||
| 43 | my $self = shift; | ||
| 44 | |||
| 45 | my ( $size_dups, $scan_count ) = ( {}, 0 ); | ||
| 46 | |||
| 47 | $self->ftl->list_dir | ||
| 48 | ( | ||
| 49 | $self->opts->{dir} => | ||
| 50 | { | ||
| 51 | recurse => 1, | ||
| 52 | callback => sub | ||
| 53 | { | ||
| 54 | ## my ( $selfdir, $subdirs, $files ) = @_; | ||
| 55 | |||
| 56 | my $files = $_[2]; # save two vars | ||
| 57 | |||
| 58 | $scan_count += @$files; | ||
| 59 | |||
| 60 | push @{ $size_dups->{ -s $_ } }, $_ | ||
| 61 | for grep { !-l $_ && defined -s $_ } @$files; | ||
| 62 | } | ||
| 63 | } | ||
| 64 | ); | ||
| 65 | |||
| 66 | delete $size_dups->{ $_ } | ||
| 67 | for grep { @{ $size_dups->{ $_ } } == 1 } | ||
| 68 | keys %$size_dups; | ||
| 69 | |||
| 70 | return $size_dups, $scan_count, $self->count_dups( $size_dups ); | ||
| 71 | } | ||
| 72 | |||
| 73 | sub toss_out_hardlinks | ||
| 74 | { | ||
| 75 | my ( $self, $size_dups ) = @_; | ||
| 76 | |||
| 77 | for my $size ( keys %$size_dups ) | ||
| 78 | { | ||
| 79 | my $group = $size_dups->{ $size }; | ||
| 80 | my %dev_inodes; | ||
| 81 | |||
| 82 | # this will automatically throw out hardlinks, with the only surviving | ||
| 83 | # file being the first asciibetically-sorted entry | ||
| 84 | $dev_inodes{ join '', ( stat $_ )[0,1] } = $_ for reverse sort @$group; | ||
| 85 | |||
| 86 | if ( scalar keys %dev_inodes == 1 ) | ||
| 87 | { | ||
| 88 | delete $size_dups->{ $size }; | ||
| 89 | } | ||
| 90 | else | ||
| 91 | { | ||
| 92 | $size_dups->{ $size } = [ values %dev_inodes ]; | ||
| 93 | } | ||
| 94 | } | ||
| 95 | |||
| 96 | return $size_dups; | ||
| 97 | } | ||
| 98 | |||
| 99 | sub weed_dups | ||
| 100 | { | ||
| 101 | my ( $self, $size_dups ) = @_; | ||
| 102 | |||
| 103 | my $zero_sized = delete $size_dups->{0}; | ||
| 104 | |||
| 105 | my $pass_count = 0; | ||
| 106 | |||
| 107 | $self->_do_weed_pass( $size_dups => $_ => ++$pass_count ) | ||
| 108 | for $self->_plan_weed_passes; | ||
| 109 | |||
| 110 | $size_dups->{0} = $zero_sized if ref $zero_sized; | ||
| 111 | |||
| 112 | return $size_dups; | ||
| 113 | } | ||
| 114 | |||
| 115 | sub digest_dups | ||
| 116 | { | ||
| 117 | my ( $self, $size_dups ) = @_; | ||
| 118 | |||
| 119 | my ( $digests, $progress, $i ) = ( {}, undef, 0 ); | ||
| 120 | |||
| 121 | my $digest_cache = {}; | ||
| 122 | my $cache_stop = $self->opts->{cachestop}; | ||
| 123 | my $max_cache = $self->opts->{cachesize}; | ||
| 124 | my $ram_caching = !! $self->opts->{ramcache}; | ||
| 125 | my $cache_size = 0; | ||
| 126 | my $cache_hits = 0; | ||
| 127 | my $cache_misses = 0; | ||
| 128 | |||
| 129 | # don't bother to hash zero-size files | ||
| 130 | $digests->{ xxhash_hex '', 0 } = delete $size_dups->{0} | ||
| 131 | if exists $size_dups->{0}; | ||
| 132 | |||
| 133 | if ( $self->opts->{progress} ) | ||
| 134 | { | ||
| 135 | my $dup_count = $self->count_dups( $size_dups ); | ||
| 136 | |||
| 137 | $progress = Term::ProgressBar->new | ||
| 138 | ( | ||
| 139 | { | ||
| 140 | name => ' ...PROGRESS', | ||
| 141 | count => $dup_count, | ||
| 142 | remove => 1, | ||
| 143 | } | ||
| 144 | ); | ||
| 145 | } | ||
| 146 | |||
| 147 | local $/; | ||
| 148 | |||
| 149 | SIZES: for my $size ( keys %$size_dups ) | ||
| 150 | { | ||
| 151 | my $group = $size_dups->{ $size }; | ||
| 152 | |||
| 153 | GROUPING: for my $file ( @$group ) | ||
| 154 | { | ||
| 155 | my $digest; | ||
| 156 | |||
| 157 | open my $fh, '<', $file or next; | ||
| 158 | |||
| 159 | my $data = <$fh>; | ||
| 160 | |||
| 161 | close $fh; | ||
| 162 | |||
| 163 | if ( $ram_caching ) | ||
| 164 | { | ||
| 165 | if ( $digest = $digest_cache->{ $data } ) | ||
| 166 | { | ||
| 167 | $cache_hits++; | ||
| 168 | } | ||
| 169 | else | ||
| 170 | { | ||
| 171 | if ( $cache_size < $max_cache && $size <= $cache_stop ) | ||
| 172 | { | ||
| 173 | $digest_cache->{ $data } = $digest = xxhash_hex $data, 0; | ||
| 174 | |||
| 175 | $cache_size++; | ||
| 176 | |||
| 177 | $cache_misses++; | ||
| 178 | } | ||
| 179 | else | ||
| 180 | { | ||
| 181 | $digest = xxhash_hex $data, 0; | ||
| 182 | } | ||
| 183 | } | ||
| 184 | } | ||
| 185 | else | ||
| 186 | { | ||
| 187 | $digest = xxhash_hex $data, 0; | ||
| 188 | } | ||
| 189 | |||
| 190 | push @{ $digests->{ $digest } }, $file; | ||
| 191 | |||
| 192 | $progress->update( ++$i ) if $progress; | ||
| 193 | } | ||
| 194 | |||
| 195 | $digest_cache = {}; # it's only worthwhile per-size-grouping | ||
| 196 | $cache_size = 0; | ||
| 197 | } | ||
| 198 | |||
| 199 | delete $digests->{ $_ } | ||
| 200 | for grep { @{ $digests->{ $_ } } == 1 } | ||
| 201 | keys %$digests; | ||
| 202 | |||
| 203 | $self->stats->{cache_hits} = $cache_hits; | ||
| 204 | $self->stats->{cache_misses} = $cache_misses; | ||
| 205 | |||
| 206 | return $digests; | ||
| 207 | } | ||
| 208 | |||
| 209 | sub sort_dups | ||
| 210 | { | ||
| 211 | my ( $self, $dups ) = @_; | ||
| 212 | |||
| 213 | # sort dup groupings | ||
| 214 | for my $identifier ( keys %$dups ) | ||
| 215 | { | ||
| 216 | my @group = @{ $dups->{ $identifier } }; | ||
| 217 | |||
| 218 | $dups->{ $identifier } = [ sort { $a cmp $b } @group ]; | ||
| 219 | } | ||
| 220 | |||
| 221 | return $dups; | ||
| 222 | } | ||
| 223 | |||
| 224 | sub show_dups # also calls $self->sort_dups before displaying output | ||
| 225 | { | ||
| 226 | my ( $self, $digests ) = @_; | ||
| 227 | my $dupes = 0; | ||
| 228 | |||
| 229 | $digests = $self->sort_dups( $digests ); | ||
| 230 | |||
| 231 | my $for_humans = sub # human-readable output | ||
| 232 | { | ||
| 233 | my ( $digest, $files ) = @_; | ||
| 234 | |||
| 235 | say sprintf 'DUPLICATES (digest: %s | size: %db)', $digest, -s $$files[0]; | ||
| 236 | |||
| 237 | say " $_" for @$files; | ||
| 238 | |||
| 239 | say ''; | ||
| 240 | }; | ||
| 241 | |||
| 242 | my $for_robots = sub # machine parseable output | ||
| 243 | { | ||
| 244 | my $files = pop; | ||
| 245 | |||
| 246 | say join "\t", @$files | ||
| 247 | }; | ||
| 248 | |||
| 249 | my $formatter = $self->opts->{format} eq 'human' ? $for_humans : $for_robots; | ||
| 250 | |||
| 251 | for my $digest | ||
| 252 | ( | ||
| 253 | sort { $digests->{ $a }->[0] cmp $digests->{ $b }->[0] } keys %$digests | ||
| 254 | ) | ||
| 255 | { | ||
| 256 | my $files = $digests->{ $digest }; | ||
| 257 | |||
| 258 | $formatter->( $digest => $files ); | ||
| 259 | |||
| 260 | $dupes += @$files - 1; | ||
| 261 | } | ||
| 262 | |||
| 263 | return $dupes | ||
| 264 | } | ||
| 265 | |||
| 266 | sub delete_dups | ||
| 267 | { | ||
| 268 | my ( $self, $digests ) = @_; | ||
| 269 | |||
| 270 | my $removed = 0; | ||
| 271 | |||
| 272 | for my $digest ( keys %$digests ) | ||
| 273 | { | ||
| 274 | my $group = $digests->{ $digest }; | ||
| 275 | |||
| 276 | say sprintf 'ORIGINAL (%s) %s', $digest, $group->[0]; | ||
| 277 | |||
| 278 | shift @$group; | ||
| 279 | |||
| 280 | for my $dup ( @$group ) | ||
| 281 | { | ||
| 282 | if ( $self->opts->{prompt} ) | ||
| 283 | { | ||
| 284 | unless ( Term::Prompt::prompt( 'y', "REMOVE DUPE? $dup", '', 'n' ) ) | ||
| 285 | { | ||
| 286 | say sprintf 'KEPT (%s) %s', $digest, $dup; | ||
| 287 | |||
| 288 | next; | ||
| 289 | } | ||
| 290 | } | ||
| 291 | |||
| 292 | unlink $dup or warn "COULD NOT REMOVE $dup! $!" and next; | ||
| 293 | |||
| 294 | $removed++; | ||
| 295 | |||
| 296 | say sprintf 'REMOVED (%s) %s', $digest, $dup; | ||
| 297 | } | ||
| 298 | |||
| 299 | say '--'; | ||
| 300 | } | ||
| 301 | |||
| 302 | say "** TOTAL DUPLICATE FILES REMOVED: $removed"; | ||
| 303 | } | ||
| 304 | |||
| 305 | sub cache_stats | ||
| 306 | { | ||
| 307 | my $self = shift; | ||
| 308 | |||
| 309 | return $self->stats->{cache_hits}, | ||
| 310 | $self->stats->{cache_misses} | ||
| 311 | } | ||
| 312 | |||
| 313 | sub say_stderr { return if shift->opts->{quiet}; warn "$_\n" for @_ }; | ||
| 314 | |||
| 315 | __PACKAGE__->meta->make_immutable; | ||
| 316 | |||
| 317 | 1; | ||
| 318 | |||
| 319 | __END__ | ||