File Coverage

blib/lib/Text/Identify/BoilerPlate.pm

Criterion	Covered	Total	%
statement	15	100	15.0
branch	0	42	0.0
condition			n/a
subroutine	5	9	55.5
pod	1	1	100.0
total	21	152	13.8

package Text::Identify::BoilerPlate;

use warnings;
use strict;
use Carp;
use base qw{ Exporter };
use Digest::MD5 qw{ md5 };

our $VERSION   = '0.3.1';
our @EXPORT_OK = qw{ rem_boilerplate };

my %config;
my %dupl;

my $LOG_FILE;

sub rem_boilerplate {

    my ( $files, $arg_ref ) = @_;

    _get_config($arg_ref);

    my @files = @$files;

    print "MIN_DUPL: $config{'min_dupl'}\n";

    if ($config{'min_dupl'} =~ s/\s*\%//) {
        my $num_of_files = @files;
        $config{'min_dupl'} =  int (
                                    ($num_of_files * $config{'min_dupl'})
                                    / 100
                                   );
        if ($config{'min_dupl'} < 2) {
            $config{'min_dupl'}=2;
        }
    }

    print "MIN_DUPL: $config{'min_dupl'}\n";


    foreach my $file (@files) {

        my @lines;
        open my $INPUT_FILE, '<', $file
            or croak "Can't open $file";
        while ( my $line = <$INPUT_FILE> ) {

            $line =~ s/^\s*//g;
            $line =~ s/\s*$//g;
            $line =~ s/\t/ /g;
            $line =~ s/ +/ /g;
            $line =~ s/\r//g;

            my $line_tmp = $line;
            if ( $config{'ignore_digits'} ) {
                $line_tmp =~ s/\d+/_D_/g;
            }
            if ( $config{'digest'} ) {
                $line_tmp = md5($line_tmp);
            }

            $dupl{$line_tmp}++;
            push @lines, $line;

        }
        $file = [ $file, \@lines ];    # name + content
    }

    # find duplicated lines
    foreach my $file (@files) {

        if ( $config{'log'} ) {
            print {$LOG_FILE} "\n-- $file->[0] --\n";
        }

        my $lines = $file->[1];
        my @lines = @$lines;

        if ( $config{'only_headers_and_footers'} ) {
            @lines = _rem_first_duplicates(@lines);
            @lines = reverse @lines;
            @lines = _rem_first_duplicates(@lines);
            @lines = reverse @lines;
        }
        else {
            @lines = _rem_duplicates(@lines);
        }

        my $filename     = $file->[0];
        my $new_filename = $filename . "." . $config{'suffix'};
        open my $OUTPUT_FILE, '>', $new_filename
            or croak "Can't open $new_filename";
        print {$OUTPUT_FILE} join( "\n", @lines );

    }

}

sub _rem_first_duplicates {

    my @input = @_;

CUT:
    foreach my $line (@input) {

        my $line_tmp = $line;
        if ( $config{'ignore_digits'} ) {
            $line_tmp =~ s/\d+/_D_/g;
        }

        if ( $config{'digest'} ) {
            $line_tmp = md5($line_tmp);
        }

        if ( $line_tmp eq '' ) {
            next CUT;
        }
        elsif ( $dupl{$line_tmp} > $config{'min_dupl'} ) {
            my $cut_line = shift @input;
            if ( $config{'log'} ) {
                print {$LOG_FILE} $cut_line, "\n";
            }
        }
        else {
            last CUT;
        }

    }

    return @input;

}

sub _rem_duplicates {

    my @input = @_;

    my $i = 0;
CUT:
    foreach my $line (@input) {

        my $line_tmp = $line;
        if ( $config{'ignore_digits'} ) {
            $line_tmp =~ s/\d+/_D_/g;
        }

        if ( $config{'digest'} ) {
            $line_tmp = md5($line_tmp);
        }

        if ( $line_tmp eq '' ) {
            $i++;
            next CUT;
        }
        elsif ( $dupl{$line_tmp} > $config{'min_dupl'} ) {

            my $cut_line = splice( @input, $i, 1 );
            if ( $config{'log'} ) {
                print {$LOG_FILE} $cut_line, "\n";
            }

        }

        $i++;

    }

    return @input;

}

sub _get_config {

    %config = (
        min_dupl                 => 3,
        suffix                   => 'content',
        ignore_digits            => 1,
        only_headers_and_footers => 1,
        digest                   => 0,
        log                      => 'text-identify-boilerplate.log'
    );

    my $arg_ref    = $_[0];
    my %config_arg = %$arg_ref;

    # merge hashes
    foreach my $opt (keys %config) {
        if (defined $config_arg{$opt}) {
            $config{$opt}=$config_arg{$opt};
        }
    }

    if ( $config{'log'} ) {
        open $LOG_FILE, '>', $config{'log'}
            or croak "Can't open $config{'log'}";
    }

    return;

}

1;    # End of Text::Identify::BoilerPlate

__END__