blib/lib/Text/Identify/BoilerPlate.pm

filter: Statementstmt 19.0% 20 / 105 Branchbran 0.0% 0 / 44 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 55.5% 5 / 9 Podpod 100% 1 / 1 totaltotal 16.3% 26 / 159 CC 28 SCAR 43.6
CC
28
Cov
13%
rem_boilerplate
156.0
_rem_first_duplicates
56.0
_rem_duplicates
56.0
CRAP 78.1
↑ Summary
1 package Text::Identify::BoilerPlate;
2  
3 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 2 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 2 ▶use base qw{ Exporter };
Subroutine BEGIN: called CC=1 SCAR=0.0
7 2 ▶use Digest::MD5 qw{ md5 };
Subroutine BEGIN: called CC=1 SCAR=0.0
8  
9 2 our $VERSION = '0.3.1';
10 2 our @EXPORT_OK = qw{ rem_boilerplate };
11  
12 2 my %config;
13 my %dupl;
14  
15 2 my $LOG_FILE;
16  
17 0 ▶sub rem_boilerplate {
Subroutine rem_boilerplate: not called CC=12 SCAR=50.5
Pod documented
18  
19 0 my ( $files, $arg_ref ) = @_;
20  
21 0 _get_config($arg_ref);
22  
23 0 my @files = @$files;
24  
25 0 print "MIN_DUPL: $config{'min_dupl'}\n";
26  
27 0 ▶ if ($config{'min_dupl'} =~ s/\s*\%//) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"min_dupl"} =~ s/\s*\%//) 0 0 0
28 0 my $num_of_files = @files;
29 $config{'min_dupl'} = int (
30 0 ($num_of_files * $config{'min_dupl'})
31 / 100
32 );
33 0 ▶ if ($config{'min_dupl'} < 2) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"min_dupl"} < 2) 0 0 0
34 0 $config{'min_dupl'}=2;
35 }
36 }
37  
38 0 print "MIN_DUPL: $config{'min_dupl'}\n";
39
40  
41 0 foreach my $file (@files) {
42  
43 0 my @lines;
44 0 ▶ open my $INPUT_FILE, '<', $file
Branch: 1 branch
BranchTrueFalseTotal
open my $INPUT_FILE, "<", $file or croak("Can't open $file") 0 0 0
45 or croak "Can't open $file";
46 0 ▶ while ( my $line = <$INPUT_FILE> ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $INPUT_FILE)) 0 0 0
47  
48 0 $line =~ s/^\s*//g;
49 0 $line =~ s/\s*$//g;
50 0 $line =~ s/\t/ /g;
51 0 $line =~ s/ +/ /g;
52 0 $line =~ s/\r//g;
53  
54 0 my $line_tmp = $line;
55 0 ▶ if ( $config{'ignore_digits'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"ignore_digits"}) 0 0 0
56 0 $line_tmp =~ s/\d+/_D_/g;
57 }
58 0 ▶ if ( $config{'digest'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"digest"}) 0 0 0
59 0 $line_tmp = md5($line_tmp);
60 }
61  
62 0 $dupl{$line_tmp}++;
63 0 push @lines, $line;
64  
65 }
66 0 $file = [ $file, \@lines ]; # name + content
67 }
68  
69 # find duplicated lines
70 0 foreach my $file (@files) {
71  
72 0 ▶ if ( $config{'log'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"log"}) 0 0 0
73 0 print {$LOG_FILE} "\n-- $file->[0] --\n";
74 }
75  
76 0 my $lines = $file->[1];
77 0 my @lines = @$lines;
78  
79 0 ▶ if ( $config{'only_headers_and_footers'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{'only_headers_and_footers'}) { } 0 0 0
80 0 @lines = _rem_first_duplicates(@lines);
81 0 @lines = reverse @lines;
82 0 @lines = _rem_first_duplicates(@lines);
83 0 @lines = reverse @lines;
84 }
85 else {
86 0 @lines = _rem_duplicates(@lines);
87 }
88  
89 0 my $filename = $file->[0];
90 0 my $new_filename = $filename . "." . $config{'suffix'};
91 0 ▶ open my $OUTPUT_FILE, '>', $new_filename
Branch: 1 branch
BranchTrueFalseTotal
open my $OUTPUT_FILE, ">", $new_filename or croak("Can't open $new_filename") 0 0 0
92 or croak "Can't open $new_filename";
93 0 print {$OUTPUT_FILE} join( "\n", @lines );
94  
95 }
96  
97 }
98  
99 0 ▶sub _rem_first_duplicates {
Subroutine _rem_first_duplicates: not called CC=7 SCAR=40.3
100  
101 0 my @input = @_;
102  
103 CUT:
104 0 foreach my $line (@input) {
105  
106 0 my $line_tmp = $line;
107 0 ▶ if ( $config{'ignore_digits'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"ignore_digits"}) 0 0 0
108 0 $line_tmp =~ s/\d+/_D_/g;
109 }
110  
111 0 ▶ if ( $config{'digest'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"digest"}) 0 0 0
112 0 $line_tmp = md5($line_tmp);
113 }
114  
115 0 ▶ if ( $line_tmp eq '' ) {
Branch: 2 branches
BranchTrueFalseTotal
if ($line_tmp eq '') { } 0 0 0
elsif ($dupl{$line_tmp} > $config{'min_dupl'}) { } 0 0 0
116 0 next CUT;
117 }
118 elsif ( $dupl{$line_tmp} > $config{'min_dupl'} ) {
119 0 my $cut_line = shift @input;
120 0 ▶ if ( $config{'log'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"log"}) 0 0 0
121 0 print {$LOG_FILE} $cut_line, "\n";
122 }
123 }
124 else {
125 0 last CUT;
126 }
127  
128 }
129  
130 0 return @input;
131  
132 }
133  
134 0 ▶sub _rem_duplicates {
Subroutine _rem_duplicates: not called CC=7 SCAR=40.3
135  
136 0 my @input = @_;
137  
138 0 my $i = 0;
139 CUT:
140 0 foreach my $line (@input) {
141  
142 0 my $line_tmp = $line;
143 0 ▶ if ( $config{'ignore_digits'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"ignore_digits"}) 0 0 0
144 0 $line_tmp =~ s/\d+/_D_/g;
145 }
146  
147 0 ▶ if ( $config{'digest'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"digest"}) 0 0 0
148 0 $line_tmp = md5($line_tmp);
149 }
150  
151 0 ▶ if ( $line_tmp eq '' ) {
Branch: 2 branches
BranchTrueFalseTotal
if ($line_tmp eq '') { } 0 0 0
elsif ($dupl{$line_tmp} > $config{'min_dupl'}) { } 0 0 0
152 0 $i++;
153 0 next CUT;
154 }
155 elsif ( $dupl{$line_tmp} > $config{'min_dupl'} ) {
156  
157 0 my $cut_line = splice( @input, $i, 1 );
158 0 ▶ if ( $config{'log'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"log"}) 0 0 0
159 0 print {$LOG_FILE} $cut_line, "\n";
160 }
161  
162 }
163  
164 0 $i++;
165  
166 }
167  
168 0 return @input;
169  
170 }
171  
172 0 ▶sub _get_config {
Subroutine _get_config: not called CC=5 SCAR=34.0
173  
174 0 %config = (
175 min_dupl => 3,
176 suffix => 'content',
177 ignore_digits => 1,
178 only_headers_and_footers => 1,
179 digest => 0,
180 log => 'text-identify-boilerplate.log'
181 );
182  
183 0 my $arg_ref = $_[0];
184 0 my %config_arg = %$arg_ref;
185  
186 # merge hashes
187 0 foreach my $opt (keys %config) {
188 0 ▶ if (defined $config_arg{$opt}) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $config_arg{$opt}) 0 0 0
189 0 $config{$opt}=$config_arg{$opt};
190 }
191 }
192  
193 0 ▶ if ( $config{'log'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($config{"log"}) 0 0 0
194 0 ▶ open $LOG_FILE, '>', $config{'log'}
Branch: 1 branch
BranchTrueFalseTotal
open $LOG_FILE, ">", $config{"log"} or croak(qq[Can't open $config{"log"}]) 0 0 0
195 or croak "Can't open $config{'log'}";
196 }
197  
198 0 return;
199  
200 }
201  
202 2 1; # End of Text::Identify::BoilerPlate
203  
204 __END__
↑ Summary