CSVUtils/csv_setop.pm

filter: Statementstmt 94.7% 126 / 133 Branchbran 86.9% 40 / 46 Conditioncond 55.5% 5 / 9 MC/DCmcdc 20.0% 1 / 5 Subroutinesub 71.4% 10 / 14 Podpod n/a totaltotal 87.9% 182 / 207 CC 45 SCAR 34.0
CC
45
Cov
91%
__ANON__
41.8
__ANON__
3.0
__ANON__
2.0
CRAP 30.1
« CSVUtils/paras2csv.pm ↑ Summary CSVUtils/csv_pick_cells.pm »
1 package App::CSVUtils::csv_setop;
2  
3 2 ▶use 5.010001;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 2 our $AUTHORITY = 'cpan:PERLANCAR'; # AUTHORITY
8 2 our $DATE = '2026-07-09'; # DATE
9 2 our $DIST = 'App-CSVUtils'; # DIST
10 2 our $VERSION = '1.038'; # VERSION
11  
12 2 ▶use App::CSVUtils qw(
Subroutine BEGIN: called CC=1 SCAR=0.0
13 gen_csv_util
14 2 );
15  
16 gen_csv_util(
17 name => 'csv_setop',
18 summary => 'Set operation (union/unique concatenation of rows, intersection/common rows, difference of rows) against several CSV files',
19 description => <<'MARKDOWN',
20  
21 This utility lets you perform one of several set options against several CSV
22 files:
23 - union
24 - intersection
25 - difference
26 - symmetric difference
27  
28 Example input:
29  
30 # file1.csv
31 a,b,c
32 1,2,3
33 4,5,6
34 7,8,9
35  
36 # file2.csv
37 a,b,c
38 1,2,3
39 4,5,7
40 7,8,9
41  
42 Output of intersection (`--intersect file1.csv file2.csv`), which will return
43 common rows between the two files:
44  
45 a,b,c
46 1,2,3
47 7,8,9
48  
49 Output of union (`--union file1.csv file2.csv`), which will return all rows with
50 duplicate removed:
51  
52 a,b,c
53 1,2,3
54 4,5,6
55 4,5,7
56 7,8,9
57  
58 Output of difference (`--diff file1.csv file2.csv`), which will return all rows
59 in the first file but not in the second:
60  
61 a,b,c
62 4,5,6
63  
64 Output of symmetric difference (`--symdiff file1.csv file2.csv`), which will
65 return all rows in the first file not in the second, as well as rows in the
66 second not in the first:
67  
68 a,b,c
69 4,5,6
70 4,5,7
71  
72 You can specify `--compare-fields` to consider some fields only, for example
73 `--union --compare-fields a,b file1.csv file2.csv`:
74  
75 a,b,c
76 1,2,3
77 4,5,6
78 7,8,9
79  
80 Each field specified in `--compare-fields` can be specified using
81 `F1:OTHER1,F2:OTHER2,...` format to refer to different field names or indexes in
82 each file, for example if `file3.csv` is:
83  
84 # file3.csv
85 Ei,Si,Bi
86 1,3,2
87 4,7,5
88 7,9,8
89  
90 Then `--union --compare-fields a:Ei,b:Bi file1.csv file3.csv` will result in:
91  
92 a,b,c
93 1,2,3
94 4,5,6
95 7,8,9
96  
97 Finally you can print out only certain fields using `--result-fields`.
98  
99 MARKDOWN
100 add_args => {
101 op => {
102 summary => 'Set operation to perform',
103 schema => ['str*', in=>[qw/intersect union diff symdiff/]],
104 req => 1,
105 cmdline_aliases => {
106 0 ▶ intersect => {is_flag=>1, summary=>'Shortcut for --op=intersect', code=>sub{ $_[0]{op} = 'intersect' }},
Subroutine __ANON__: not called CC=1 SCAR=6.9
107 0 ▶ union => {is_flag=>1, summary=>'Shortcut for --op=union' , code=>sub{ $_[0]{op} = 'union' }},
Subroutine __ANON__: not called CC=1 SCAR=6.9
108 0 ▶ diff => {is_flag=>1, summary=>'Shortcut for --op=diff' , code=>sub{ $_[0]{op} = 'diff' }},
Subroutine __ANON__: not called CC=1 SCAR=6.9
109 0 ▶ symdiff => {is_flag=>1, summary=>'Shortcut for --op=symdiff' , code=>sub{ $_[0]{op} = 'symdiff' }},
Subroutine __ANON__: not called CC=1 SCAR=6.9
110 },
111 },
112 ignore_case => {
113 schema => 'bool*',
114 cmdline_aliases => {i=>{}},
115 },
116 compare_fields => {
117 schema => ['str*'],
118 },
119 result_fields => {
120 schema => ['str*'],
121 },
122 },
123  
124 links => [
125 {url=>'prog:setop'},
126 ],
127  
128 reads_multiple_csv => 1,
129  
130 tags => ['category:combining', 'set'],
131  
132 16 ▶ on_begin => sub {
Subroutine __ANON__: called CC=2 SCAR=7.0
133 16 my $r = shift;
134  
135 # check arguments
136 die [400, "Please specify at least 2 files"]
137 16 ▶ unless @{ $r->{util_args}{input_filenames} } >= 2;
Branch: 1 branch
BranchTrueFalseTotal
unless @{$r->{"util_args"}{"input_filenames"};} >= 2 0 16 16
138  
139 # these are the keys we add to the stash
140 16 $r->{all_input_data_rows} = []; # array of all data rows, one elem for each input file
141 16 $r->{all_input_fields} = []; # array of input_fields, one elem for each input file
142 16 $r->{all_input_fields_idx} = []; # array of input_fields_idx, one elem for each input file
143 },
144  
145 32 ▶ on_input_header_row => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
146 32 my $r = shift;
147  
148 32 $r->{all_input_fields} [ $r->{input_filenum}-1 ] = $r->{input_fields};
149 32 $r->{all_input_fields_idx}[ $r->{input_filenum}-1 ] = $r->{input_fields_idx};
150 32 $r->{all_input_data_rows} [ $r->{input_filenum}-1 ] = [];
151 },
152  
153 96 ▶ on_input_data_row => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
154 96 my $r = shift;
155  
156 96 push @{ $r->{all_input_data_rows}[ $r->{input_filenum}-1 ] },
157 96 $r->{input_row};
158 },
159  
160 16 ▶ after_close_input_files => sub {
Subroutine __ANON__: called CC=41 SCAR=37.3
161 16 require Tie::IxHash;
162  
163 16 my $r = shift;
164  
165 16 my $op = $r->{util_args}{op};
166 16 my $ci = $r->{util_args}{ignore_case};
167 16 my $num_files = @{ $r->{util_args}{input_filenames} };
168  
169 16 my @compare_fields; # elem = [fieldname-for-file1, fieldname-for-file2, ...]
170 16 ▶ if (defined $r->{util_args}{compare_fields}) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $r->{'util_args'}{'compare_fields'}) { } 9 7 16
171 my @ff = ref($r->{util_args}{compare_fields}) eq 'ARRAY' ?
172 0 ▶ @{$r->{util_args}{compare_fields}} : split(/,/, $r->{util_args}{compare_fields});
Branch: 1 branch
BranchTrueFalseTotal
ref $r->{'util_args'}{'compare_fields'} eq 'ARRAY' ? : 0 9 9
173 9 for my $field_idx (0..$#ff) {
174 13 my @ff2 = split /:/, $ff[$field_idx];
175 13 for (@ff2+1 .. $num_files) {
176 13 push @ff2, $ff2[0];
177 }
178 13 $compare_fields[$field_idx] = \@ff2;
179 }
180 # XXX check that specified fields exist
181 } else {
182 7 for my $field_idx (0..$#{ $r->{all_input_fields}[0] }) {
183 $compare_fields[$field_idx] = [
184 21 map { $r->{all_input_fields}[0][$field_idx] } 0..$num_files-1];
185 }
186 }
187  
188 16 my @result_fields; # elem = fieldname, ...
189 16 ▶ if (defined $r->{util_args}{result_fields}) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $r->{'util_args'}{'result_fields'}) { } 4 12 16
190 @result_fields = ref($r->{util_args}{result_fields}) eq 'ARRAY' ?
191 0 ▶ @{$r->{util_args}{result_fields}} : split(/,/, $r->{util_args}{result_fields});
Branch: 1 branch
BranchTrueFalseTotal
ref $r->{'util_args'}{'result_fields'} eq 'ARRAY' ? : 0 4 4
192 # XXX check that specified fields exist
193 } else {
194 12 @result_fields = @{ $r->{all_input_fields}[0] };
195 }
196 16 $r->{output_fields} = \@result_fields;
197  
198 16 tie my(%res), 'Tie::IxHash';
199  
200 96 ▶ my $code_get_compare_key = sub {
Subroutine __ANON__: called CC=3 SCAR=11.0
201 96 my ($file_idx, $row_idx) = @_;
202 96 my $row = $r->{all_input_data_rows}[$file_idx][$row_idx];
203 my $key = join "|", map {
204 96 my $field = $compare_fields[$_][$file_idx];
205 204 my $field_idx = $r->{all_input_fields_idx}[$file_idx]{$field};
206 204 ▶ my $val = defined $field_idx ? $row->[$field_idx] : "";
Branch: 1 branch
BranchTrueFalseTotal
defined $field_idx ? : 204 0 204
207 204 ▶ $val = uc $val if $ci;
Branch: 1 branch
BranchTrueFalseTotal
if $ci 48 156 204
208 204 $val;
209 } 0..$#compare_fields;
210 #say "D:compare_key($file_idx, $row_idx)=<$key>";
211 96 $key;
212 16 };
213  
214 29 ▶ my $code_print_result_row = sub {
Subroutine __ANON__: called CC=2 SCAR=7.0
215 29 my ($file_idx, $row) = @_;
216 my @res_row = map {
217 29 my $field = $result_fields[$_];
218 79 my $field_idx = $r->{all_input_fields_idx}[$file_idx]{$field};
219 79 ▶ defined $field_idx ? $row->[$field_idx] : "";
Branch: 1 branch
BranchTrueFalseTotal
defined $field_idx ? : 79 0 79
220 } 0..$#result_fields;
221 29 $r->{code_print_row}->(\@res_row);
222 16 };
223  
224 16 ▶ if ($op eq 'intersect') {
Branch: 4 branches
BranchTrueFalseTotal
if ($op eq 'intersect') { } 4 12 16
elsif ($op eq 'union') { } 4 8 12
elsif ($op eq 'diff') { } 4 4 8
elsif ($op eq 'symdiff') { } 4 0 4
225 4 for my $file_idx (0..$num_files-1) {
226 8 ▶ if ($file_idx == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == 0) { } 4 4 8
227 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
228 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
229 12 ▶ $res{$key} //= [1, $row_idx]; # [num_of_occurrence, row_idx]
Condition50%1/2
$res{$key} //= [1, $row_idx]
l!l
012
MC/DC0%0/1
$res{$key} //= [1, $row_idx]
$res{$key}
Truth table50%1/2
A
A= $res{$key}
Aresult
11
00
230 }
231 } else {
232 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
233 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
234 12 ▶ if ($res{$key} && $res{$key}[0] == $file_idx) {
Branch: 1 branch
BranchTrueFalseTotal
if ($res{$key} and $res{$key}[0] == $file_idx) 10 2 12
Condition66%2/3
$res{$key} and $res{$key}[0] == $file_idx
!ll&&!rl&&r
2010
MC/DC50%1/2
$res{$key} and $res{$key}[0] == $file_idx
$res{$key} $res{$key}[0] == $file_idx
Truth table66%2/3
A and B
A= $res{$key}B= $res{$key}[0] == $file_idx
ABresult
0X0
100
111
235 10 $res{$key}[0]++;
236 }
237 }
238 }
239  
240 # print result
241 8 ▶ if ($file_idx == $num_files-1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == $num_files - 1) 4 4 8
242 4 for my $key (keys %res) {
243 $code_print_result_row->(
244 0, $r->{all_input_data_rows}[0][$res{$key}[1]])
245 12 ▶ if $res{$key}[0] == $num_files;
Branch: 1 branch
BranchTrueFalseTotal
if $res{$key}[0] == $num_files 10 2 12
246 }
247 }
248 } # for file_idx
249  
250 } elsif ($op eq 'union') {
251  
252 4 for my $file_idx (0..$num_files-1) {
253 8 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
254 24 my $key = $code_get_compare_key->($file_idx, $row_idx);
255 24 ▶ next if $res{$key}++;
Branch: 1 branch
BranchTrueFalseTotal
if $res{$key}++ 11 13 24
256 13 my $row = $r->{all_input_data_rows}[$file_idx][$row_idx];
257 13 $code_print_result_row->($file_idx, $row);
258 }
259 } # for file_idx
260  
261 } elsif ($op eq 'diff') {
262  
263 4 for my $file_idx (0..$num_files-1) {
264 8 ▶ if ($file_idx == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == 0) { } 4 4 8
265 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
266 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
267 12 ▶ $res{$key} //= [$file_idx, $row_idx];
Condition50%1/2
$res{$key} //= [$file_idx, $row_idx]
l!l
012
MC/DC0%0/1
$res{$key} //= [$file_idx, $row_idx]
$res{$key}
Truth table50%1/2
A
A= $res{$key}
Aresult
11
00
268 }
269 } else {
270 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
271 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
272 12 delete $res{$key};
273 }
274 }
275  
276 # print result
277 8 ▶ if ($file_idx == $num_files-1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == $num_files - 1) 4 4 8
278 4 for my $key (keys %res) {
279 2 my ($file_idx, $row_idx) = @{ $res{$key} };
280 $code_print_result_row->(
281 2 0, $r->{all_input_data_rows}[$file_idx][$row_idx]);
282 }
283 }
284 } # for file_idx
285  
286 } elsif ($op eq 'symdiff') {
287  
288 4 for my $file_idx (0..$num_files-1) {
289 8 ▶ if ($file_idx == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == 0) { } 4 4 8
290 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
291 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
292 12 ▶ $res{$key} //= [1, $file_idx, $row_idx]; # [num_of_occurrence, file_idx, row_idx]
Condition50%1/2
$res{$key} //= [1, $file_idx, $row_idx]
l!l
012
MC/DC0%0/1
$res{$key} //= [1, $file_idx, $row_idx]
$res{$key}
Truth table50%1/2
A
A= $res{$key}
Aresult
11
00
293 }
294 } else {
295 4 for my $row_idx (0..$#{ $r->{all_input_data_rows}[$file_idx] }) {
296 12 my $key = $code_get_compare_key->($file_idx, $row_idx);
297 12 ▶ if (!$res{$key}) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $res{$key}) { } 2 10 12
298 2 $res{$key} = [1, $file_idx, $row_idx];
299 } else {
300 10 $res{$key}[0]++;
301 }
302 }
303 }
304  
305 # print result
306 8 ▶ if ($file_idx == $num_files-1) {
Branch: 1 branch
BranchTrueFalseTotal
if ($file_idx == $num_files - 1) 4 4 8
307 4 for my $key (keys %res) {
308 14 my ($num_occur, $file_idx, $row_idx) = @{ $res{$key} };
309 $code_print_result_row->(
310 14 ▶ 0, $r->{all_input_data_rows}[$file_idx][$row_idx])
Branch: 1 branch
BranchTrueFalseTotal
if $num_occur == 1 4 10 14
311 if $num_occur == 1;
312 }
313 }
314 } # for file_idx
315  
316 } else {
317  
318 0 die [400, "Unknown/unimplemented op '$op'"];
319  
320 }
321  
322 #use DD; dd +{
323 # compare_fields => \@compare_fields,
324 # result_fields => \@result_fields,
325 # all_input_data_rows=>$r->{all_input_data_rows},
326 # all_input_fields=>$r->{all_input_fields},
327 # all_input_fields_idx=>$r->{all_input_fields_idx},
328 #};
329 },
330 2 );
331  
332 2 1;
333 # ABSTRACT: Set operation (union/unique concatenation of rows, intersection/common rows, difference of rows) against several CSV files
334  
335 __END__
« CSVUtils/paras2csv.pm ↑ Summary CSVUtils/csv_pick_cells.pm »