CSVUtils/csv_concat.pm

filter: Statementstmt 100% 82 / 82 Branchbran 100% 12 / 12 Conditioncond 100% 3 / 3 MC/DCmcdc 100% 2 / 2 Subroutinesub 100% 8 / 8 Podpod n/a totaltotal 100% 107 / 107 CC 14 SCAR 22.7
CC
14
Cov
100%
__ANON__
14.0
BEGIN
1.0
BEGIN
1.0
CRAP 9.7
« CSVUtils/csv_rtrim.pm ↑ Summary CSVUtils/csv_intrange.pm »
1 package App::CSVUtils::csv_concat;
2  
3 2 ▶use 5.010001;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 2 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 2 ▶use Log::ger;
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 2 our $AUTHORITY = 'cpan:PERLANCAR'; # AUTHORITY
9 2 our $DATE = '2026-07-09'; # DATE
10 2 our $DIST = 'App-CSVUtils'; # DIST
11 2 our $VERSION = '1.038'; # VERSION
12  
13 2 ▶use App::CSVUtils qw(
Subroutine BEGIN: called CC=1 SCAR=0.0
14 gen_csv_util
15 2 );
16  
17 gen_csv_util(
18 name => 'csv_concat',
19 summary => 'Concatenate several CSV files together, '.
20 'collecting all the fields',
21 description => <<'MARKDOWN',
22  
23 Example, concatenating this CSV:
24  
25 col1,col2
26 1,2
27 3,4
28  
29 and:
30  
31 col2,col4
32 a,b
33 c,d
34 e,f
35  
36 and:
37  
38 col3
39 X
40 Y
41  
42 will result in:
43  
44 col1,col2,col4,col3
45 1,2,
46 3,4,
47 ,a,b
48 ,c,d
49 ,e,f
50 ,,,X
51 ,,,Y
52  
53 When `--overlay` option is enabled, the result will be:
54  
55 col1,col2,col4,col3
56 1,2,b,X
57 3,4,d,Y
58 ,e,f,
59  
60 When `--overlay` as well as `--overwrite-fields` option are enabled, the result
61 will be:
62  
63 col1,col2,col4,col3
64 1,a,b,X
65 3,c,d,Y
66 ,e,f,
67  
68 Keywords: join, merge, overlay
69  
70 MARKDOWN
71 add_args => {
72 overlay => {
73 summary => 'Whether to overlay rows from second and subsequent CSV files to the first',
74 schema => 'bool*',
75 description => <<'MARKDOWN',
76  
77 By default, rows from the second CSV file will be added after all the rows from
78 the first CSV are added, and so on. However, when this option is enabled, the
79 rows the second and subsequent CSV files will be added together (overlaid). See
80 the utility's example for an illustration.
81  
82 See also the `--overwrite-fields` option.
83  
84 MARKDOWN
85 },
86 overwrite_fields => {
87 summary => 'Whether fields from subsequent CSV files should overwrite existing fields from previous CSV files',
88 schema => 'bool*',
89 description => <<'MARKDOWN',
90  
91 When in overlay mode (`--overlay`), by default the value for a field is
92 retrieved from the first CSV file that has the field. With `--overwrite-fields`
93 option enabled, the value will be retrieved from the last CSV that has the
94 field. See the utility's example for an illustration.
95  
96 MARKDOWN
97 },
98 },
99 tags => ['category:combining', 'join', 'merge'],
100  
101 reads_multiple_csv => 1,
102  
103 3 ▶ before_open_input_files => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
104 3 my $r = shift;
105  
106 # we add the following keys to the stash
107 3 $r->{all_input_fields} = [];
108 3 $r->{all_input_fh} = [];
109 },
110  
111 9 ▶ on_input_header_row => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
112 9 my $r = shift;
113  
114 # after we read the header row of each input file, we record the fields
115 # as well as the filehandle, so we can resume reading the data rows
116 # later. before printing all the rows, we collect all the fields from
117 # all files first.
118  
119 9 push @{ $r->{all_input_fields} }, $r->{input_fields};
120 9 push @{ $r->{all_input_fh} }, $r->{input_fh};
121 9 $r->{wants_skip_file}++;
122 },
123  
124 3 ▶ after_close_input_files => sub {
Subroutine __ANON__: called CC=14 SCAR=26.4
125 3 my $r = shift;
126  
127 # as described in the on_input_header_row hook, we have saved the input
128 # fields as well as the filehandles. so even though we are called in the
129 # after_close_input_files hook, where the main routine supposedly has
130 # closed all input files, we actually have saved the filehandles and can
131 # resume reading from them.
132  
133 # collect all output fields
134 3 $r->{output_fields} = [];
135 3 $r->{output_fields_idx} = {};
136 3 for my $i (0 .. $#{ $r->{all_input_fields} }) {
137 9 my $input_fields = $r->{all_input_fields}[$i];
138 9 for my $j (0 .. $#{ $input_fields }) {
139 21 my $field = $input_fields->[$j];
140 21 ▶ unless (grep {$field eq $_} @{ $r->{output_fields} }) {
Branch: 1 branch
BranchTrueFalseTotal
unless (grep {$field eq $_;} @{$$r{"output_fields"};}) 12 9 21
141 12 push @{ $r->{output_fields} }, $field;
142 12 $r->{output_fields_idx}{$field} = $#{ $r->{output_fields} };
143 }
144 }
145 }
146  
147 3 my $csv = $r->{input_parser};
148  
149 3 ▶ if ($r->{util_args}{overlay}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($r->{'util_args'}{'overlay'}) { } 2 1 3
150  
151 2 my $overwrite_fields = $r->{util_args}{overwrite_fields};
152 2 my $output_fields_idx = $r->{output_fields_idx};
153 2 while (1) {
154 8 my $has_not_eof;
155 8 my $combined_row = [("") x @{ $r->{output_fields} }];
156 8 my %seen_fields;
157 8 for my $i (0 .. $#{ $r->{all_input_fh} }) {
158 24 my $fh = $r->{all_input_fh}[$i];
159  
160 24 ▶ next if eof($fh);
Branch: 1 branch
BranchTrueFalseTotal
if eof $fh 8 16 24
161 16 $has_not_eof++;
162 16 my $row = $csv->getline($fh);
163 16 my $input_fields = $r->{all_input_fields}[$i];
164 16 for my $j (0 .. $#{ $input_fields }) {
165 36 my $field = $input_fields->[$j];
166 36 ▶ if (!($seen_fields{$field}++) || $overwrite_fields) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $seen_fields{$field}++ or $overwrite_fields) 29 7 36
Condition100%3/3
not $seen_fields{$field}++ or $overwrite_fields
l!l&&r!l&&!r
2277
MC/DC100%2/2
not $seen_fields{$field}++ or $overwrite_fields
not $seen_fields{$field}++ $overwrite_fields
Truth table100%3/3
not(A) or B
A= not $seen_fields{$field}++B= $overwrite_fields
ABresult
1X1
011
000
167 29 $combined_row->[ $output_fields_idx->{$field} ] = $row->[$j];
168 }
169 }
170 } # for all_input_fh
171 8 ▶ last unless $has_not_eof;
Branch: 1 branch
BranchTrueFalseTotal
unless $has_not_eof 2 6 8
172 6 $r->{code_print_row}->($combined_row);
173 } # while 1
174  
175 } else {
176  
177 # print all the data rows
178 1 for my $i (0 .. $#{ $r->{all_input_fh} }) {
179 log_trace "[%d/%d] Adding rows from file #%d ...",
180 3 $i+1, scalar(@{$r->{all_input_fh}}), $i+1;
181 3 my $fh = $r->{all_input_fh}[$i];
182 3 my $input_fields = $r->{all_input_fields}[$i];
183 3 ▶ while (my $row = $csv->getline($fh)) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $csv->getline($fh)) 8 3 11
184 8 my $combined_row = [("") x @{ $r->{output_fields} }];
185 8 for my $j (0 .. $#{ $input_fields }) {
186 18 my $field = $input_fields->[$j];
187 18 $combined_row->[ $r->{output_fields_idx}{$field} ] = $row->[$j];
188 }
189 8 $r->{code_print_row}->($combined_row);
190 }
191 } # for all input fh
192  
193 }
194 },
195 2 );
196  
197 2 1;
198 # ABSTRACT: Concatenate several CSV files together, collecting all the fields
199  
200 __END__
« CSVUtils/csv_rtrim.pm ↑ Summary CSVUtils/csv_intrange.pm »