Pipeline/Input/DelimitedText.pm

filter: Statementstmt 92.1% 59 / 64 Branchbran 82.1% 23 / 28 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 8 / 8 Podpod n/a totaltotal 90.0% 90 / 100 CC 17 SCAR 23.5
CC
17
Cov
89%
run
15.7
BUILD
3.0
BEGIN
1.0
CRAP 10.5
« Pipeline/Input/Excel.pm ↑ Summary Pipeline.pm »
1 =pod
2
3 =head1 NAME
4
5 ETL::Pipeline::Input::DelimitedText - Input source for CSV, tab, or pipe
6 delimited files
7
8 =head1 SYNOPSIS
9
10 use ETL::Pipeline;
11 ETL::Pipeline->new( {
12 input => ['DelimitedText', iname => qr/\.csv$/i],
13 mapping => {First => 'Header1', Second => 'Header2'},
14 output => ['UnitTest']
15 } )->process;
16
17 =head1 DESCRIPTION
18
19 B<ETL::Pipeline::Input::DelimitedText> defines an input source for reading
20 CSV (comma seperated variable), tab delimited, or pipe delimited files. It
21 uses L<Text::CSV> for parsing.
22
23 B<ETL::Pipeline::Input::DelimitedText> expects a standard CSV file. A lot of
24 hand built exporters often forget quote marks, use invalid characters, or don't
25 escape the quotes. If you experience trouble with a file, experiment with the
26 options to L<Text::CSV>.
27
28 =cut
29
30 package ETL::Pipeline::Input::DelimitedText;
31 2 ▶use Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
32
33 2 ▶use 5.014000;
Subroutine BEGIN: called CC=1 SCAR=0.0
34 2 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
35
36 2 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
37 2 ▶use Text::CSV;
Subroutine BEGIN: called CC=1 SCAR=0.0
38
39
40 2 our $VERSION = '3.00';
41
42
43 =head1 METHODS & ATTRIBUTES
44
45 =head2 Arguments for L<ETL::Pipeline/input>
46
47 B<ETL::Pipeline::Input::DelimitedText> consumes the
48 L<ETL::Pipeline::Input::File> and L<ETL::Pipeline::Input::File::Table> roles.
49 It supports all of the attributes from these two.
50
51 In addition, B<ETL::Pipeline::Input::DelimitedText> uses the options from
52 L<Text::CSV>. See L<Text::CSV> for a list.
53
54 # Pipe delimited, allowing embedded new lines.
55 $etl->input( 'DelimitedText',
56 iname => qr/\.dat$/i,
57 sep_char => '|',
58 binary => 1
59 );
60
61 =cut
62
63 8 ▶sub BUILD {
Subroutine BUILD: called CC=3 SCAR=11.0
64 8 my $self= shift;
65 8 my $arguments = shift;
66
67 8 my %options;
68 8 foreach my $key (Text::CSV::known_attributes) {
69 256 ▶ $options{$key} = $arguments->{$key} if exists $arguments->{$key};
Branch: 1 branch
BranchTrueFalseTotal
if exists $arguments->{$key} 0 256 256
70 }
71 8 $self->_csv_options( \%options );
72 }
73
74
75 =head3 skipping
76
77 Optional. If you use a code reference for B<skipping>, this input source sends a
78 line of plain text. The text is B<not> parsed into fields. I assume that you're
79 skipping report headers, not formatted data.
80
81 If you pass an integer, the input source completely skips over that many lines.
82 It reads and discards the lines without parsing.
83
84 =head2 Methods
85
86 =head3 run
87
88 This is the main loop. It opens the file, reads records, and closes it when
89 done. This is the place to look if there are problems.
90
91 L<ETL::Pipeline> automatically calls this method.
92
93 =cut
94
95 7 ▶sub run {
Subroutine run: called CC=15 SCAR=27.6
96 7 my ($self, $etl) = @_;
97
98 7 my $csv = Text::CSV->new( $self->_csv_options );
99 7 my $path = $self->path;
100
101 # Open the file.
102 7 my $handle = $path->openr();
103 7 ▶ croak "Cannot read '$path'" unless defined $handle;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $handle 0 7 7
104
105 # Skip over report headers. These are not data. They are extra rows put
106 # there for report formats. The data starts after these rows.
107 7 my $line;
108 7 my $skip = $self->skipping;
109 7 ▶ if (ref( $skip ) eq 'CODE') {
Branch: 2 branches
BranchTrueFalseTotal
if (ref $skip eq 'CODE') { } 1 6 7
elsif ($skip > 0) { } 1 5 6
110 1 ▶ while (!$handle->eof) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($handle->eof) 2 0 2
111 2 $line = $handle->getline;
112 2 ▶ last if !$skip->( $line );
Branch: 1 branch
BranchTrueFalseTotal
unless &$skip($line) 1 1 2
113 }
114 } elsif ($skip > 0) {
115 1 $handle->getline foreach (1 .. $skip);
116 }
117
118 # Load field names.
119 7 ▶ unless ($self->no_column_names) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($self->no_column_names) 6 1 7
120 6 my $fields;
121 6 ▶ if (defined $line) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $line) { } 1 5 6
122 1 $fields = [$csv->parse( $line )];
123 1 $line = undef;
124 } else {
125 5 $fields = $csv->getline( $handle );
126 }
127
128 6 ▶ if (defined $fields) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $fields) 3 3 6
129 3 my @names;
130 3 ▶ while (my ($index, $value) = each @$fields) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each @$fields) 10 3 13
131 10 push @names, {$value => $index};
132 }
133 3 $etl->aliases( @names );
134 }
135 }
136
137 # Read and process each line.
138 7 ▶ while (!$csv->eof) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($csv->eof) 11 7 18
139 11 my $fields;
140 11 ▶ if (defined $line) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $line) { } 0 11 11
141 0 $fields = $csv->parse( $line );
142 0 $line = undef;
143 } else {
144 11 $fields = $csv->getline( $handle );
145 }
146
147 11 ▶ if (defined $fields) {
Branch: 2 branches
BranchTrueFalseTotal
if (defined $fields) { } 7 4 11
elsif (not $csv->eof) { } 0 4 4
148 7 $etl->record( $fields );
149 } elsif (!$csv->eof) {
150 0 my $at = $csv->record_number;
151 0 my ($code, $message, $position) = $csv->error_diag;
152 0 croak "CSV file '$path', error $code: $message at character $position (record $at)";
153 }
154 }
155
156 # Close the file when done.
157 7 close $handle;
158 }
159
160
161 #-------------------------------------------------------------------------------
162 # Internal methods and attributes
163
164 # Text::CSV options passed into the object constructor. I either needed to
165 # store the options or a Text::CSV object. I chose to store the options. The
166 # "run" method uses them to create a Text::CSV object.
167 2 has '_csv_options' => (
168 is => 'rw',
169 isa => 'HashRef[Any]',
170 );
171
172
173 =head1 SEE ALSO
174
175 L<ETL::Pipeline>, L<ETL::Pipeline::Input>, L<ETL::Pipeline::Input::File>,
176 L<ETL::Pipeline::Input::File::Table>, L<Text::CSV>
177
178 =cut
179
180 2 with 'ETL::Pipeline::Input';
181 2 with 'ETL::Pipeline::Input::File';
182 2 with 'ETL::Pipeline::Input::File::Table';
183
184
185 =head1 AUTHOR
186
187 Robert Wohlfarth <robert.j.wohlfarth@vumc.org>
188
189 =head1 LICENSE
190
191 Copyright 2021 (c) Vanderbilt University Medical Center
192
193 This program is free software; you can redistribute it and/or modify it under
194 the same terms as Perl itself.
195
196 =cut
197
198 2 ▶no Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
199 2 __PACKAGE__->meta->make_immutable;
« Pipeline/Input/Excel.pm ↑ Summary Pipeline.pm »