Pipeline/Input/File/List.pm

filter: Statementstmt 100% 48 / 48 Branchbran 90.0% 9 / 10 Conditioncond 66.6% 2 / 3 MC/DCmcdc 50.0% 1 / 2 Subroutinesub 100% 11 / 11 Podpod 50.0% 1 / 2 totaltotal 94.7% 72 / 76 CC 8 SCAR 10.2
CC
8
Cov
97%
next_path
5.0
__ANON__
4.0
BEGIN
1.0
CRAP 2.8
« Pipeline/Input/JsonFiles.pm ↑ Summary Pipeline/Input/XmlFiles.pm »
1 =pod
2
3 =head1 NAME
4
5 ETL::Pipeline::Input::File::List - Role for input sources with multiple files
6
7 =head1 SYNOPSIS
8
9 # In the input source...
10 use Moose;
11 with 'ETL::Pipeline::Input';
12 with 'ETL::Pipeline::Input::File::List';
13 ...
14 sub run {
15 my ($self, $etl) = @_;
16 ...
17 while (my $path = $self->next_path( $etl )) {
18 ...
19 }
20 }
21
22 =head1 DESCRIPTION
23
24 This is a role used by input sources. It defines everything you need to process
25 multiple input files of the same format. The role uses L<Path::Class::Rule> to
26 locate matching files.
27
28 Your input source calls the L</next_path> method in a loop. That's it. The role
29 automatically processes constructor arguments that match L<Path::Class::Rule>
30 criteria. It then builds a list of matching files the first time your code calls
31 L</next_path>.
32
33 =cut
34
35 package ETL::Pipeline::Input::File::List;
36
37 4 ▶use 5.014000;
Subroutine BEGIN: called CC=1 SCAR=0.0
38
39 4 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
40 4 ▶use Moose::Role;
Subroutine BEGIN: called CC=1 SCAR=0.0
41 4 ▶use MooseX::Types::Path::Class;
Subroutine BEGIN: called CC=1 SCAR=0.0
42 4 ▶use Path::Class::Rule;
Subroutine BEGIN: called CC=1 SCAR=0.0
43
44
45 4 our $VERSION = '3.00';
46
47
48 =head1 METHODS & ATTRIBUTES
49
50 =head2 Arguments for L<ETL::Pipeline/input>
51
52 B<ETL::Pipeline::Input::File::List> accepts any of the tests provided by
53 L<Path::Iterator::Rule>. The value of the argument is passed directly into the
54 test. For boolean tests (e.g. readable, exists, etc.), pass an C<undef> value.
55
56 B<ETL::Pipeline::Input::File> automatically applies the C<file> filter. Do not
57 pass C<file> through L<ETL::Pipeline/input>.
58
59 C<iname> is the most common one that I use. It matches the file name, supports
60 wildcards and regular expressions, and is case insensitive.
61
62 # Search using a regular expression...
63 $etl->input( 'XmlFiles', iname => qr/\.xml$/ );
64
65 # Search using a file glob...
66 $etl->input( 'XmlFiles', iname => '*.xml' );
67
68 =cut
69
70 # BUILD in the consuming class will override this one. I add a fake BUILD in
71 # case the class doesn't have one. The method modifier then runs the code to
72 # extract search criteria from the constructor arguments. The modifier will
73 # run even if the consuming class has its own BUILD.
74 # https://www.perlmonks.org/?node_id=837369
75 16 ▶sub BUILD {}
Subroutine BUILD: called CC=1 SCAR=0.0
Pod undocumented
76
77 16 ▶after 'BUILD' => sub {
Subroutine __ANON__: called CC=4 SCAR=13.9
78 16 my $self = shift;
79 16 my $arguments = shift;
80
81 16 ▶ while (my ($name, $value) = each %$arguments) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %$arguments) 32 16 48
82 32 ▶ $self->_add_criteria( $name, $value )
Branch: 1 branch
BranchTrueFalseTotal
if $name ne "file" and "Path::Class::Rule"->can($name) 14 18 32
Condition66%2/3
$name ne "file" and "Path::Class::Rule"->can($name)
!ll&&!rl&&r
01814
MC/DC50%1/2
$name ne "file" and "Path::Class::Rule"->can($name)
$name ne "file" "Path::Class::Rule"->can($name)
Truth table66%2/3
A and B
A= $name ne "file"B= "Path::Class::Rule"->can($name)
ABresult
0X0
100
111
83 if $name ne 'file' && Path::Class::Rule->can( $name );
84 }
85 4 };
86
87
88 =head3 path
89
90 L<Path::Class::File> object for the currently selected file. This is first file
91 that matches the criteria. When you call L</next_path>, it finds the next match
92 and sets B<path>.
93
94 So B<path> always points to the current file. It should be used by your input
95 source class as the file name.
96
97 # Inside the input source class...
98 while ($self->next_path( $etl )) {
99 open my $io, '<', $self->path;
100 ...
101 }
102
103 C<undef> means no more matches.
104
105 =cut
106
107 4 has 'path' => (
108 coerce => 1,
109 is => 'ro',
110 isa => 'Path::Class::File|Undef',
111 writer => '_set_path',
112 );
113
114
115 =head2 Methods
116
117 =head3 next_path
118
119 Looks for the next match in the list and sets the L</path> attribute. It also
120 returns the matching path. Your input source class should setup a loop calling
121 this method. Inside the loop, process each file.
122
123 B<next_path> takes one parameter - the L<ETL::Pipeline> object. The method
124 matches files in L<ETL::Pipeline/data_in>.
125
126 =cut
127
128 72 ▶sub next_path {
Subroutine next_path: called CC=5 SCAR=16.1
Pod documented
129 72 my ($self, $etl) = @_;
130
131 72 ▶ if ($self->_list_built) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->_list_built) { } 57 15 72
132 # Get the next file from the list. We'll return "undef" if you query
133 # beyond the end of the list.
134 57 $self->_next_file;
135 } else {
136 # Build the list the first time through.
137 15 my $rule = Path::Class::Rule->new->file;
138 15 foreach my $pair ($self->_search_criteria) {
139 14 my $name = $pair->[0];
140 14 my $value = $pair->[1];
141
142 14 eval "\$rule = \$rule->$name( \$value )";
143 14 ▶ croak $@ unless $@ eq '';
Branch: 1 branch
BranchTrueFalseTotal
unless $@ eq "" 0 14 14
144 }
145 15 $self->_matches( $rule->all( $etl->data_in ) );
146 15 $self->_list_built( 1 );
147 }
148
149 # Set "position" to something more readable.
150 72 my $file = $self->_set_path( $self->_file( $self->_file_index ) );
151
152 72 ▶ if (defined $file) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $file) { } 57 15 72
153 57 $self->source( $file->relative( $etl->work_in )->stringify );
154 57 $etl->status( 'INFO', 'Next file' );
155 15 } else { $self->source( '' ); }
156
157 72 return $file;
158 }
159
160
161 #-------------------------------------------------------------------------------
162 # Internal methods and attributes
163
164 # Search criteria for the file list. I capture the criteria from the constructor
165 # but don't build the iterator until the loop kicks off. Since the search
166 # depends on "data_in", this allows the user to setup the pipeline in whatever
167 # order they want and it will do the right thing.
168 has '_criteria' => (
169 16 ▶ default => sub { {} },
Subroutine __ANON__: called CC=1 SCAR=0.0
170 4 handles => {_add_criteria => 'set', _search_criteria => 'kv'},
171 is => 'ro',
172 isa => 'HashRef[Any]',
173 traits => [qw/Hash/],
174 );
175
176
177 # Index into "_file_list" for the current file. This counter is used to loop
178 # through the list by calling "next_path".
179 4 has '_file_index' => (
180 default => 0,
181 handles => {_next_file => 'inc'},
182 is => 'ro',
183 isa => 'Int',
184 traits => [qw/Counter/],
185 );
186
187
188 # List of files that match the search criteria. The list is built at the
189 # beginning of the pipeline. So your pipeline can't add files on the fly.
190 has '_file_list' => (
191 16 ▶ default => sub { [] },
Subroutine __ANON__: called CC=1 SCAR=0.0
192 4 handles => {_file => 'get', _matches => 'push'},
193 is => 'ro',
194 isa => 'ArrayRef[Any]',
195 traits => [qw/Array/],
196 );
197
198
199 # Since the list always exists, I needed a way to tell the difference between
200 # "no matches" and "not built yet". That way, "next_record" can build the list
201 # on the first pass.
202 4 has '_list_built' => (
203 default => 0,
204 is => 'rw',
205 isa => 'Bool',
206 );
207
208
209 =head1 SEE ALSO
210
211 L<ETL::Pipeline>, L<ETL::Pipeline::Input>, L<Path::Class::File>,
212 L<Path::Class::Rule>, L<Path::Iterator::Rule>
213
214 =cut
215
216 =head1 AUTHOR
217
218 Robert Wohlfarth <robert.j.wohlfarth@vumc.org>
219
220 =head1 LICENSE
221
222 Copyright 2021 (c) Vanderbilt University Medical Center
223
224 This program is free software; you can redistribute it and/or modify it under
225 the same terms as Perl itself.
226
227 =cut
228
229 4 ▶no Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
230
231 # Required by Perl to load the module.
232 4 1;
« Pipeline/Input/JsonFiles.pm ↑ Summary Pipeline/Input/XmlFiles.pm »