blib/lib/Catmandu/Importer.pm

filter: Statementstmt 93.3% 126 / 135 Branchbran 80.0% 32 / 40 Conditioncond 94.1% 16 / 17 MC/DCmcdc 90.0% 9 / 10 Subroutinesub 90.4% 19 / 21 Podpod 0.0% 0 / 2 totaltotal 89.7% 202 / 225 CC 30 SCAR 18.9
CC
30
Cov
91%
_build_fh
12.1
_build_file
11.0
_build_http_client
6.1
CRAP 6.6
« blib/lib/Catmandu/Cmd/run.pm ↑ Summary blib/lib/Catmandu/Validator.pm »
1 package Catmandu::Importer;
2  
3 29 ▶use Catmandu::Sane;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 29 our $VERSION = '1.0201_02';
6  
7 29 ▶use Catmandu::Util qw(io data_at is_value is_string is_array_ref is_hash_ref);
Subroutine BEGIN: called CC=1 SCAR=0.0
8 29 ▶use LWP::UserAgent;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 29 ▶use LWP::UserAgent::Determined;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 29 ▶use HTTP::Request ();
Subroutine BEGIN: called CC=1 SCAR=0.0
11 29 ▶use URI ();
Subroutine BEGIN: called CC=1 SCAR=0.0
12 29 ▶use URI::Template ();
Subroutine BEGIN: called CC=1 SCAR=0.0
13 29 ▶use Moo::Role;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 29 ▶use namespace::clean;
Subroutine BEGIN: called CC=1 SCAR=0.0
15  
16 29 with 'Catmandu::Logger';
17 29 with 'Catmandu::Iterable';
18 29 with 'Catmandu::Fixable';
19 29 with 'Catmandu::Serializer';
20  
21 around generator => sub {
22 61 ▶ my ($orig, $self) = @_;
Subroutine __ANON__: called CC=3 SCAR=11.0
23 61 my $generator = $orig->($self);
24  
25 61 ▶ if (my $fixer = $self->_fixer) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $fixer = $self->_fixer) 9 52 61
26 9 $generator = $fixer->fix($generator);
27 }
28  
29 61 ▶ if (defined(my $path = $self->data_path)) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $path = $self->data_path)) 2 59 61
30 return sub {
31 10 ▶ state @buf;
Subroutine __ANON__: called CC=3 SCAR=11.0
32 10 while (1) {
33 14 ▶ return shift @buf if @buf;
Branch: 1 branch
BranchTrueFalseTotal
if @buf 8 6 14
34  
35 # TODO use something faster than data_at
36 6 ▶ @buf = data_at($path, $generator->() // return);
Condition100%2/2
&$generator() // (return)
l!l
42
MC/DC100%1/1
&$generator() // (return)
&$generator()
Truth table100%2/2
A
A= &$generator()
Aresult
11
00
37 4 next;
38 }
39 2 };
40 }
41  
42 59 $generator;
43 29 };
44  
45 29 has file => (is => 'lazy', init_arg => undef);
46 29 has _file_template =>
47 (is => 'ro', predicate => 'has_file', init_arg => 'file');
48 29 has variables => (is => 'ro', predicate => 1);
49 29 has fh => (is => 'ro', lazy => 1, builder => 1);
50 29 has encoding => (is => 'ro', builder => 1);
51 29 has data_path => (is => 'ro');
52 29 has user_agent => (is => 'ro');
53 29 has http_method => (is => 'lazy');
54 29 has http_headers => (is => 'lazy');
55 29 has http_agent => (is => 'ro', predicate => 1);
56 29 has http_max_redirect => (is => 'ro', predicate => 1);
57 81 ▶has http_timeout => (is => 'ro', default => sub {180}); # LWP default
Subroutine __ANON__: called CC=1 SCAR=0.0
58 81 ▶has http_verify_hostname => (is => 'ro', default => sub {1});
Subroutine __ANON__: called CC=1 SCAR=0.0
59 29 has http_retry => (is => 'ro', predicate => 1);
60 29 has http_timing => (is => 'ro', predicate => 1);
61 29 has http_body => (is => 'ro', predicate => 1);
62 29 has _http_client => (
63 is => 'ro',
64 lazy => 1,
65 builder => '_build_http_client',
66 init_arg => 'user_agent'
67 );
68 29 has ignore_404 => (is => 'ro');
69  
70 sub _build_encoding {
71 76 ▶ ':utf8';
Subroutine _build_encoding: called CC=1 SCAR=0.0
72 }
73  
74 sub _build_file {
75 42 ▶ my ($self) = @_;
Subroutine _build_file: called CC=11 SCAR=24.0
76 42 ▶ return \*STDIN unless $self->has_file;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->has_file 0 42 42
77 42 my $file = $self->_file_template;
78 42 ▶ if (is_string($file) && $self->has_variables) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_string($file) and $self->has_variables) 6 36 42
Condition100%3/3
is_string($file) and $self->has_variables
!ll&&!rl&&r
24126
MC/DC100%2/2
is_string($file) and $self->has_variables
is_string($file) $self->has_variables
Truth table100%3/3
A and B
A= is_string($file)B= $self->has_variables
ABresult
0X0
100
111
79 6 my $template = URI::Template->new($file);
80 6 my $vars = $self->variables;
81 6 ▶ if (is_value($vars)) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_value($vars)) 3 3 6
82 3 $vars = [split ',', $vars];
83 }
84 6 ▶ if (is_array_ref($vars)) {
Branch: 1 branch
BranchTrueFalseTotal
if (Ref::Util::XS::is_plain_arrayref($vars)) 4 2 6
85 4 my @keys = $template->variables;
86 4 my @vals = @$vars;
87 4 $vars = {};
88 4 ▶ $vars->{shift @keys} = shift @vals while @keys && @vals;
Branch: 1 branch
BranchTrueFalseTotal
@keys and @vals and do { $$vars{shift @keys} = shift @vals } 6 4 10
Condition66%2/3
@keys and @vals
!ll&&!rl&&r
406
MC/DC50%1/2
@keys and @vals
@keys @vals
Truth table66%2/3
A and B
A= @keysB= @vals
ABresult
0X0
100
111
89 }
90 6 $file = $template->process_to_string(%$vars);
91 }
92 42 ▶ if (is_string($file) && $file !~ m!^https?://! && !-r $file) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_string($file) and not $file =~ m[^https?://] and not -r $file) 1 41 42
Condition100%6/6
is_string($file) and not $file =~ m[^https?://] and not -r $file
!ll&&!rl&&r
31101
is_string($file) and not $file =~ m[^https?://]
!ll&&!rl&&r
24711
MC/DC100%3/3
is_string($file) and not $file =~ m[^https?://] and not -r $file
is_string($file) not $file =~ m[^https?://] not -r $file
Truth table100%4/4
A and not(B) and not(C)
A= is_string($file)B= not $file =~ m[^https?://]C= not -r $file
ABCresult
0XX0
10X0
1100
1111
93 1 Catmandu::BadArg->throw("file '$file' doesn't exist");
94 }
95 41 $file;
96 }
97  
98 sub _build_fh {
99 39 ▶ my ($self) = @_;
Subroutine _build_fh: called CC=12 SCAR=24.9
100 39 my $file = $self->file;
101 38 my $body;
102 38 ▶ if (is_string($file) && $file =~ m!^https?://!) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_string($file) and $file =~ m[^https?://]) 7 31 38
Condition100%3/3
is_string($file) and $file =~ m[^https?://]
!ll&&!rl&&r
21107
MC/DC100%2/2
is_string($file) and $file =~ m[^https?://]
is_string($file) $file =~ m[^https?://]
Truth table100%3/3
A and B
A= is_string($file)B= $file =~ m[^https?://]
ABresult
0X0
100
111
103 7 my $req = HTTP::Request->new($self->http_method, $file,
104 $self->http_headers);
105  
106 7 ▶ if ($self->has_http_body) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->has_http_body) 3 4 7
107 3 $body = $self->http_body;
108  
109 3 ▶ if (ref $body) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $body) 0 3 3
110 0 $body = $self->serialize($body);
111 }
112  
113 3 ▶ if ($self->has_variables) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->has_variables) 3 0 3
114 3 my $vars = $self->variables;
115 3 ▶ if (is_hash_ref($vars)) { # named variables
Branch: 1 branch
BranchTrueFalseTotal
if (Ref::Util::XS::is_plain_hashref($vars)) { } 1 2 3
116 1 for my $key (keys %$vars) {
117 1 my $var = $vars->{$key};
118 1 $body =~ s/{$key}/$var/;
119 }
120 }
121 else { # positional variables
122 2 ▶ if (is_value($vars)) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_value($vars)) 2 0 2
123 2 $vars = [split ',', $vars];
124 }
125 2 for my $var (@$vars) {
126 6 $body =~ s/{\w+}/$var/;
127 }
128 }
129 }
130  
131 3 $req->content($body);
132 }
133  
134 7 my $res = $self->_http_client->request($req);
135 7 ▶ unless ($res->is_success) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($res->is_success) 1 6 7
136 1 my $res_headers = [];
137 1 for my $header ($res->header_field_names) {
138 1 my $val = $res->header($header);
139 1 push @$res_headers, $header, $val;
140 }
141 Catmandu::HTTPError->throw(
142 {
143 1 code => $res->code,
144 message => $res->status_line,
145 url => $file,
146 method => $self->http_method,
147 request_headers => $self->http_headers,
148 request_body => $body,
149 response_headers => $res_headers,
150 response_body => $res->decoded_content,
151 }
152 );
153 }
154  
155 6 my $content = $res->decoded_content;
156 6 return io(\$content, mode => 'r', binmode => $_[0]->encoding);
157 }
158  
159 31 io($file, mode => 'r', binmode => $_[0]->encoding);
160 }
161  
162 sub _build_http_headers {
163 7 ▶ [];
Subroutine _build_http_headers: called CC=1 SCAR=0.0
164 }
165  
166 sub _build_http_method {
167 4 ▶ 'GET';
Subroutine _build_http_method: called CC=1 SCAR=0.0
168 }
169  
170 sub _build_http_client {
171 1 ▶ my ($self) = @_;
Subroutine _build_http_client: called CC=5 SCAR=18.0
172 1 my $ua;
173 1 ▶ if ($self->has_http_timing) {
Branch: 2 branches
BranchTrueFalseTotal
if ($self->has_http_timing) { } 0 1 1
elsif ($self->has_http_retry) { } 0 1 1
174 0 $ua = LWP::UserAgent::Determined->new;
175 0 $ua->timing($self->http_timing);
176 }
177 elsif ($self->has_http_retry) {
178 0 $ua = LWP::UserAgent::Determined->new;
179 0 $ua->timing(join(',', ($self->http_timeout) x $self->http_retry));
180 }
181 else {
182 1 $ua = LWP::UserAgent->new;
183 1 $ua->timeout($self->http_timeout);
184 }
185 1 ▶ $ua->agent($self->http_agent) if $self->has_http_agent;
Branch: 1 branch
BranchTrueFalseTotal
if $self->has_http_agent 0 1 1
186 1 ▶ $ua->max_redirect($self->http_max_redirect)
Branch: 1 branch
BranchTrueFalseTotal
if $self->has_http_max_redirect 0 1 1
187 if $self->has_http_max_redirect;
188 1 $ua->ssl_opts(verify_hostname => $self->http_verify_hostname);
189 1 $ua->protocols_allowed([qw(http https)]);
190 1 $ua->env_proxy;
191 1 $ua;
192 }
193  
194 sub readline {
195 0 ▶ warnings::warnif("deprecated",
Subroutine readline: not called CC=1 SCAR=6.9
Pod undocumented
196 "readline is deprecated, fh->getline instead");
197 0 $_[0]->fh->getline;
198 }
199  
200 sub readall {
201 0 ▶ warnings::warnif("deprecated",
Subroutine readall: not called CC=1 SCAR=6.9
Pod undocumented
202 "readall is deprecated, join('',fh->getlines) instead");
203 0 join '', $_[0]->fh->getlines;
204 }
205  
206 29 1;
207  
208 __END__
209  
210 =pod
211  
212 =head1 NAME
213  
214 Catmandu::Importer - Namespace for packages that can import
215  
216 =head1 SYNOPSIS
217  
218 # From the command line
219  
220 # JSON is an importer and YAML an exporter
221 $ catmandu convert JSON to YAML < data.json
222  
223 # OAI is an importer and JSON an exporter
224 $ catmandu convert OAI --url http://biblio.ugent.be/oai to JSON
225  
226 # Fetch remote content
227 $ catmandu convert JSON --file http://example.com/data.json to YAML
228
229 # From Perl
230
231 use Catmandu;
232 use Data::Dumper;
233  
234 my $importer = Catmandu->importer('JSON', file => 'data.json');
235  
236 $importer->each(sub {
237 my $item = shift;
238 print Dumper($item);
239 });
240  
241 my $num = $importer->count;
242  
243 my $first_item = $importer->first;
244  
245 # Convert OAI to JSON in Perl
246 my $importer = Catmandu->importer('OAI', url => 'http://biblio.ugent.be/oai');
247 my $exporter = Catmandu->exporter('JSON');
248  
249 $exporter->add_many($importer);
250  
251 =head1 DESCRIPTION
252  
253 A Catmandu::Importer is a Perl package that can generate structured data from
254 sources such as JSON, YAML, XML, RDF or network protocols such as Atom, OAI-PMH,
255 SRU and even DBI databases. Given an Catmandu::Importer a programmer can read
256 data from using one of the many L<Catmandu::Iterable> methods:
257  
258  
259 $importer->to_array;
260 $importer->count;
261 $importer->each(\&callback);
262 $importer->first;
263 $importer->rest;
264 ...etc...
265  
266 Every Catmandu::Importer is also L<Catmandu::Fixable> and thus inherits a 'fix'
267 parameter that can be set in the constructor. When given a 'fix' parameter, then each
268 item returned by the generator will be automatically Fixed using one or
269 more L<Catmandu::Fix>es.
270 E.g.
271
272 my $importer = Catmandu->importer('JSON',fix => ['upcase(title)']);
273 $importer->each( sub {
274 my $item = shift ; # Every $item->{title} is now upcased...
275  
276 });
277  
278 # or via a Fix file
279 my $importer = Catmandu->importer('JSON',fix => ['/my/fixes.txt']);
280 $importer->each( sub {
281 my $item = shift ; # Every $item->{title} is now upcased...
282  
283 });
284  
285 =head1 CONFIGURATION
286  
287 =over
288  
289 =item file
290  
291 Read input from a local file given by its path. If the path looks like a
292 url, the content will be fetched first and then passed to the importer.
293 Alternatively a scalar reference can be passed to read from a string.
294  
295 =item fh
296  
297 Read input from an L<IO::Handle>. If not specified, L<Catmandu::Util::io> is used to
298 create the input stream from the C<file> argument or by using STDIN.
299  
300 =item encoding
301  
302 Binmode of the input stream C<fh>. Set to C<:utf8> by default.
303  
304 =item fix
305  
306 An ARRAY of one or more Fix-es or Fix scripts to be applied to imported items.
307  
308 =item data_path
309  
310 The data at C<data_path> is imported instead of the original data.
311  
312 # given this imported item:
313 {abc => [{a=>1},{b=>2},{c=>3}]}
314 # with data_path 'abc', this item gets imported instead:
315 [{a=>1},{b=>2},{c=>3}]
316 # with data_path 'abc.*', 3 items get imported:
317 {a=>1}
318 {b=>2}
319 {c=>3}
320  
321 =item variables
322  
323 Variables given here will interpolate the C<file> and C<http_body> options. The
324 syntax is the same as L<URI::Template>.
325  
326 # named arguments
327 my $importer = Catmandu->importer('JSON',
328 file => 'http://{server}/{path}',
329 variables => {server => 'biblio.ugent.be', path => 'file.json'},
330 );
331  
332 # positional arguments
333 my $importer = Catmandu->importer('JSON',
334 file => 'http://{server}/{path}',
335 variables => 'biblio.ugent.be,file.json',
336 );
337  
338 # or
339 my $importer = Catmandu->importer('JSON',
340 url => 'http://{server}/{path}',
341 variables => ['biblio.ugent.be','file.json'],
342 );
343  
344 # or via the command line
345 $ catmandu convert JSON --file 'http://{server}/{path}' --variables 'biblio.ugent.be,file.json'
346  
347 =back
348  
349 =head1 HTTP CONFIGURATION
350  
351 These options are only relevant if C<file> is a url. See L<LWP::UserAgent> for details about these options.
352  
353 =over
354  
355 =item http_body
356  
357 Set the GET/POST message body.
358  
359 =item http_method
360  
361 Set the type of HTTP request 'GET', 'POST' , ...
362  
363 =item http_headers
364  
365 A reference to a HTTP::Headers objects.
366  
367 =back
368  
369 =head2 Set an own HTTP client
370  
371 =over
372  
373 =item user_agent(LWP::UserAgent->new(...))
374  
375 Set an own HTTP client
376  
377 =back
378  
379 =head2 Alternative set the parameters of the default client
380  
381 =over
382  
383 =item http_agent
384  
385 A string containing the name of the HTTP client.
386  
387 =item http_max_redirect
388  
389 Maximum number of HTTP redirects allowed.
390  
391 =item http_timeout
392  
393 Maximum execution time.
394  
395 =item http_verify_hostname
396  
397 Verify the SSL certificate.
398  
399 =item http_retry
400  
401 Maximum times to retry the HTTP request if it temporarily fails. Default is not
402 to retry. See L<LWP::User::UserAgent::Determined> for the HTTP status codes
403 that initiate a retry.
404  
405 =item http_timing
406  
407 Maximum times and timeouts to retry the HTTP request if it temporarily fails. Default is not
408 to retry. See L<LWP::User::UserAgent::Determined> for the HTTP status codes
409 that initiate a retry and the format of the timing value.
410  
411 =back
412  
413 =head1 METHODS
414  
415 =head2 first, each, rest , ...
416  
417 See L<Catmandu::Iterable> for all inherited methods.
418  
419 =head1 CODING
420  
421 Create your own importer by creating a Perl package in the Catmandu::Importer namespace that
422 implements C<Catmandu::Importer>. Basically, you need to create a method 'generate' which
423 returns a callback that creates one Perl hash for each call:
424  
425 my $importer = Catmandu::Importer::Hello->new;
426  
427 $importer->generate(); # record
428 $importer->generate(); # next record
429 $importer->generate(); # undef = end of stream
430  
431 Here is an example of a simple C<Hello> importer:
432  
433 package Catmandu::Importer::Hello;
434  
435 use Catmandu::Sane;
436 use Moo;
437  
438 with 'Catmandu::Importer';
439  
440 sub generator {
441 my ($self) = @_;
442 state $fh = $self->fh;
443 my $n = 0;
444 return sub {
445 $self->log->debug("generating record " . ++$n);
446 my $name = $self->fh->readline;
447 return defined $name ? { "hello" => $name } : undef;
448 };
449 }
450  
451 1;
452  
453 This importer can be called via the command line as:
454  
455 $ catmandu convert Hello to JSON < /tmp/names.txt
456 $ catmandu convert Hello to YAML < /tmp/names.txt
457 $ catmandu import Hello to MongoDB --database_name test < /tmp/names.txt
458  
459 Or, via Perl
460  
461 use Catmandu;
462  
463 my $importer = Catmandu->importer('Hello', file => '/tmp/names.txt');
464 $importer->each(sub {
465 my $items = shift;
466 });
467  
468 =head1 SEE ALSO
469  
470 L<Catmandu::Iterable> , L<Catmandu::Fix> ,
471 L<Catmandu::Importer::CSV>, L<Catmandu::Importer::JSON> , L<Catmandu::Importer::YAML>
472  
473 =cut
« blib/lib/Catmandu/Cmd/run.pm ↑ Summary blib/lib/Catmandu/Validator.pm »