Importer/getJSON.pm

filter: Statementstmt 89.6% 121 / 135 Branchbran 71.1% 37 / 52 Conditioncond 45.0% 9 / 20 MC/DCmcdc 25.0% 3 / 12 Subroutinesub 93.3% 28 / 30 Podpod 100% 4 / 4 totaltotal 79.8% 202 / 253 CC 35 SCAR 20.4
CC
35
Cov
81%
request
21.7
construct_url
10.4
_trigger_cache
7.7
CRAP 7.7
↑ Summary Fix/get_json.pm »
1 package Catmandu::Importer::getJSON;
2  
3 4 our $VERSION = '0.52';
4 4 our $CACHE;
5  
6 4 ▶use Catmandu::Sane;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 4 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 4 ▶use JSON;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 4 ▶use Furl;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 4 ▶use Scalar::Util qw(blessed);
Subroutine BEGIN: called CC=1 SCAR=0.0
11 4 ▶use URI::Template;
Subroutine BEGIN: called CC=1 SCAR=0.0
12  
13 4 with 'Catmandu::Importer';
14  
15 has url => (
16 is => 'rw',
17 8 ▶ trigger => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
18 8 $_[0]->{url} = _url_template_or_url( $_[1] );
19 }
20 4 );
21  
22 4 has from => ( is => 'ro' );
23 11 ▶has timeout => ( is => 'ro', default => sub { 10 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
24 4 has agent => ( is => 'ro' );
25 4 has proxy => ( is => 'ro' );
26 4 has dry => ( is => 'ro' );
27 has headers => (
28 is => 'ro',
29 23 ▶ default => sub { [ 'Accept' => 'application/json' ] }
Subroutine __ANON__: called CC=1 SCAR=0.0
30 4 );
31 4 has wait => ( is => 'ro' );
32 4 has cache => ( is => 'ro', trigger => 1 );
33 has client => (
34 is => 'ro',
35 lazy => 1,
36 0 ▶ builder => sub {
Subroutine _build_client: not called CC=1 SCAR=6.9
Subroutine __ANON__: not called CC=1 SCAR=6.9
37 Furl->new(
38 0 map { $_ => $_[0]->{$_} } grep { defined $_[0]->{$_} }
39 qw(timeout agent proxy),
40 );
41 }
42 4 );
43 23 ▶has json => ( is => 'ro', default => sub { JSON->new->utf8(1) } );
Subroutine __ANON__: called CC=1 SCAR=0.0
44 4 has time => ( is => 'rw' );
45 22 ▶has warn => ( is => 'ro', default => sub { 0 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
46  
47 11 ▶sub _url_template_or_url {
Subroutine _url_template_or_url: called CC=4 SCAR=14.0
48 11 my ($url) = @_;
49  
50 11 ▶ if ( !blessed $url) {
Branch: 1 branch
BranchTrueFalseTotal
unless (builtin::blessed($url)) 11 0 11
51 11 $url = URI::Template->new($url);
52 }
53  
54 11 ▶ if ( $url->isa('URI::Template') ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($url->isa("URI::Template")) 11 0 11
55 11 ▶ unless ( my @variables = $url->variables ) {
Branch: 1 branch
BranchTrueFalseTotal
unless (my(@variables) = $url->variables) 5 6 11
56 5 $url = URI->new("$url");
57 }
58 }
59 11 return $url;
60 }
61  
62 {
63  
64 package Importer::getJSON::MemoryCache;
65 4 ▶ use JSON;
Subroutine BEGIN: called CC=1 SCAR=0.0
66 4 our $JSON = JSON->new->utf8;
67 4 ▶ sub new { bless {}, $_[0] }
Subroutine new: called CC=1 SCAR=0.0
68  
69 6 ▶ sub get {
Subroutine get: called CC=1 SCAR=0.0
70 6 eval { $JSON->decode( $_[0]->{ $_[1] } ) };
71 }
72 2 ▶ sub set { $_[0]->{ $_[1] } = ref $_[2] ? $JSON->encode( $_[2] ) : '' }
Subroutine set: called CC=2 SCAR=7.6
Branch: 1 branch
BranchTrueFalseTotal
ref $_[2] ? : 2 0 2
73 }
74 4 $CACHE = Importer::getJSON::MemoryCache->new;
75  
76 {
77  
78 package Importer::getJSON::FileCache;
79 4 ▶ use JSON;
Subroutine BEGIN: called CC=1 SCAR=0.0
80 4 ▶ use Catmandu::Util qw(read_json);
Subroutine BEGIN: called CC=1 SCAR=0.0
81 4 ▶ use Digest::MD5 qw(md5_hex);
Subroutine BEGIN: called CC=1 SCAR=0.0
82 4 our $JSON = JSON->new->utf8;
83  
84 1 ▶ sub new {
Subroutine new: called CC=1 SCAR=0.0
85 1 my ( $class, $dir ) = @_;
86 1 $dir =~ s{/$}{};
87 1 bless { dir => $dir }, $class;
88 }
89  
90 5 ▶ sub file {
Subroutine file: called CC=1 SCAR=0.0
91 5 my ( $self, $url ) = @_;
92 5 $self->{dir} . '/' . md5_hex( $url->as_string ) . '.json';
93 }
94  
95 3 ▶ sub get {
Subroutine get: called CC=1 SCAR=0.0
96 3 eval { read_json( $_[0]->file( $_[1] ) ) };
97 }
98  
99 2 ▶ sub set {
Subroutine set: called CC=2 SCAR=7.1
100 2 my ( $self, $url, $data ) = @_;
101 2 open my $fh, ">", $self->file($url);
102 2 ▶ print $fh ( ref $data ? $JSON->encode($data) : '' );
Branch: 1 branch
BranchTrueFalseTotal
ref $data ? : 2 0 2
103 }
104 }
105  
106 4 ▶sub _trigger_cache {
Subroutine _trigger_cache: called CC=7 SCAR=20.4
107 15 my ( $self, $cache ) = @_;
108  
109 15 ▶ if ( blessed $cache and $cache->can('get') and $cache->can('set') ) {
Branch: 3 branches
BranchTrueFalseTotal
if (builtin::blessed($cache) and $cache->can('get') and $cache->can('set')) { } 0 15 15
elsif ($cache and -d $cache) { } 1 14 15
elsif ($cache) { } 2 12 14
Condition33%2/6
builtin::blessed($cache) and $cache->can("get") and $cache->can("set")
!ll&&!rl&&r
1500
builtin::blessed($cache) and $cache->can("get")
!ll&&!rl&&r
1500
MC/DC0%0/3
builtin::blessed($cache) and $cache->can("get") and $cache->can("set")
builtin::blessed($cache) $cache->can("get") $cache->can("set")
Truth table25%1/4
A and B and C
A= builtin::blessed($cache)B= $cache->can("get")C= $cache->can("set")
ABCresult
0XX0
10X0
1100
1111
110  
111 # use cache object
112 }
113 ▶ elsif ( $cache and -d $cache ) {
Condition100%3/3
$cache and -d $cache
!ll&&!rl&&r
1221
MC/DC100%2/2
$cache and -d $cache
$cache -d $cache
Truth table100%3/3
A and B
A= $cacheB= -d $cache
ABresult
0X0
100
111
114 1 $cache = Importer::getJSON::FileCache->new($cache);
115 }
116 elsif ($cache) {
117 2 $cache = $CACHE;
118 }
119  
120 15 $self->{cache} = $cache;
121 }
122  
123 sub generator {
124 my ($self) = @_;
125  
126 if ( $self->from ) {
127 return sub {
128 state $data = do {
129 my $r = $self->request( $self->from );
130 ( ref $r // '' ) eq 'ARRAY' ? $r : [$r];
131 };
132 return shift @$data;
133 }
134 }
135  
136 sub {
137 state $fh = $self->fh;
138 state $data;
139  
140 if ( $data and ref $data eq 'ARRAY' and @$data ) {
141 return shift @$data;
142 }
143  
144 my $url;
145 until ($url) {
146 my $line = <$fh> // return;
147 chomp $line;
148 $line =~ s/^\s+|\s+$//g;
149 next if $line eq ''; # ignore empty lines
150  
151 my $request = eval { $self->request_hook($line) };
152 $url = $self->construct_url($request);
153 if ( !$url ) {
154 warn "failed to construct URL: $line\n" if $self->warn;
155 $self->log->warn("failed to construct URL: $line");
156 }
157 }
158  
159 $data = $self->request($url);
160  
161 return ( ref $data // '' ) eq 'ARRAY' ? shift @$data : $data;
162 }
163 }
164  
165 33 ▶sub request_hook {
Subroutine request_hook: called CC=2 SCAR=6.9
Pod documented
166 33 my ( $self, $line ) = @_;
167 33 ▶ return $line =~ /^\s*{/ ? $self->json->decode($line) : $line;
Branch: 1 branch
BranchTrueFalseTotal
$line =~ /^\s*{/ ? : 7 26 33
168 }
169  
170 29 ▶sub construct_url {
Subroutine construct_url: called CC=10 SCAR=23.4
Pod documented
171 31 my $self = shift;
172 38 ▶ my $url = @_ > 1 ? _url_template_or_url(shift) : $self->url;
Branch: 1 branch
BranchTrueFalseTotal
@_ > 1 ? : 3 26 29
173 29 my $request = shift;
174  
175 # Template or query variables
176 29 ▶ if ( ref $request and not blessed $request) {
Branch: 4 branches
BranchTrueFalseTotal
if (ref $request and not builtin::blessed($request)) { } 9 20 29
elsif (builtin::blessed($request) and $request->isa('URI::URL')) { } 0 20 20
elsif ($request =~ m[^https?://]) { } 15 5 20
elsif ($request =~ m[^/]) { } 4 1 5
Condition66%2/3
ref $request and not builtin::blessed($request)
!ll&&!rl&&r
2009
MC/DC50%1/2
ref $request and not builtin::blessed($request)
ref $request not builtin::blessed($request)
Truth table66%2/3
A and not(B)
A= ref $requestB= not builtin::blessed($request)
ABresult
0X0
100
111
177 9 ▶ return unless blessed $url;
Branch: 1 branch
BranchTrueFalseTotal
unless builtin::blessed($url) 0 9 9
178 9 ▶ if ( $url->isa('URI::Template') ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($url->isa('URI::Template')) { } 5 4 9
179 5 $url = $url->process($request);
180 }
181 else {
182 4 $url = $url->clone;
183 4 $url->query_form($request);
184 }
185 9 return $url;
186 }
187 ▶ elsif ( blessed $request and $request->isa('URI::URL') ) {
Condition33%1/3
builtin::blessed($request) and $request->isa("URI::URL")
!ll&&!rl&&r
2000
MC/DC0%0/2
builtin::blessed($request) and $request->isa("URI::URL")
builtin::blessed($request) $request->isa("URI::URL")
Truth table33%1/3
A and B
A= builtin::blessed($request)B= $request->isa("URI::URL")
ABresult
0X0
100
111
188 0 return $request;
189 }
190 elsif ( $request =~ /^https?:\/\// ) { # plain URL
191 15 return URI->new($request);
192 }
193 elsif ( $request =~ /^\// ) { # URL path (and optional query)
194 4 $url = "$url";
195 4 $url =~ s{/$}{};
196 4 $request =~ s{\s+$}{};
197 4 return URI->new( $url . $request );
198 }
199  
200 1 return;
201 }
202  
203 34 ▶sub request {
Subroutine request: called CC=13 SCAR=30.8
Pod documented
204 34 my ( $self, $url ) = @_;
205  
206 34 $self->log->debug($url);
207  
208 34 my $json = '';
209  
210 34 ▶ if ( $self->dry ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->dry) 13 21 34
211 13 return { url => "$url" };
212 }
213  
214 21 ▶ if ( $self->cache ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->cache) 9 12 21
215 9 $json = $self->cache->get($url);
216 9 ▶ if ( defined $json ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $json) 5 4 9
217 5 ▶ return ref $json ? $json : undef;
Branch: 1 branch
BranchTrueFalseTotal
ref $json ? : 5 0 5
218 }
219 }
220  
221 16 ▶ if ( $self->wait and $self->time ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->wait and $self->time) 0 16 16
Condition33%1/3
$self->wait and $self->time
!ll&&!rl&&r
1600
MC/DC0%0/2
$self->wait and $self->time
$self->wait $self->time
Truth table33%1/3
A and B
A= $self->waitB= $self->time
ABresult
0X0
100
111
222 0 ▶ my $elapsed = ( $self->time // time ) - time;
Condition0%0/2
$self->time // CORE::time
l!l
00
MC/DC0%0/1
$self->time // CORE::time
$self->time
Truth table0%0/2
A
A= $self->time
Aresult
11
00
223 0 sleep( $self->wait - $elapsed );
224 }
225 16 $self->time(time);
226  
227 16 my $response = $self->client->get( $url, $self->headers );
228 16 ▶ if ( $response->is_success ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($response->is_success) { } 16 0 16
229 16 my $content = $response->decoded_content;
230 16 my $data = $self->json->decode($content);
231 16 $json = $self->response_hook($data);
232 }
233 else {
234 0 ▶ warn "request failed: $url\n" if $self->warn;
Branch: 1 branch
BranchTrueFalseTotal
if $self->warn 0 0 0
235 0 $self->log->warn("request failed: $url");
236 0 ▶ if ( $response->status =~ /^4/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($response->status =~ /^4/) { } 0 0 0
237 0 $json = '';
238 }
239 else {
240 0 return;
241 }
242 }
243  
244 16 ▶ if ( $self->cache ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->cache) 4 12 16
245 4 $self->cache->set( $url, $json );
246 }
247  
248 16 ▶ return ref $json ? $json : undef;
Branch: 1 branch
BranchTrueFalseTotal
ref $json ? : 16 0 16
249 }
250  
251 16 ▶sub response_hook { $_[1] }
Subroutine response_hook: called CC=1 SCAR=0.0
Pod documented
252  
253 4 1;
254 __END__
↑ Summary Fix/get_json.pm »