InternetData/Database.pm

filter: Statementstmt 100% 137 / 137 Branchbran 86.1% 31 / 36 Conditioncond 75.0% 15 / 20 MC/DCmcdc 61.5% 8 / 13 Subroutinesub 100% 39 / 39 Podpod 50.0% 7 / 14 totaltotal 91.5% 237 / 259 CC 26 SCAR 9.3
CC
26
Cov
95%
_assert_database
6.0
download_url_p
5.0
download_p
4.0
CRAP 2.5
« InternetData/Result.pm ↑ Summary InternetData/OauthError.pm »
1 package InternetData::Database;
2  
3 8 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 8 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 8 ▶use Carp ();
Subroutine BEGIN: called CC=1 SCAR=0.0
7  
8 8 ▶use InternetData::Error;
Subroutine BEGIN: called CC=1 SCAR=0.0
9  
10 8 our $VERSION = '1.16.0';
11  
12 # The formats a database is published in. Anything else is refused before it
13 # reaches the API, whose 400 would cost a round trip and name nothing to act on.
14 8 ▶use constant FORMATS => qw(csvgz mmdb);
Subroutine BEGIN: called CC=1 SCAR=0.0
15 8 my %FORMAT = map { $_ => 1 } FORMATS;
16  
17 # The values `list` reports for standing and license_type, so a caller can branch
18 # on each without spelling the list. license_type is undef for an unlicensed family.
19 8 ▶use constant STANDINGS => qw(licensed expired unlicensed);
Subroutine BEGIN: called CC=1 SCAR=0.0
20 8 ▶use constant LICENSE_TYPES => qw(evaluation standard redistribute);
Subroutine BEGIN: called CC=1 SCAR=0.0
21  
22 # Every database this organization may see, with where each one stands.
23 #
24 # NOT only the licensed ones: `standing` says whether a database is yours today,
25 # was, or has never been bought, so a caller can see what else is published
26 # without a sales email.
27 #
28 # What comes back is the SERVER's answer for THIS key and nothing else, so a
29 # listing held from one key is not an answer for another. Nothing here is cached
30 # for exactly that reason.
31 228 ▶sub list {
Subroutine list: called CC=1 SCAR=0.0
Pod documented
32 228 my $self = shift;
33 228 $self->_assert_blocking_ok('list');
34 227 return $self->{client}->_wait($self->list_p(@_));
35 }
36  
37 229 ▶sub list_p {
Subroutine list_p: called CC=1 SCAR=0.0
Pod undocumented
38 229 my ($self, %options) = @_;
39 229 return $self->_body_p('list', \%options, '/api/v2/database/list');
40 }
41  
42 # What is inside one database: schema, sample rows, row count and per-format
43 # sizes. Poll it to decide whether today's build is worth fetching, and read
44 # `$meta->{size}{$format}` to size a transfer before starting it.
45 54 ▶sub metadata {
Subroutine metadata: called CC=1 SCAR=0.0
Pod documented
46 54 my $self = shift;
47 54 $self->_assert_blocking_ok('metadata');
48 54 return $self->{client}->_wait($self->metadata_p(@_));
49 }
50  
51 54 ▶sub metadata_p {
Subroutine metadata_p: called CC=3 SCAR=11.0
Pod undocumented
52 54 my ($self, $id, %options) = @_;
53 54 ▶ Carp::croak('database->metadata: expected a database ID') if !defined $id || !length $id;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $id and length $id 1 53 54
Condition66%2/3
defined $id and length $id
!ll&&!rl&&r
0153
MC/DC50%1/2
defined $id and length $id
defined $id length $id
Truth table66%2/3
A and B
A= defined $idB= length $id
ABresult
0X0
100
111
54 53 return $self->_body_p('metadata', \%options, '/api/v2/database/metadata', id => $id);
55 }
56  
57 # The digests published alongside one file.
58 #
59 # Returns the WHOLE set rather than one algorithm: which digests a database
60 # publishes is the API's choice, not ours, and picking one here is how a caller
61 # ends up holding undef against a perfectly healthy API.
62 23 ▶sub checksums {
Subroutine checksums: called CC=1 SCAR=0.0
Pod documented
63 23 my $self = shift;
64 23 $self->_assert_blocking_ok('checksums');
65 23 return $self->{client}->_wait($self->checksums_p(@_));
66 }
67  
68 23 ▶sub checksums_p {
Subroutine checksums_p: called CC=1 SCAR=0.0
Pod undocumented
69 23 my ($self, $id, $format, %options) = @_;
70 23 _assert_database('checksums', $id, $format);
71 19 return $self->_body_p(
72 'checksums', \%options, '/api/v2/database/checksum', id => $id, format => $format,
73 );
74 }
75  
76 # Your organization's recent download attempts, newest first, refusals included:
77 # a denial is what answers "it stopped working", and its absence answers nothing.
78 48 ▶sub downloads {
Subroutine downloads: called CC=1 SCAR=0.0
Pod documented
79 48 my $self = shift;
80 48 $self->_assert_blocking_ok('downloads');
81 48 return $self->{client}->_wait($self->downloads_p(@_));
82 }
83  
84 49 ▶sub downloads_p {
Subroutine downloads_p: called CC=2 SCAR=6.9
Pod undocumented
85 49 my ($self, %options) = @_;
86 49 my $limit = delete $options{limit};
87 49 ▶ return $self->_body_p(
Branch: 1 branch
BranchTrueFalseTotal
defined $limit ? : 1 48 49
88 'downloads', \%options, '/api/v2/database/downloads',
89 defined $limit ? (limit => $limit) : (),
90 );
91 }
92  
93 # The time-limited URL for one file.
94 #
95 # The API answers 302 and this returns the Location without following it, so the
96 # caller decides how to transfer a file that can run to gigabytes. The link
97 # carries its own authorization, which is what makes it safe to hand to another
98 # process: it names no credential of yours. The link only has to be valid when a
99 # transfer starts, so one already running isn't cut off when it lapses.
100 43 ▶sub download_url {
Subroutine download_url: called CC=1 SCAR=0.0
Pod documented
101 43 my $self = shift;
102 43 $self->_assert_blocking_ok('download_url');
103 43 return $self->{client}->_wait($self->download_url_p(@_));
104 }
105  
106 63 ▶sub download_url_p {
Subroutine download_url_p: called CC=5 SCAR=16.1
Pod undocumented
107 63 my ($self, $id, $format, %options) = @_;
108 63 _assert_database('download_url', $id, $format);
109 59 my $client = $self->{client};
110 59 $client->_check_options('database->download_url', \%options, 'build', 'retries', 'timeout');
111 50 my $build = $options{build};
112 50 ▶ Carp::croak("InternetData::database->download_url: build must be a build's name, a string")
Branch: 1 branch
BranchTrueFalseTotal
if ref $build 2 48 50
113 if ref $build;
114 48 ▶ my $url = $client->_url('/api/v2/database/download', id => $id, format => $format,
Branch: 1 branch
BranchTrueFalseTotal
defined $build && length $build ? : 4 44 48
Condition100%3/3
defined $build and length $build
!ll&&!rl&&r
4314
MC/DC100%2/2
defined $build and length $build
defined $build length $build
Truth table100%3/3
A and B
A= defined $buildB= length $build
ABresult
0X0
100
111
115 defined $build && length $build ? (build => $build) : ());
116 48 ▶ my $retries = defined $options{retries} ? $options{retries} : $client->{retries};
Branch: 1 branch
BranchTrueFalseTotal
defined $options{'retries'} ? : 0 48 48
117 52 ▶ return $client->_retry_p($retries, sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
118 50 ▶ $client->_get_p($url, $options{timeout})->then(sub {
Subroutine __ANON__: called CC=3 SCAR=11.0
119 50 my $res = shift->res;
120 50 ▶ return _location($res) if $res->code == 302;
Branch: 1 branch
BranchTrueFalseTotal
if $res->code == 302 26 24 50
121 # A 2xx here means the user agent followed the redirect and read the
122 # database into memory. Naming the cause beats reporting a shape
123 # mismatch a caller cannot act on.
124 24 ▶ die InternetData::Error->new(
Branch: 1 branch
BranchTrueFalseTotal
if $res->is_success 3 21 24
125 kind => 'server_error', status => $res->code,
126 message => 'expected a redirect to object storage but got '
127 . $res->code . '; the user agent must not follow redirects',
128 ) if $res->is_success;
129 21 die InternetData::Error->from_response($res->code, $res->headers, $res->json);
130 52 });
131 48 });
132 }
133  
134 # Stream one file to a path, and return the bytes written.
135 26 ▶sub download {
Subroutine download: called CC=1 SCAR=0.0
Pod documented
136 26 my $self = shift;
137 26 $self->_assert_blocking_ok('download');
138 26 return $self->{client}->_wait($self->download_p(@_));
139 }
140  
141 27 ▶sub download_p {
Subroutine download_p: called CC=4 SCAR=13.9
Pod undocumented
142 27 my ($self, $id, $format, $path, %options) = @_;
143 # Everything a caller can get wrong is refused before the file is opened, so
144 # a mistyped ID cannot leave a stray .part behind.
145 27 _assert_database('download', $id, $format);
146 23 $self->{client}->_check_options('database->download', \%options, 'build', 'retries');
147 15 ▶ Carp::croak('database->download: expected a destination path')
Branch: 1 branch
BranchTrueFalseTotal
unless defined $path and length $path 1 14 15
Condition66%2/3
defined $path and length $path
!ll&&!rl&&r
1014
MC/DC50%1/2
defined $path and length $path
defined $path length $path
Truth table66%2/3
A and B
A= defined $pathB= length $path
ABresult
0X0
100
111
148 if !defined $path || !length $path;
149  
150 # The bytes land beside the destination and are renamed into place, so a
151 # transfer that dies half way leaves no short file that reads as a whole
152 # database. Opened BEFORE the request: an unwritable path costs no quota.
153 14 my $partial = "$path.part";
154 14 ▶ open my $handle, '>', $partial
Branch: 1 branch
BranchTrueFalseTotal
open my $handle, ">", $partial or Carp::croak("database->download: cannot open ${partial}: $!") 13 1 14
155 or Carp::croak("database->download: cannot open $partial: $!");
156 13 binmode $handle;
157  
158 205 ▶ return $self->_transfer_p('download', $id, $format, \%options, sub {
Subroutine __ANON__: called CC=2 SCAR=7.2
159 # A failure writing is the caller's to read rather than ours to retry: a
160 # full disk and a reset socket are different problems.
161 205 ▶ print {$handle} $_[0] or die "could not write the database to $partial: $!";
Branch: 1 branch
BranchTrueFalseTotal
print {$handle;} $_[0] or die "could not write the database to ${partial}: $!" 205 0 205
162 10 ▶ })->then(sub {
Subroutine __ANON__: called CC=3 SCAR=11.4
163 10 my $written = shift;
164 10 ▶ close $handle or die "could not write the database to $partial: $!";
Branch: 1 branch
BranchTrueFalseTotal
close $handle or die "could not write the database to ${partial}: $!" 10 0 10
165 10 ▶ rename $partial, $path or die "could not move the database into place at $path: $!";
Branch: 1 branch
BranchTrueFalseTotal
rename $partial, $path or die "could not move the database into place at ${path}: $!" 10 0 10
166 10 return $written;
167 3 ▶ })->catch(sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
168 3 my $error = shift;
169 3 close $handle;
170 3 unlink $partial;
171 3 die $error;
172 13 });
173 }
174  
175 # One file's bytes, in memory.
176 18 ▶sub download_bytes {
Subroutine download_bytes: called CC=1 SCAR=0.0
Pod documented
177 18 my $self = shift;
178 18 $self->_assert_blocking_ok('download_bytes');
179 18 return $self->{client}->_wait($self->download_bytes_p(@_));
180 }
181  
182 19 ▶sub download_bytes_p {
Subroutine download_bytes_p: called CC=1 SCAR=0.0
Pod undocumented
183 19 my ($self, $id, $format, %options) = @_;
184 19 _assert_database('download_bytes', $id, $format);
185 15 my $bytes = '';
186 4 ▶ return $self->_transfer_p('download_bytes', $id, $format, \%options, sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
187 4 $bytes .= $_[0];
188 15 ▶ })->then(sub { return $bytes });
Subroutine __ANON__: called CC=1 SCAR=0.0
189 }
190  
191 162 ▶sub _new {
Subroutine _new: called CC=1 SCAR=0.0
192 162 my ($class, $client) = @_;
193 162 return bless { client => $client }, $class;
194 }
195  
196 # The 302 is followed as a SECOND request. `retries` covers it only until the
197 # first byte reaches the caller: object storage failing before that is retried
198 # like any 5xx, but a transfer that dies part way is not repeated, or the second
199 # copy would append to the bytes already written. A per-call `timeout` is refused
200 # rather than spent on the link call alone, where a caller would read it as
201 # bounding the transfer, which nothing does.
202 28 ▶sub _transfer_p {
Subroutine _transfer_p: called CC=2 SCAR=6.9
203 28 my ($self, $method, $id, $format, $options, $on_chunk) = @_;
204 28 my $client = $self->{client};
205 28 $client->_check_options("database->$method", $options, 'build', 'retries');
206 19 ▶ my $retries = defined $options->{retries} ? $options->{retries} : $client->{retries};
Branch: 1 branch
BranchTrueFalseTotal
defined $options->{'retries'} ? : 0 19 19
207 19 my $delivered = 0;
208 209 ▶ my $count = sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
209 209 $delivered += length $_[0];
210 209 $on_chunk->(@_);
211 19 };
212 16 ▶ return $self->download_url_p($id, $format, %$options)->then(sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
213 16 my $url = shift;
214 16 ▶ return $client->_retry_p($retries, sub { $client->_stream_p($url, $count) }, sub { !$delivered });
Subroutine __ANON__: called CC=1 SCAR=0.0
Subroutine __ANON__: called CC=1 SCAR=0.0
215 19 });
216 }
217  
218 # The member each call returns from one level down, or none for a call that
219 # returns the whole answer, and the shape the whole answer must have. The shapes
220 # are the pinned spec's schemas: a member it requires and never sends as null is
221 # required, a nullable or optional one may be absent or null, and every one
222 # present has the spec's type at every depth, so a version served as "3" fails
223 # the call rather than reaching the caller as a string. A `date` or
224 # `date-time` member is RFC 3339's, as its `format` says, on a day that exists.
225 # `standing`, `license_type`, `format` and `outcome` stay strings, open sets as
226 # the spec documents them, and a sample row is a hash whatever its columns.
227 8 my $VERSION_SHAPE = [fields => [
228 [id => ['string'], 1], [version => ['integer'], 1], [summary => ['string'], 1],
229 [formats => [list => ['string']], 1], [sample_formats => [list => ['string']]],
230 ]];
231 8 my $DATABASE_SHAPE = [fields => [
232 [base => ['string'], 1], [name => ['string'], 1], [summary => ['string'], 1],
233 [standing => ['string'], 1], [open => ['boolean'], 1], [license_type => ['string']],
234 [starts => ['date-time']], [expires => ['date-time']], [renews_at => ['date-time']],
235 [notice_due_at => ['date-time']], [versions => [list => $VERSION_SHAPE], 1],
236 ]];
237 8 my $COLUMN_SHAPE = [fields => [[name => ['string'], 1], [type => ['string'], 1], [description => ['string']]]];
238 8 my $METADATA_SHAPE = [fields => [
239 [id => ['string'], 1], [update_freq => ['string']], [updated => ['date'], 1],
240 [entries => ['integer'], 1], [schema => [map_of => [list => $COLUMN_SHAPE]], 1],
241 [sample => [map_of => [list => ['object']]]], [size => [map_of => ['integer']], 1],
242 [sample_size => [map_of => ['integer']]], [sample_entries => ['integer']],
243 ]];
244 8 my $CHECKSUMS_SHAPE = [fields => [map { [$_ => ['string'], 1] } qw(md5 sha1 sha256 sha512)]];
245 8 my $DOWNLOAD_SHAPE = [fields => [
246 [dataset_id => ['string'], 1], [format => ['string'], 1], [outcome => ['string'], 1],
247 [sample => ['boolean'], 1], [open => ['boolean'], 1], [bytes => ['integer']],
248 [http_status => ['integer']], [apikey_id => ['string']], [client_ip => ['string']],
249 [user_agent => ['string']], [created => ['date-time'], 1],
250 ]];
251 8 my %UNWRAP = (
252 list => [databases => [fields => [[databases => [list => $DATABASE_SHAPE], 1]]]],
253 metadata => [undef, $METADATA_SHAPE],
254 checksums => [checksums => [fields => [
255 [id => ['string'], 1], [format => ['string'], 1], [checksums => $CHECKSUMS_SHAPE, 1],
256 ]]],
257 downloads => [downloads => [fields => [[downloads => [list => $DOWNLOAD_SHAPE], 1]]]],
258 );
259  
260 350 ▶sub _body_p {
Subroutine _body_p: called CC=3 SCAR=11.0
261 350 my ($self, $method, $options, $path, @query) = @_;
262 350 my $client = $self->{client};
263 350 $client->_check_options("database->$method", $options, 'retries', 'timeout');
264 332 my $url = $client->_url($path, @query);
265 332 ▶ my $retries = defined $options->{retries} ? $options->{retries} : $client->{retries};
Branch: 1 branch
BranchTrueFalseTotal
defined $options->{'retries'} ? : 2 330 332
266 332 ▶ my @unwrap = @{ $UNWRAP{$method} || [] };
Condition50%1/2
$UNWRAP{$method} || []
l!l
3320
MC/DC0%0/1
$UNWRAP{$method} || []
$UNWRAP{$method}
Truth table50%1/2
A
A= $UNWRAP{$method}
Aresult
11
00
267 332 ▶ return $client->_retry_p($retries, sub { $client->_json_p($url, undef, $options->{timeout}, @unwrap) });
Subroutine __ANON__: called CC=1 SCAR=0.0
268 }
269  
270 132 ▶sub _assert_database {
Subroutine _assert_database: called CC=6 SCAR=17.9
271 132 my ($method, $id, $format) = @_;
272 132 ▶ Carp::croak("database->$method: expected a database ID") if !defined $id || !length $id;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $id and length $id 2 130 132
Condition100%3/3
defined $id and length $id
!ll&&!rl&&r
11130
MC/DC100%2/2
defined $id and length $id
defined $id length $id
Truth table100%3/3
A and B
A= defined $idB= length $id
ABresult
0X0
100
111
273 130 ▶ Carp::croak("database->$method: expected a format") if !defined $format || !length $format;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $format and length $format 2 128 130
Condition66%2/3
defined $format and length $format
!ll&&!rl&&r
20128
MC/DC50%1/2
defined $format and length $format
defined $format length $format
Truth table66%2/3
A and B
A= defined $formatB= length $format
ABresult
0X0
100
111
274 Carp::croak("database->$method: '$format' is not a published format; expected one of "
275 128 ▶ . join(', ', FORMATS)) unless $FORMAT{$format};
Branch: 1 branch
BranchTrueFalseTotal
unless $FORMAT{$format} 12 116 128
276 }
277  
278 26 ▶sub _location {
Subroutine _location: called CC=3 SCAR=11.0
279 26 my ($res) = @_;
280 26 my $location = $res->headers->location;
281 26 ▶ die InternetData::Error->new(
Branch: 1 branch
BranchTrueFalseTotal
unless defined $location and length $location 3 23 26
Condition66%2/3
defined $location and length $location
!ll&&!rl&&r
3023
MC/DC50%1/2
defined $location and length $location
defined $location length $location
Truth table66%2/3
A and B
A= defined $locationB= length $location
ABresult
0X0
100
111
282 kind => 'server_error', status => $res->code,
283 message => 'the API redirected without a Location header',
284 ) if !defined $location || !length $location;
285 23 return $location;
286 }
287  
288 440 ▶sub _assert_blocking_ok {
Subroutine _assert_blocking_ok: called CC=1 SCAR=0.0
289 440 my ($self, $method) = @_;
290 440 $self->{client}->_assert_blocking_ok("database->$method");
291 }
292  
293 8 1;
294  
295 __END__
« InternetData/Result.pm ↑ Summary InternetData/OauthError.pm »