Avro/DataFileReader.pm

filter: Statementstmt 86.3% 146 / 169 Branchbran 65.7% 46 / 70 Conditioncond 70.0% 14 / 20 MC/DCmcdc 25.0% 3 / 12 Subroutinesub 93.3% 28 / 30 Podpod 0.0% 0 / 17 totaltotal 74.5% 237 / 318 CC 44 SCAR 15.6
CC
44
Cov
80%
skip
20.0
read_block_header
7.0
eob
6.8
CRAP 4.8
« Avro/Schema.pm ↑ Summary Avro/BinaryDecoder.pm »
1 # Licensed to the Apache Software Foundation (ASF) under one
2 # or more contributor license agreements. See the NOTICE file
3 # distributed with this work for additional information
4 # regarding copyright ownership. The ASF licenses this file
5 # to you under the Apache License, Version 2.0 (the
6 # "License"); you may not use this file except in compliance
7 # with the License. You may obtain a copy of the License at
8 #
9 # https://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing,
12 # software distributed under the License is distributed on an
13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 # KIND, either express or implied. See the License for the
15 # specific language governing permissions and limitations
16 # under the License.
17  
18 package Avro::DataFileReader;
19 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
20 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
21  
22 1 ▶use Object::Tiny qw{
Subroutine BEGIN: called CC=1 SCAR=0.0
23 fh
24 reader_schema
25 sync_marker
26 block_max_size
27 1 };
28  
29 1 ▶use constant MARKER_SIZE => 16;
Subroutine BEGIN: called CC=1 SCAR=0.0
30  
31 # TODO: refuse to read a block more than block_max_size, instead
32 # do partial reads
33  
34 1 ▶use Avro::DataFile;
Subroutine BEGIN: called CC=1 SCAR=0.0
35 1 ▶use Avro::BinaryDecoder;
Subroutine BEGIN: called CC=1 SCAR=0.0
36 1 ▶use Avro::Schema;
Subroutine BEGIN: called CC=1 SCAR=0.0
37 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
38 1 ▶use Compress::Zstd;
Subroutine BEGIN: called CC=1 SCAR=0.0
39 1 ▶use IO::Uncompress::Bunzip2 qw(bunzip2);
Subroutine BEGIN: called CC=1 SCAR=0.0
40 1 ▶use IO::Uncompress::RawInflate ;
Subroutine BEGIN: called CC=1 SCAR=0.0
41 1 ▶use Fcntl();
Subroutine BEGIN: called CC=1 SCAR=0.0
42  
43 1 our $VERSION = '1.12.2';
44  
45 7 ▶sub new {
Subroutine new: called CC=3 SCAR=11.2
Pod undocumented
46 7 my $class = shift;
47 7 my $datafile = $class->SUPER::new(@_);
48  
49 7 my $schema = $datafile->{reader_schema};
50 croak "schema is invalid"
51 7 ▶ if $schema && ! eval { $schema->isa("Avro::Schema") };
Branch: 1 branch
BranchTrueFalseTotal
if $schema and not eval { do { $schema->isa("Avro::Schema") } } 0 7 7
Condition66%2/3
$schema and not eval { do { $schema->isa("Avro::Schema") } }
!ll&&!rl&&r
250
MC/DC0%0/2
$schema and not eval { do { $schema->isa("Avro::Schema") } }
$schema not eval { do { $schema->isa("Avro::Schema") } }
Truth table66%2/3
A and not(B)
A= $schemaB= not eval { do { $schema->isa("Avro::Schema") } }
ABresult
0X0
100
111
52  
53 7 return $datafile;
54 }
55  
56 7 ▶sub codec {
Subroutine codec: called CC=2 SCAR=6.9
Pod undocumented
57 7 my $datafile = shift;
58 7 ▶ return $datafile->metadata->{'avro.codec'} || 'null';
Condition100%2/2
$datafile->metadata->{'avro.codec'} || 'null'
l!l
51
MC/DC100%1/1
$datafile->metadata->{'avro.codec'} || 'null'
$datafile->metadata->{'avro.codec'}
Truth table100%2/2
A
A= $datafile->metadata->{'avro.codec'}
Aresult
11
00
59 }
60  
61 8 ▶sub writer_schema {
Subroutine writer_schema: called CC=2 SCAR=6.9
Pod undocumented
62 8 my $datafile = shift;
63 8 ▶ unless (exists $datafile->{_writer_schema}) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $datafile->{"_writer_schema"}) 6 2 8
64 6 my $json_schema = $datafile->metadata->{'avro.schema'};
65 6 $datafile->{_writer_schema} = Avro::Schema->parse($json_schema);
66 }
67 8 return $datafile->{_writer_schema};
68 }
69  
70 21 ▶sub metadata {
Subroutine metadata: called CC=3 SCAR=11.0
Pod undocumented
71 21 my $datafile = shift;
72 21 ▶ unless (exists $datafile->{_metadata}) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $datafile->{"_metadata"}) 7 14 21
73 7 my $header = $datafile->header;
74 6 ▶ $datafile->{_metadata} = $header->{meta} || {};
Condition50%1/2
$header->{'meta'} || {}
l!l
60
MC/DC0%0/1
$header->{'meta'} || {}
$header->{'meta'}
Truth table50%1/2
A
A= $header->{'meta'}
Aresult
11
00
75 }
76 20 return $datafile->{_metadata};
77 }
78  
79 7 ▶sub header {
Subroutine header: called CC=2 SCAR=7.0
Pod undocumented
80 7 my $datafile = shift;
81 7 ▶ unless (exists $datafile->{_header}) {
Branch: 1 branch
BranchTrueFalseTotal
unless (exists $datafile->{"_header"}) 7 0 7
82 7 $datafile->{_header} = $datafile->read_file_header;
83 }
84  
85 6 return $datafile->{_header};
86 }
87  
88 7 ▶sub read_file_header {
Subroutine read_file_header: called CC=5 SCAR=16.2
Pod undocumented
89 7 my $datafile = shift;
90  
91 my $data = Avro::BinaryDecoder->decode(
92 reader_schema => $Avro::DataFile::HEADER_SCHEMA,
93 writer_schema => $Avro::DataFile::HEADER_SCHEMA,
94 reader => $datafile->{fh},
95 7 );
96 croak "Magic '$data->{magic}' doesn't match"
97 7 ▶ unless $data->{magic} eq Avro::DataFile->AVRO_MAGIC;
Branch: 1 branch
BranchTrueFalseTotal
unless $data->{"magic"} eq "Avro::DataFile"->AVRO_MAGIC 0 7 7
98  
99 $datafile->{sync_marker} = $data->{sync}
100 7 ▶ or croak "sync marker appears invalid";
Branch: 1 branch
BranchTrueFalseTotal
$datafile->{"sync_marker"} = $data->{"sync"} or croak("sync marker appears invalid") 7 0 7
101  
102 7 ▶ my $codec = $data->{meta}{'avro.codec'} || 'null';
Condition100%2/2
$data->{'meta'}{'avro.codec'} || 'null'
l!l
61
MC/DC100%1/1
$data->{'meta'}{'avro.codec'} || 'null'
$data->{'meta'}{'avro.codec'}
Truth table100%2/2
A
A= $data->{'meta'}{'avro.codec'}
Aresult
11
00
103  
104 7 ▶ throw Avro::DataFile::Error::UnsupportedCodec($codec)
Branch: 1 branch
BranchTrueFalseTotal
unless "Avro::DataFile"->is_codec_valid($codec) 1 6 7
105 unless Avro::DataFile->is_codec_valid($codec);
106  
107 6 return $data;
108 }
109  
110 5 ▶sub all {
Subroutine all: called CC=4 SCAR=13.9
Pod undocumented
111 5 my $datafile = shift;
112  
113 5 my @objs;
114 my @block_objs;
115 5 ▶ do {
Branch: 1 branch
BranchTrueFalseTotal
not @block_objs or do { do { if ($datafile->eof) { @block_objs = (); } else { $datafile->read_block_header if $datafile->eob; @block_objs = $datafile->read_to_block_end; push @objs, @block_objs; } } } 4 4 8
116 9 ▶ if ($datafile->eof) {
Branch: 1 branch
BranchTrueFalseTotal
if ($datafile->eof) { } 4 5 9
117 4 @block_objs = ();
118 }
119 else {
120 5 ▶ $datafile->read_block_header if $datafile->eob;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->eob 5 0 5
121 4 @block_objs = $datafile->read_to_block_end;
122 4 push @objs, @block_objs;
123 }
124  
125 } until !@block_objs;
126  
127 return @objs
128 4 }
129  
130 5 ▶sub next {
Subroutine next: called CC=5 SCAR=17.4
Pod undocumented
131 5 my $datafile = shift;
132 5 my $count = shift;
133  
134 5 my @objs;
135  
136 5 ▶ return () if $datafile->eof;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->eof 1 4 5
137 4 ▶ $datafile->read_block_header if $datafile->eob;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->eob 2 2 4
138  
139 4 my $block_count = $datafile->{object_count};
140  
141 4 ▶ if ($block_count < $count) {
Branch: 1 branch
BranchTrueFalseTotal
if ($block_count < $count) { } 0 4 4
142 0 push @objs, $datafile->read_to_block_end;
143 0 ▶ croak "Didn't read as many objects than expected"
Branch: 1 branch
BranchTrueFalseTotal
unless scalar @objs == $block_count 0 0 0
144 unless scalar @objs == $block_count;
145  
146 0 push @objs, $datafile->next($count - $block_count);
147 }
148 else {
149 4 push @objs, $datafile->read_within_block($count);
150 }
151 4 return @objs;
152 }
153  
154 8 ▶sub read_within_block {
Subroutine read_within_block: called CC=4 SCAR=13.9
Pod undocumented
155 8 my $datafile = shift;
156 8 my $count = shift;
157  
158 8 my $reader = $datafile->reader;
159 8 my $writer_schema = $datafile->writer_schema;
160 8 ▶ my $reader_schema = $datafile->reader_schema || $writer_schema;
Condition66%2/3
$datafile->reader_schema || $writer_schema
l!l&&r!l&&!r
440
MC/DC0%0/2
$datafile->reader_schema || $writer_schema
$datafile->reader_schema $writer_schema
Truth table66%2/3
A || B
A= $datafile->reader_schemaB= $writer_schema
ABresult
1X1
011
000
161 8 my @objs;
162 8 ▶ while ($count-- > 0 && $datafile->{object_count} > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($count-- > 0 and $datafile->{"object_count"} > 0) 105 8 113
Condition66%2/3
$count-- > 0 and $datafile->{"object_count"} > 0
!ll&&!rl&&r
80105
MC/DC50%1/2
$count-- > 0 and $datafile->{"object_count"} > 0
$count-- > 0 $datafile->{"object_count"} > 0
Truth table66%2/3
A and B
A= $count-- > 0B= $datafile->{"object_count"} > 0
ABresult
0X0
100
111
163 105 push @objs, Avro::BinaryDecoder->decode(
164 writer_schema => $writer_schema,
165 reader_schema => $reader_schema,
166 reader => $reader,
167 );
168 105 $datafile->{object_count}--;
169 }
170 8 return @objs;
171 }
172  
173 0 ▶sub skip {
Subroutine skip: not called CC=4 SCAR=30.0
Pod undocumented
174 0 my $datafile = shift;
175 0 my $count = shift;
176  
177 0 my $block_count = $datafile->{object_count};
178 0 ▶ if ($block_count <= $count) {
Branch: 1 branch
BranchTrueFalseTotal
if ($block_count <= $count) { } 0 0 0
179 0 ▶ $datafile->skip_to_block_end
Branch: 1 branch
BranchTrueFalseTotal
$datafile->skip_to_block_end or croak("Cannot skip to end of block!") 0 0 0
180 or croak "Cannot skip to end of block!";
181 0 $datafile->skip($count - $block_count);
182 }
183 else {
184 0 my $writer_schema = $datafile->writer_schema;
185 ## could probably be optimized
186 0 ▶ while ($count--) {
Branch: 1 branch
BranchTrueFalseTotal
if ($count--) 0 0 0
187 0 Avro::BinaryDecoder->skip($writer_schema, $datafile->reader);
188 0 $datafile->{object_count}--;
189 }
190 }
191 }
192  
193 7 ▶sub read_block_header {
Subroutine read_block_header: called CC=7 SCAR=19.5
Pod undocumented
194 7 my $datafile = shift;
195 7 my $fh = $datafile->{fh};
196 7 my $codec = $datafile->codec;
197  
198 6 ▶ $datafile->header unless $datafile->{_header};
Branch: 1 branch
BranchTrueFalseTotal
unless $datafile->{"_header"} 0 6 6
199  
200 6 $datafile->{object_count} = Avro::BinaryDecoder->decode_long(
201 undef, undef, $fh,
202 );
203 6 $datafile->{block_size} = Avro::BinaryDecoder->decode_long(
204 undef, undef, $fh,
205 );
206 6 $datafile->{block_start} = tell $fh;
207  
208 6 ▶ return if $codec eq 'null';
Branch: 1 branch
BranchTrueFalseTotal
if $codec eq "null" 2 4 6
209  
210 ## we need to read the entire block into memory, to inflate it
211 4 ▶ my $nread = read $fh, my $block, $datafile->{block_size} + MARKER_SIZE
Branch: 1 branch
BranchTrueFalseTotal
my $nread = read($fh, my $block, $datafile->{"block_size"} + 16) or croak("Error reading from file: $!") 4 0 4
212 or croak "Error reading from file: $!";
213  
214 ## remove the marker
215 4 my $marker = substr $block, -(MARKER_SIZE), MARKER_SIZE, '';
216 4 $datafile->{block_marker} = $marker;
217  
218 ## this is our new reader
219 4 $datafile->{reader} = do {
220 4 ▶ if ($codec eq 'deflate') {
Branch: 3 branches
BranchTrueFalseTotal
if ($codec eq 'deflate') { } 1 3 4
elsif ($codec eq 'bzip2') { } 2 1 3
elsif ($codec eq 'zstandard') { } 1 0 1
221 1 IO::Uncompress::RawInflate->new(\$block);
222 }
223 elsif ($codec eq 'bzip2') {
224 2 my $uncompressed;
225 2 bunzip2 \$block => \$uncompressed;
226 2 do { open $fh, '<', \$uncompressed; $fh };
227 }
228 elsif ($codec eq 'zstandard') {
229 1 do { open $fh, '<', \(decompress(\$block)); $fh };
230 }
231 };
232  
233 4 return;
234 }
235  
236 4 ▶sub verify_marker {
Subroutine verify_marker: called CC=4 SCAR=14.3
Pod undocumented
237 4 my $datafile = shift;
238  
239 4 my $marker = $datafile->{block_marker};
240 4 ▶ unless (defined $marker) {
Branch: 1 branch
BranchTrueFalseTotal
unless (defined $marker) 1 3 4
241 ## we are in the fh case
242 1 read $datafile->{fh}, $marker, MARKER_SIZE;
243 }
244  
245 4 ▶ unless (($marker || "") eq $datafile->sync_marker) {
Branch: 1 branch
BranchTrueFalseTotal
unless (($marker || "") eq $datafile->sync_marker) 0 4 4
Condition50%1/2
$marker || ''
l!l
40
MC/DC0%0/1
$marker || ''
$marker
Truth table50%1/2
A
A= $marker
Aresult
11
00
246 0 croak "Oops synchronization issue (marker mismatch)";
247 }
248 4 return;
249 }
250  
251 0 ▶sub skip_to_block_end {
Subroutine skip_to_block_end: not called CC=2 SCAR=17.9
Pod undocumented
252 0 my $datafile = shift;
253  
254 0 ▶ if (my $reader = $datafile->{reader}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $reader = $datafile->{"reader"}) 0 0 0
255 0 seek $reader, 0, Fcntl->SEEK_END;
256 0 return;
257 }
258  
259 my $remaining_size = $datafile->{block_size}
260 + $datafile->{block_start}
261 0 - tell $datafile->{fh};
262  
263 0 seek $datafile->{fh}, $remaining_size, 0;
264 0 $datafile->verify_marker; ## will do a read
265 0 return 1;
266 }
267  
268 4 ▶sub read_to_block_end {
Subroutine read_to_block_end: called CC=1 SCAR=0.0
Pod undocumented
269 4 my $datafile = shift;
270  
271 4 my $reader = $datafile->reader;
272 4 my @objs = $datafile->read_within_block( $datafile->{object_count} );
273 4 $datafile->verify_marker;
274 4 return @objs;
275 }
276  
277 12 ▶sub reader {
Subroutine reader: called CC=2 SCAR=7.1
Pod undocumented
278 12 my $datafile = shift;
279 12 ▶ return $datafile->{reader} || $datafile->{fh};
Condition66%2/3
$datafile->{'reader'} || $datafile->{'fh'}
l!l&&r!l&&!r
930
MC/DC0%0/2
$datafile->{'reader'} || $datafile->{'fh'}
$datafile->{'reader'} $datafile->{'fh'}
Truth table66%2/3
A || B
A= $datafile->{'reader'}B= $datafile->{'fh'}
ABresult
1X1
011
000
280 }
281  
282 ## end of block
283 12 ▶sub eob {
Subroutine eob: called CC=6 SCAR=19.1
Pod undocumented
284 12 my $datafile = shift;
285  
286 12 ▶ return 1 if $datafile->eof;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->eof 1 11 12
287  
288 11 ▶ if ($datafile->{reader}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($datafile->{'reader'}) { } 4 7 11
289 4 ▶ return 1 if $datafile->{reader}->eof;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->{"reader"}->eof 0 4 4
290 }
291 else {
292 7 my $pos = tell $datafile->{fh};
293 7 ▶ return 1 unless $datafile->{block_start};
Branch: 1 branch
BranchTrueFalseTotal
unless $datafile->{"block_start"} 7 0 7
294 0 ▶ return 1 if $pos >= $datafile->{block_start} + $datafile->{block_size};
Branch: 1 branch
BranchTrueFalseTotal
if $pos >= $datafile->{"block_start"} + $datafile->{"block_size"} 0 0 0
295 }
296 4 return 0;
297 }
298  
299 29 ▶sub eof {
Subroutine eof: called CC=4 SCAR=13.9
Pod undocumented
300 29 my $datafile = shift;
301 29 ▶ if ($datafile->{reader}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($datafile->{"reader"}) 14 15 29
302 14 ▶ return 0 unless $datafile->{reader}->eof;
Branch: 1 branch
BranchTrueFalseTotal
unless $datafile->{"reader"}->eof 8 6 14
303 }
304 21 ▶ return 1 if $datafile->{fh}->eof;
Branch: 1 branch
BranchTrueFalseTotal
if $datafile->{"fh"}->eof 7 14 21
305 14 return 0;
306 }
307  
308 package Avro::DataFile::Error::UnsupportedCodec;
309 1 ▶use parent 'Error::Simple';
Subroutine BEGIN: called CC=1 SCAR=0.0
310  
311 1 1;
« Avro/Schema.pm ↑ Summary Avro/BinaryDecoder.pm »