blib/lib/PDL/IO/Storable.pm

filter: Statementstmt 94.6% 107 / 113 Branchbran 69.5% 32 / 46 Conditioncond 50.0% 7 / 14 MC/DCmcdc 22.2% 2 / 9 Subroutinesub 94.1% 16 / 17 Podpod 0.0% 0 / 9 totaltotal 78.8% 164 / 208 CC 29 SCAR 23.5
CC
29
Cov
84%
pdlunpack
21.8
STORABLE_thaw
4.1
unpack64bit
3.2
CRAP 10.5
« lib/PDL/MatrixOps.pm ↑ Summary blib/lib/PDL/PP/PDLCode.pm »
1 package PDL::IO::Storable;
2  
3 11 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 11 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5  
6 11 our @EXPORT_OK = qw();
7 11 our %EXPORT_TAGS = (Func=>\@EXPORT_OK);
8 11 our $VERSION = '2.096';
9 11 our @ISA = ( 'PDL::Exporter' );
10  
11 11 ▶use PDL::Core;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 11 ▶use PDL::Exporter;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 11 ▶use DynaLoader;
Subroutine BEGIN: called CC=1 SCAR=0.0
14  
15 =head1 NAME
16  
17 PDL::IO::Storable - helper functions to make PDL usable with serialisation packages
18  
19 =head1 SYNOPSIS
20  
21 use Storable;
22 use PDL::IO::Storable;
23 $hash = {
24 'foo' => 42,
25 'bar' => zeroes(23,45),
26 };
27 store $hash, 'perlhash.dat';
28  
29 use JSON::MaybeXS;
30 $encoder = JSON::MaybeXS->new(allow_tags => 1);
31 my $ndarray = xvals (5,2);
32 my $encoded_json = $encoder->encode ($ndarray);
33 my $decoded_ndarray = $encoder->decode ($encoded_json);
34  
35 =head1 DESCRIPTION
36  
37 Serialisation packages such as C<Storable>, C<Sereal>, C<JSON::MaybeXS>
38 and C<CBOR::XS> implement
39 object persistence for Perl data structures that can
40 contain arbitrary Perl objects. This module implements the relevant methods to
41 be able to store and retrieve ndarrays via C<Storable> as well as packages that support
42 the L<Types::Serialiser> protocol (currently L<Sereal>, L<CBOR::XS> and JSON packages).
43  
44 Note that packages supporting the C<Types::Serialiser> protocol need to have their
45 respective flags enabled so that the FREEZE and THAW callbacks are used.
46  
47 Note also that while L<JSON::MaybeXS> is supported, if it has to
48 fall back to L<JSON::PP>, it will fail. L<JSON::XS> treats the data
49 it gets back from C<FREEZE> as items to encode, while L<JSON::PP>
50 treats the list it gets as strings already encoded. They are
51 fundamentally incompatible, so this module supports the JSON::XS
52 option.
53  
54 Finally, L<data flow|PDL::Dataflow> is not preserved. If a data structure
55 containing ndarrays connected by data flow is serialised then this will need to be
56 explicitly reinstated on deserialisation.
57  
58 =head1 FUNCTIONS
59  
60 =cut
61  
62 { package # hide from PAUSE
63 PDL;
64 11 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
65 # routines to make PDL work with Storable >= 1.03
66  
67 # pdlpack() serializes an ndarray, while pdlunpack() unserializes it. Earlier
68 # versions of PDL didn't control for endianness, type sizes and enumerated type
69 # values; this made stored data unportable across different architectures and
70 # PDL versions. This is no longer the case, but the reading code is still able
71 # to read the old files. The old files have no meta-information in them so it's
72 # impossible to read them correctly with 100% accuracy, but we try to make an
73 # educated guess
74 #
75 # Old data format:
76 #
77 # int type
78 # int ndims
79 # int dims[ndims]
80 # data
81 #
82 # Note that here all the sizes and endiannesses are the native. This is
83 # un-portable. Furthermore, the "type" is an enum, and its values could change
84 # between PDL versions. Here I assume that the old format input data is indeed
85 # native, so the old data files have the same portability issues, but at least
86 # things will remain working and broken in the same way they were before
87 #
88 #
89 # New format:
90 #
91 # uint64 0xFFFF FFFF FFFF FFFF # meant to be different from the old-style data
92 # char type[16] # ' '-padded, left-aligned type string such as 'PDL_LL'
93 # uint32 sizeof(type) # little-endian
94 # uint32 one # native-endian. Used to determine the endianness
95 # uint64 ndims # little-endian
96 # uint64 dims[ndims] # little-endian
97 # data
98 #
99 # The header data is all little-endian here. The data is stored with native
100 # endianness. On load it is checked, and a swap happens, if it is required
101  
102 13 ▶sub pdlpack {
Subroutine pdlpack: called CC=3 SCAR=11.2
Pod undocumented
103 13 my ($pdl) = @_;
104  
105 13 my $hdr = pack( 'c8A16VL',
106 (-1) x 8,
107 $pdl->type->symbol,
108 PDL::Core::howbig( $pdl->get_datatype ), 1 );
109  
110 # I'd like this to be simply
111 # my $dimhdr = pack( 'Q<*', $pdl->getndims, $pdl->dims )
112 # but my pack() may not support Q, so I break it up manually
113 #
114 # if sizeof(int) == 4 here, then $_>>32 will not return 0 necessarily (this in
115 # undefined). I thus manually make sure this is the case
116 #
117 13 ▶ my $noMSW = (PDL::Core::howbig($PDL::Types::PDL_IND) < 8) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
PDL::Core::howbig($PDL::Types::PDL_IND) < 8 ? : 0 13 13
118 my $dimhdr = pack( 'V*',
119 13 ▶ map( { $_ & 0xFFFFFFFF, $noMSW ? 0 : ($_ >> 32) } ($pdl->getndims, $pdl->dims ) ) );
Branch: 1 branch
BranchTrueFalseTotal
$noMSW ? : 0 28 28
120  
121 13 my $dref = $pdl->get_dataref;
122 13 return $hdr . $dimhdr . $$dref;
123 }
124  
125 18 ▶sub pdlunpack {
Subroutine pdlunpack: called CC=19 SCAR=30.8
Pod undocumented
126 11 ▶ use Config ();
Subroutine BEGIN: called CC=1 SCAR=0.0
127 18 my ($pdl,$pack) = @_;
128  
129 18 my ($type, $ndims);
130 18 my @dims = ();
131  
132 18 my $do_swap = 0;
133  
134 # first I try to infer the type of this storable
135 18 my $offset = 0;
136 18 my @magicheader = unpack( "ll", substr( $pack, $offset ) );
137 18 $offset += 8;
138  
139 18 ▶ if( $magicheader[0] != -1 ||
Branch: 1 branch
BranchTrueFalseTotal
if ($magicheader[0] != -1 or $magicheader[1] != -1) { } 2 16 18
Condition66%2/3
$magicheader[0] != -1 or $magicheader[1] != -1
l!l&&r!l&&!r
2016
MC/DC50%1/2
$magicheader[0] != -1 or $magicheader[1] != -1
$magicheader[0] != -1 $magicheader[1] != -1
Truth table66%2/3
A or B
A= $magicheader[0] != -1B= $magicheader[1] != -1
ABresult
1X1
011
000
140 $magicheader[1] != -1 )
141 {
142 2 ▶ print "PDL::IO::Storable detected an old-style pdl\n" if $PDL::verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $PDL::verbose 0 2 2
143  
144 # old-style data. I leave the data sizes, endianness native, since I don't
145 # know any better. This at least won't break anything.
146 #
147 # The "type" however needs attention. Most-recent old-format data had these
148 # values for the type:
149 #
150 # enum { byte,
151 # short,
152 # unsigned short,
153 # long,
154 # long long,
155 # float,
156 # double }
157 #
158 # The $type I read from the file is assumed to be in this enum even though
159 # PDL may have added other types in the middle of this enum.
160 2 my @reftypes = ($PDL::Types::PDL_B,
161 $PDL::Types::PDL_S,
162 $PDL::Types::PDL_U,
163 $PDL::Types::PDL_L,
164 $PDL::Types::PDL_LL,
165 $PDL::Types::PDL_F,
166 $PDL::Types::PDL_D);
167  
168 2 my $stride = $Config::Config{intsize};
169 2 ($type,$ndims) = unpack 'i2', $pack;
170 2 ▶ @dims = $ndims > 0 ? unpack 'i*', substr $pack, 2*$stride,
Branch: 1 branch
BranchTrueFalseTotal
$ndims > 0 ? : 2 0 2
171 $ndims*$stride : ();
172  
173 2 $offset = (2+$ndims)*$stride;
174  
175 2 ▶ if( $type < 0 || $type >= @reftypes )
Branch: 1 branch
BranchTrueFalseTotal
if ($type < 0 or $type >= @reftypes) 0 2 2
Condition33%1/3
$type < 0 or $type >= @reftypes
l!l&&r!l&&!r
002
MC/DC0%0/2
$type < 0 or $type >= @reftypes
$type < 0 $type >= @reftypes
Truth table33%1/3
A or B
A= $type < 0B= $type >= @reftypes
ABresult
1X1
011
000
176 {
177 0 croak "Reading in old-style pdl with unknown type: $type. Giving up.";
178 }
179 2 $type = $reftypes[$type];
180 }
181 else
182 {
183 16 ▶ print "PDL::IO::Storable detected a new-style pdl\n" if $PDL::verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $PDL::verbose 0 16 16
184  
185 # new-style data. I KNOW the data sizes, endianness and the type enum
186 16 my ($typestring) = unpack( 'A16', substr( $pack, $offset ) );
187 16 $offset += 16;
188  
189 16 $typestring =~ s#\s+$##;
190 16 $type = eval { PDL::Type->new($typestring)->enum };
191 16 ▶ if( $@ )
Branch: 1 branch
BranchTrueFalseTotal
if ($@) 0 16 16
192 {
193 0 croak "PDL::IO::Storable couldn't parse type string '$typestring'. Giving up";
194 }
195  
196 16 my ($sizeof) = unpack( 'V', substr( $pack, $offset ) );
197 16 $offset += 4;
198 16 ▶ if( $sizeof != PDL::Core::howbig( $type ) )
Branch: 1 branch
BranchTrueFalseTotal
if ($sizeof != PDL::Core::howbig($type)) 0 16 16
199 {
200 0 croak
201 "PDL::IO::Storable sees mismatched data type sizes when reading data of type '$typestring'\n" .
202 "Stored data has sizeof = $sizeof, while here it is " . PDL::Core::howbig( $type ) . ".\n" .
203 "Giving up";
204 }
205  
206 # check the endianness, if the "1" I read is interpreted as "1" on my
207 # machine then the endiannesses match, and I can just read the data
208 16 my ($one) = unpack( 'L', substr( $pack, $offset ) );
209 16 $offset += 4;
210  
211 16 ▶ if( $one == 1 )
Branch: 1 branch
BranchTrueFalseTotal
if ($one == 1) { } 15 1 16
212 {
213 15 ▶ print "PDL::IO::Storable detected matching endianness\n" if $PDL::verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $PDL::verbose 0 15 15
214 }
215 else
216 {
217 1 ▶ print "PDL::IO::Storable detected non-matching endianness. Correcting data on load\n" if $PDL::verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $PDL::verbose 0 1 1
218  
219 # mismatched endianness. Let's make sure it's a big/little issue, not
220 # something weird. If mismatched, the '00000001' should be seen as
221 # '01000000'
222 1 ▶ if( $one != 0x01000000 )
Branch: 1 branch
BranchTrueFalseTotal
if ($one != 16777216) 0 1 1
223 {
224 0 croak
225 "PDL::IO::Storable sees confused endianness. A '1' was read as '$one'.\n" .
226 "This is neither matching nor swapped endianness. I don't know what's going on,\n" .
227 "so I'm giving up."
228 }
229  
230 # all righty. Everything's fine, but I need to swap all the data
231 1 $do_swap = 1;
232 }
233  
234 # mostly this acts like unpack('Q<'...), but works even if my unpack()
235 # doesn't support 'Q'. This also makes sure that my PDL_Indx is large enough
236 # to read this ndarray
237 sub unpack64bit
238 31 ▶ {
Subroutine unpack64bit: called CC=3 SCAR=11.6
Pod undocumented
239 31 my ($count, $pack, $offset) = @_;
240  
241 return map
242 {
243 31 my ($lsw, $msw) = unpack('VV', substr($$pack, $$offset));
244 35 $$offset += 8;
245  
246 35 ▶ croak( "PDL::IO::Storable tried reading a file with dimensions that don't fit into 32 bits.\n" .
Branch: 1 branch
BranchTrueFalseTotal
if PDL::Core::howbig($PDL::Types::PDL_IND) < 8 and $msw != 0 0 35 35
Condition33%1/3
PDL::Core::howbig($PDL::Types::PDL_IND) < 8 and $msw != 0
!ll&&!rl&&r
3500
MC/DC0%0/2
PDL::Core::howbig($PDL::Types::PDL_IND) < 8 and $msw != 0
PDL::Core::howbig($PDL::Types::PDL_IND) < 8 $msw != 0
Truth table33%1/3
A and B
A= PDL::Core::howbig($PDL::Types::PDL_IND) < 8B= $msw != 0
ABresult
0X0
100
111
247 "However here PDL_Indx can't store a number so large. Giving up." )
248 if( PDL::Core::howbig($PDL::Types::PDL_IND) < 8 && $msw != 0 );
249  
250 35 (($msw << 32) | $lsw)
251 } (1..$count);
252 }
253  
254 16 ($ndims) = unpack64bit( 1, \$pack, \$offset );
255 16 ▶ @dims = unpack64bit( $ndims, \$pack, \$offset ) if $ndims > 0;
Branch: 1 branch
BranchTrueFalseTotal
if $ndims > 0 15 1 16
256 }
257  
258 18 ▶ print "thawing PDL, Dims: [",join(',',@dims),"]\n" if $PDL::verbose;
Branch: 1 branch
BranchTrueFalseTotal
if $PDL::verbose 0 18 18
259  
260 11 ▶ use Scalar::Util qw /blessed/;
Subroutine BEGIN: called CC=1 SCAR=0.0
261 18 ▶ if (!blessed $pdl) { # set_datatype needs an object, not a class name
Branch: 1 branch
BranchTrueFalseTotal
unless (builtin::blessed($pdl)) 5 13 18
262 5 $pdl = $pdl->new;
263 }
264 18 $pdl->set_sv_to_null_pdl; # make this a real ndarray -- this is the tricky bit!
265 18 $pdl->set_datatype($type);
266 18 $pdl->setdims([@dims]);
267 18 my $dref = $pdl->get_dataref;
268 18 $$dref = substr $pack, $offset;
269 18 $pdl->upd_data;
270 18 ▶ $pdl->type->bswap->($pdl) if $do_swap && PDL::Core::howbig($type) != 1;
Branch: 1 branch
BranchTrueFalseTotal
if $do_swap and PDL::Core::howbig($type) != 1 1 17 18
Condition66%2/3
$do_swap and PDL::Core::howbig($type) != 1
!ll&&!rl&&r
1701
MC/DC50%1/2
$do_swap and PDL::Core::howbig($type) != 1
$do_swap PDL::Core::howbig($type) != 1
Truth table66%2/3
A and B
A= $do_swapB= PDL::Core::howbig($type) != 1
ABresult
0X0
100
111
271 18 return $pdl;
272 }
273  
274 14 ▶sub STORABLE_freeze {
Subroutine STORABLE_freeze: called CC=2 SCAR=6.9
Pod undocumented
275 14 my ($self, $cloning) = @_;
276 14 ▶ return UNIVERSAL::isa($self, "HASH") ? ("",{%$self}) # hash ref -> Storable
Branch: 1 branch
BranchTrueFalseTotal
UNIVERSAL::isa($self, 'HASH') ? : 1 13 14
277 : pdlpack($self); # pack the ndarray into a long string
278 }
279  
280 19 ▶sub STORABLE_thaw {
Subroutine STORABLE_thaw: called CC=4 SCAR=14.0
Pod undocumented
281 19 my ($pdl,$cloning,$serial,$hashref) = @_;
282 19 my $class = ref $pdl;
283 19 ▶ if (defined $hashref) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $hashref) { } 1 18 19
284 1 ▶ croak "serial data with hashref!" unless ($serial//'') eq "";
Branch: 1 branch
BranchTrueFalseTotal
unless ($serial // "") eq "" 0 1 1
Condition50%1/2
$serial // ""
l!l
10
MC/DC0%0/1
$serial // ""
$serial
Truth table50%1/2
A
A= $serial
Aresult
11
00
285 1 @$pdl{keys %$hashref} = values %$hashref;
286 } else {
287 # all the magic is happening in pdlunpack
288 18 $pdl->pdlunpack($serial); # unpack our serial into this sv
289 }
290 }
291  
292 # have these as PDL methods
293  
294 =head2 store
295  
296 =for ref
297  
298 store an ndarray using L<Storable>
299  
300 =for example
301  
302 $x = random 12,10;
303 $x->store('myfile');
304  
305 =cut
306  
307 =head2 freeze
308  
309 =for ref
310  
311 freeze an ndarray using L<Storable>
312  
313 =for example
314  
315 $x = random 12,10;
316 $frozen = $x->freeze;
317  
318 =cut
319  
320 0 ▶sub store { require Storable; Storable::store(@_) }
Subroutine store: not called CC=1 SCAR=6.9
Pod undocumented
321 1 ▶sub freeze { require Storable; Storable::freeze(@_) }
Subroutine freeze: called CC=1 SCAR=0.0
Pod undocumented
322  
323 10 ▶sub FREEZE {
Subroutine FREEZE: called CC=2 SCAR=6.9
Pod undocumented
324 10 my ($self, $serialiser, @data) = @_;
325 # non-JSON can use the Storable code
326 10 ▶ return $self->STORABLE_freeze ($serialiser, @data)
Branch: 1 branch
BranchTrueFalseTotal
if $serialiser ne "JSON" 5 5 10
327 if $serialiser ne 'JSON';
328 # JSON needs to use a plain text format to avoid encoding issues
329 # so we unpdl it and let JSON do the rest.
330 # Stringify type so it does not need FREEZE/THAW methods
331 5 my $type = '' . $self->type;
332 5 return ($type, $self->unpdl);
333 }
334  
335 10 ▶sub THAW {
Subroutine THAW: called CC=2 SCAR=6.9
Pod undocumented
336 10 my ($class, $serialiser, @data) = @_;
337 10 ▶ return $class->STORABLE_thaw (undef, @data)
Branch: 1 branch
BranchTrueFalseTotal
if $serialiser ne "JSON" 5 5 10
338 if $serialiser ne 'JSON';
339 # The type has been stringified when frozen so reinstantiate it.
340 5 $data[0] = PDL::Type->new($data[0]);
341 5 return PDL->new (@data);
342 }
343  
344 }
345  
346 =head1 AUTHOR
347  
348 Copyright (C) 2013 Dima Kogan <dima@secretsauce.net>
349 Copyright (C) 2002 Christian Soeller <c.soeller@auckland.ac.nz>
350 All rights reserved. There is no warranty. You are allowed
351 to redistribute this software / documentation under certain
352 conditions. For details, see the file COPYING in the PDL
353 distribution. If this file is separated from the PDL distribution,
354 the copyright notice should be included in the file.
355  
356 =cut
357  
358 11 1;
« lib/PDL/MatrixOps.pm ↑ Summary blib/lib/PDL/PP/PDLCode.pm »