srl_reader_decompress.h

filter: Statementstmt 34.8% 30 / 86 Branchbran 24.0% 12 / 50 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 30.8% 42 / 136 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« srl_common.h ↑ Summary srl_reader_varint.h »
1 #ifndef SRL_READER_DECOMPRESS_H_
2 #define SRL_READER_DECOMPRESS_H_
3  
4 #include "srl_inline.h"
5 #include "srl_common.h"
6 #include "srl_reader.h"
7 #include "srl_reader_error.h"
8 #include "srl_reader_varint.h"
9 #include "srl_protocol.h"
10  
11 /* Older supported Perls do not define this allocation limit. */
12 #ifndef SSize_t_MAX
13 # define SSize_t_MAX ((SSize_t)(~(Size_t)0 >> 1))
14 #endif
15  
16 #if defined(HAVE_CSNAPPY)
17 #include <csnappy.h>
18 #else
19 /* PLEASE READ!!!
20 * Since the decompression code was migrated to this header file
21 * and it's not a good practise to include C files into headers (actually, doing so
22 * cases 'duplicate symbol' errors in Sereal::Path, expected huh?) and I didn't
23 * found right way how to compile snappy's implementation files and link them
24 * (main problem is that snappy files in a subdirectory) I'm only including
25 * header file here and a dev should include C file in his code.
26 *
27 * It should be something like this:
28 * #if !defined(HAVE_CSNAPPY)
29 * # include "snappy/csnappy_decompress.c"
30 * #endif
31 */
32 #include "snappy/csnappy.h"
33 #endif
34  
35 #if defined(HAVE_ZSTD)
36 #include <zstd.h>
37 #else
38 #include "zstd/zstd.h"
39 #endif
40  
41 #if defined(HAVE_MINIZ)
42 #include <miniz.h>
43 #else
44 #include "miniz.h"
45 #endif
46  
47 /* Creates a new buffer of size header_len + body_len + 1 and swaps it into place
48 * of the current reader's buffer. Sets reader position to right after the
49 * header and makes the reader state internally consistent. The buffer is
50 * owned by a mortal SV which is returned. */
51 /* TODO reuse the buffer */
52  
53 SRL_STATIC_INLINE SV *
54 2 srl_realloc_empty_buffer(pTHX_ srl_reader_buffer_t *buf,
55 const STRLEN header_len,
56 const STRLEN body_len,
57 const STRLEN compressed_len,
58 char *compression_type,
59 size_t max_uncompressed_size)
60 {
61 SV *b_sv;
62 srl_reader_char_ptr b;
63  
64 /* Sereal::Merger & Sereal::Path::Iterator do not have this option yet */
65 2 ▶ if ((max_uncompressed_size != 0) && (body_len > max_uncompressed_size))
Branch: 2 branches
BranchTrueFalseTotal
if ((max_uncompressed_size != 0) && (body_len > max_uncompressed_size)) 0 2 2
if ((max_uncompressed_size != 0) && (body_len > max_uncompressed_size)) 0 0 0
66 0 croak("The expected uncompressed size is larger than the allowed maximum size");
67  
68 /* Let perl clean this up. Yes, it's not the most efficient thing
69 * ever, but it's just one mortal per full decompression, so not
70 * a bottle-neck. */
71 if (0) {
72 b_sv = sv_2mortal( newSV(header_len + body_len + 1 ));
73 } else {
74 STRLEN total_requested;
75 2 STRLEN max_str_len = (STRLEN)~(STRLEN)0;
76 char *tmp_buf;
77  
78 2 ▶ if (expect_false(body_len == max_str_len
Branch: 3 branches
BranchTrueFalseTotal
if (expect_false(body_len == max_str_len 2 0 2
if (expect_false(body_len == max_str_len 0 2 2
if (expect_false(body_len == max_str_len 0 2 2
79 || header_len > max_str_len - body_len - 1))
80 0 croak("Decompressed buffer is impossibly large. Refusing to decode.");
81 2 total_requested = header_len + body_len + 1;
82  
83 /* safesysmalloc() exits the process on failure unless PL_nomemok
84 * is set; a document can declare any size, so let it return NULL.
85 * Sizes beyond SSize_t_MAX are refused up front: no allocation can
86 * satisfy them, and DEBUGGING perls croak on them with PL_nomemok
87 * still set. */
88 2 ▶ if (expect_false(total_requested > (STRLEN)SSize_t_MAX)) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false(total_requested > (STRLEN)SSize_t_MAX)) { 0 2 2
89 0 tmp_buf = NULL;
90 }
91 else {
92 2 PL_nomemok = TRUE;
93 2 Newx(tmp_buf, total_requested, char);
94 2 PL_nomemok = FALSE;
95 }
96  
97 2 ▶ if (!tmp_buf)
Branch: 1 branch
BranchTrueFalseTotal
if (!tmp_buf) 2 0 2
98 2 croak("Insufficient memory to '%s' decompress. Size compressed=%"UVuf" uncompressed=%"UVuf,
99 compression_type, (UV)compressed_len, (UV)total_requested );
100  
101 /* This allocation is installed directly as a Perl PV, so keep its
102 * mandatory trailing NUL even though the decoder uses explicit lengths. */
103 0 tmp_buf[total_requested - 1] = '\0';
104 /* Nothing writes the header bytes, but they are inside the buffer
105 * (and V1 offsets are relative to them): never expose them unset. */
106 0 Zero(tmp_buf, header_len, char);
107  
108 0 b_sv= sv_newmortal();
109 0 sv_upgrade(b_sv,SVt_PV);
110 0 SvPVX(b_sv)= tmp_buf;
111 0 SvPOK_on(b_sv);
112 0 SvCUR(b_sv)= total_requested - 1;
113 0 SvLEN(b_sv)= total_requested;
114 }
115 0 b = (srl_reader_char_ptr) SvPVX(b_sv);
116  
117 0 buf->start = b;
118 0 buf->pos = b + header_len;
119 0 buf->end = buf->pos + body_len;
120  
121 /*SRL_RDR_UPDATE_BODY_POS(buf); // XXX caller *MUST* call by himself/herself */
122 0 return b_sv;
123 }
124  
125 /* Decompress a Snappy-compressed document body and put the resulting document
126 * body back in the place of the old compressed blob. The function internaly
127 * creates temporary buffer which is owned by mortal SV. If the caller is
128 * interested in keeping the buffer around for longer time, it should pass
129 * buf_owner parameter and unmortalize it.
130 * The caller *MUST* call SRL_RDR_UPDATE_BODY_POS right after existing from this function. */
131  
132 SRL_STATIC_INLINE UV
133 0 srl_decompress_body_snappy(pTHX_ srl_reader_buffer_t *buf, U8 encoding_flags, SV** buf_owner, size_t max_uncompressed_size)
134 {
135 SV *buf_sv;
136 int header_len;
137 int decompress_ok;
138 uint32_t dest_len, expected_dest_len;
139 UV bytes_consumed;
140  
141 srl_reader_char_ptr old_pos;
142 0 const STRLEN sereal_header_len = (STRLEN) SRL_RDR_POS_OFS(buf);
143 0 const STRLEN compressed_packet_len =
144 encoding_flags == SRL_PROTOCOL_ENCODING_SNAPPY_INCREMENTAL
145 0 ? (STRLEN) srl_read_varint_uv_length(aTHX_ buf, " while reading compressed packet size")
146 0 ▶ : (STRLEN) SRL_RDR_SPACE_LEFT(buf);
Branch: 1 branch
BranchTrueFalseTotal
: (STRLEN) SRL_RDR_SPACE_LEFT(buf); 0 0 0
147  
148 /* All bufl's above here, or we break C89 compilers */
149 0 ▶ if (expect_false( (STRLEN)(uint32_t)compressed_packet_len != compressed_packet_len ))
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( (STRLEN)(uint32_t)compressed_packet_len != compressed_packet_len )) 0 0 0
150 0 SRL_RDR_ERROR(buf, "Snappy-compressed packet is too large for the Snappy decoder");
151  
152 0 old_pos = buf->pos;
153 0 bytes_consumed = compressed_packet_len + SRL_RDR_POS_OFS(buf);
154 0 header_len = csnappy_get_uncompressed_length((char *)buf->pos,
155 compressed_packet_len,
156 &dest_len);
157  
158 0 ▶ if (header_len == CSNAPPY_E_HEADER_BAD)
Branch: 1 branch
BranchTrueFalseTotal
if (header_len == CSNAPPY_E_HEADER_BAD) 0 0 0
159 0 SRL_RDR_ERROR(buf, "Invalid Snappy header in Snappy-compressed Sereal packet");
160  
161 0 expected_dest_len = dest_len;
162  
163 /* Allocate output buffer and swap it into place within the bufoder. */
164 0 buf_sv = srl_realloc_empty_buffer(aTHX_ buf, sereal_header_len, (STRLEN)dest_len,
165 compressed_packet_len, "Snappy", max_uncompressed_size);
166 0 ▶ if (buf_owner) *buf_owner = buf_sv;
Branch: 1 branch
BranchTrueFalseTotal
if (buf_owner) *buf_owner = buf_sv; 0 0 0
167  
168 0 decompress_ok = csnappy_decompress_noheader((char *)(old_pos + header_len),
169 compressed_packet_len - header_len,
170 0 (char *)buf->pos,
171 &dest_len);
172  
173 0 ▶ if (expect_false( decompress_ok != 0 )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( decompress_ok != 0 )) { 0 0 0
174 0 SRL_RDR_ERRORf1(buf, "Snappy decompression of Sereal packet payload failed with error %i!",
175 decompress_ok);
176 }
177  
178 /* The Snappy block header sizes the output buffer, but the raw decoder
179 * reports the number of bytes actually produced through dest_len. Do not
180 * let a short stream expose the unwritten tail as Sereal input. */
181 0 ▶ if (expect_false( dest_len != expected_dest_len )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( dest_len != expected_dest_len )) { 0 0 0
182 0 SRL_RDR_ERRORf2(buf,
183 "Snappy uncompressed size mismatch: packet declares %"UVuf" bytes, stream produced %"UVuf" bytes",
184 (UV)expected_dest_len, (UV)dest_len);
185 }
186  
187 0 return bytes_consumed;
188 }
189  
190 /* Decompress a zlib-compressed document body and put the resulting
191 * document body back in the place of the old compressed blob. The function
192 * internaly creates temporary buffer which is owned by mortal SV. If the
193 * caller is interested in keeping the buffer around for longer time, it should
194 * pass buf_owner parameter and unmortalize it.
195 * The caller *MUST* call SRL_RDR_UPDATE_BODY_POS right after existing from this function. */
196  
197 SRL_STATIC_INLINE UV
198 1 srl_decompress_body_zlib(pTHX_ srl_reader_buffer_t *buf, SV** buf_owner, size_t max_uncompressed_size)
199 {
200 SV *buf_sv;
201 mz_ulong tmp;
202 int decompress_ok;
203 UV bytes_consumed;
204 srl_reader_char_ptr old_pos;
205 1 const STRLEN sereal_header_len = (STRLEN)SRL_RDR_POS_OFS(buf);
206 1 const UV uncompressed_packet_len_uv = srl_read_varint_uv(aTHX_ buf);
207 1 const STRLEN uncompressed_packet_len = (STRLEN)uncompressed_packet_len_uv;
208 const STRLEN compressed_packet_len =
209 1 (STRLEN)srl_read_varint_uv_length(aTHX_ buf, " while reading compressed packet size");
210  
211 1 ▶ SRL_RDR_ASSERT_SPACE(buf, compressed_packet_len, " while reading compressed packet");
Branch: 3 branches
BranchTrueFalseTotal
SRL_RDR_ASSERT_SPACE(buf, compressed_packet_len, " while reading compressed packet"); 1 0 1
SRL_RDR_ASSERT_SPACE(buf, compressed_packet_len, " while reading compressed packet"); 0 1 1
SRL_RDR_ASSERT_SPACE(buf, compressed_packet_len, " while reading compressed packet"); 0 1 1
212 1 ▶ if (expect_false( (UV)uncompressed_packet_len != uncompressed_packet_len_uv
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( (UV)uncompressed_packet_len != uncompressed_packet_len_uv 0 1 1
213 || (UV)(mz_ulong)uncompressed_packet_len_uv != uncompressed_packet_len_uv ))
214 0 SRL_RDR_ERROR(buf, "ZLIB uncompressed size is too large for this platform");
215 if (expect_false( (STRLEN)(mz_ulong)compressed_packet_len != compressed_packet_len ))
216 SRL_RDR_ERROR(buf, "ZLIB-compressed packet is too large for the zlib decoder");
217  
218 /* All decl's above here, or we break C89 compilers */
219 1 old_pos = buf->pos;
220 1 bytes_consumed = compressed_packet_len + SRL_RDR_POS_OFS(buf);
221  
222 /* Allocate output buffer and swap it into place within the decoder. */
223 1 buf_sv = srl_realloc_empty_buffer(aTHX_ buf, sereal_header_len, uncompressed_packet_len,
224 compressed_packet_len, "ZLIB", max_uncompressed_size);
225 0 ▶ if (buf_owner) *buf_owner = buf_sv;
Branch: 1 branch
BranchTrueFalseTotal
if (buf_owner) *buf_owner = buf_sv; 0 0 0
226  
227 0 tmp = uncompressed_packet_len;
228 0 decompress_ok = mz_uncompress((unsigned char *)buf->pos,
229 &tmp, old_pos, compressed_packet_len);
230  
231 0 ▶ if (expect_false( decompress_ok != Z_OK )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( decompress_ok != Z_OK )) { 0 0 0
232 0 SRL_RDR_ERRORf1(buf, "ZLIB decompression of Sereal packet payload failed with error %i!", decompress_ok);
233 }
234  
235 /* The packet's uncompressed-length field is used to size buf->end before
236 * decompression. mz_uncompress also reports the actual output length in
237 * tmp; accepting a shorter stream would leave the rest of that buffer
238 * uninitialized but still readable by the Sereal parser. */
239 0 ▶ if (expect_false( tmp != (mz_ulong)uncompressed_packet_len )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( tmp != (mz_ulong)uncompressed_packet_len )) { 0 0 0
240 0 SRL_RDR_ERRORf2(buf,
241 "ZLIB uncompressed size mismatch: packet declares %"UVuf" bytes, stream produced %"UVuf" bytes",
242 (UV)uncompressed_packet_len, (UV)tmp);
243 }
244  
245 0 return bytes_consumed;
246 }
247  
248 /* Decompress a zstd-compressed document body and put the resulting document
249 * body back in the place of the old compressed blob. The function internaly
250 * creates temporary buffer which is owned by mortal SV. If the caller is
251 * interested in keeping the buffer around for longer time, it should pass
252 * buf_owner parameter and unmortalize it. The caller *MUST* call
253 * SRL_RDR_UPDATE_BODY_POS right after existing from this function. */
254  
255 SRL_STATIC_INLINE UV
256 1 srl_decompress_body_zstd(pTHX_ srl_reader_buffer_t *buf, SV** buf_owner, size_t max_uncompressed_size)
257 {
258 SV *buf_sv;
259 UV bytes_consumed;
260 size_t decompress_code;
261  
262 srl_reader_char_ptr old_pos;
263 unsigned long long uncompressed_packet_len;
264 1 const STRLEN sereal_header_len = (STRLEN) SRL_RDR_POS_OFS(buf);
265 1 const STRLEN compressed_packet_len = (STRLEN) srl_read_varint_uv_length(aTHX_ buf,
266 " while reading compressed packet size");
267  
268 /* All bufl's above here, or we break C89 compilers */
269 1 old_pos = buf->pos;
270 1 bytes_consumed = compressed_packet_len + SRL_RDR_POS_OFS(buf);
271  
272 1 uncompressed_packet_len = ZSTD_getFrameContentSize((const void *)buf->pos, (size_t) compressed_packet_len);
273 1 ▶ if (expect_false(uncompressed_packet_len == 0))
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false(uncompressed_packet_len == 0)) 0 1 1
274 0 SRL_RDR_ERROR(buf, "Invalid zstd packet with no data");
275 1 ▶ if (expect_false(uncompressed_packet_len >= ZSTD_CONTENTSIZE_ERROR))
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false(uncompressed_packet_len >= ZSTD_CONTENTSIZE_ERROR)) 0 1 1
276 0 SRL_RDR_ERROR(buf, "Invalid zstd packet with unknown uncompressed size");
277 if (expect_false( (unsigned long long)(STRLEN)uncompressed_packet_len != uncompressed_packet_len
278 || (unsigned long long)(size_t)uncompressed_packet_len != uncompressed_packet_len ))
279 SRL_RDR_ERROR(buf, "Zstd uncompressed size is too large for this platform");
280  
281 /* Allocate output buffer and swap it into place within the decoder. */
282 1 buf_sv = srl_realloc_empty_buffer(aTHX_ buf, sereal_header_len, (STRLEN) uncompressed_packet_len,
283 compressed_packet_len, "zstd", max_uncompressed_size);
284 0 ▶ if (buf_owner) *buf_owner = buf_sv;
Branch: 1 branch
BranchTrueFalseTotal
if (buf_owner) *buf_owner = buf_sv; 0 0 0
285  
286 0 decompress_code = ZSTD_decompress((void *)buf->pos, (size_t) uncompressed_packet_len,
287 (void *)old_pos, (size_t) compressed_packet_len);
288  
289 0 ▶ if (expect_false( ZSTD_isError(decompress_code) )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( ZSTD_isError(decompress_code) )) { 0 0 0
290 0 SRL_RDR_ERRORf1(buf, "Zstd decompression of Sereal packet payload failed with error %s!",
291 ZSTD_getErrorName(decompress_code));
292 }
293  
294 0 ▶ if (expect_false( decompress_code != (size_t)uncompressed_packet_len )) {
Branch: 1 branch
BranchTrueFalseTotal
if (expect_false( decompress_code != (size_t)uncompressed_packet_len )) { 0 0 0
295 0 SRL_RDR_ERRORf2(buf,
296 "Zstd uncompressed size mismatch: packet declares %"UVuf" bytes, stream produced %"UVuf" bytes",
297 (UV)uncompressed_packet_len, (UV)decompress_code);
298 }
299  
300 0 return bytes_consumed;
301 }
302  
303 #endif
« srl_common.h ↑ Summary srl_reader_varint.h »