zopfli-one.c

filter: Statementstmt 78.3% 1206 / 1540 Branchbran 60.3% 673 / 1116 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 70.7% 1879 / 2656 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« gzip-zopfli-perl.c ↑ Summary Zopfli.xs »
1 /*
2 Copyright 2011 Google Inc. All Rights Reserved.
3  
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7  
8 http://www.apache.org/licenses/LICENSE-2.0
9  
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15  
16 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18 */
19  
20 /* blocksplitter.h */
21 /*
22 Copyright 2011 Google Inc. All Rights Reserved.
23  
24 Licensed under the Apache License, Version 2.0 (the "License");
25 you may not use this file except in compliance with the License.
26 You may obtain a copy of the License at
27  
28 http://www.apache.org/licenses/LICENSE-2.0
29  
30 Unless required by applicable law or agreed to in writing, software
31 distributed under the License is distributed on an "AS IS" BASIS,
32 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 See the License for the specific language governing permissions and
34 limitations under the License.
35  
36 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
37 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
38 */
39  
40 /*
41 Functions to choose good boundaries for block splitting. Deflate allows encoding
42 the data in multiple blocks, with a separate Huffman tree for each block. The
43 Huffman tree itself requires some bytes to encode, so by choosing certain
44 blocks, you can either hurt, or enhance compression. These functions choose good
45 ones that enhance it.
46 */
47  
48 #ifndef ZOPFLI_BLOCKSPLITTER_H_
49 #define ZOPFLI_BLOCKSPLITTER_H_
50  
51 #include <stdlib.h>
52  
53 /* lz77.h */
54 /*
55 Copyright 2011 Google Inc. All Rights Reserved.
56  
57 Licensed under the Apache License, Version 2.0 (the "License");
58 you may not use this file except in compliance with the License.
59 You may obtain a copy of the License at
60  
61 http://www.apache.org/licenses/LICENSE-2.0
62  
63 Unless required by applicable law or agreed to in writing, software
64 distributed under the License is distributed on an "AS IS" BASIS,
65 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
66 See the License for the specific language governing permissions and
67 limitations under the License.
68  
69 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
70 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
71 */
72  
73 /*
74 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
75 compression.
76 */
77  
78 #ifndef ZOPFLI_LZ77_H_
79 #define ZOPFLI_LZ77_H_
80  
81 #include <stdlib.h>
82  
83 /* cache.h */
84 /*
85 Copyright 2011 Google Inc. All Rights Reserved.
86  
87 Licensed under the Apache License, Version 2.0 (the "License");
88 you may not use this file except in compliance with the License.
89 You may obtain a copy of the License at
90  
91 http://www.apache.org/licenses/LICENSE-2.0
92  
93 Unless required by applicable law or agreed to in writing, software
94 distributed under the License is distributed on an "AS IS" BASIS,
95 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
96 See the License for the specific language governing permissions and
97 limitations under the License.
98  
99 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
100 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
101 */
102  
103 /*
104 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
105 */
106  
107 #ifndef ZOPFLI_CACHE_H_
108 #define ZOPFLI_CACHE_H_
109  
110 /* util.h */
111 /*
112 Copyright 2011 Google Inc. All Rights Reserved.
113  
114 Licensed under the Apache License, Version 2.0 (the "License");
115 you may not use this file except in compliance with the License.
116 You may obtain a copy of the License at
117  
118 http://www.apache.org/licenses/LICENSE-2.0
119  
120 Unless required by applicable law or agreed to in writing, software
121 distributed under the License is distributed on an "AS IS" BASIS,
122 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
123 See the License for the specific language governing permissions and
124 limitations under the License.
125  
126 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
127 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
128 */
129  
130 /*
131 Several utilities, including: #defines to try different compression results,
132 basic deflate specification values and generic program options.
133 */
134  
135 #ifndef ZOPFLI_UTIL_H_
136 #define ZOPFLI_UTIL_H_
137  
138 #include <string.h>
139 #include <stdlib.h>
140  
141 /* Minimum and maximum length that can be encoded in deflate. */
142 #define ZOPFLI_MAX_MATCH 258
143 #define ZOPFLI_MIN_MATCH 3
144  
145 /* Number of distinct literal/length and distance symbols in DEFLATE */
146 #define ZOPFLI_NUM_LL 288
147 #define ZOPFLI_NUM_D 32
148  
149 /*
150 The window size for deflate. Must be a power of two. This should be 32768, the
151 maximum possible by the deflate spec. Anything less hurts compression more than
152 speed.
153 */
154 #define ZOPFLI_WINDOW_SIZE 32768
155  
156 /*
157 The window mask used to wrap indices into the window. This is why the
158 window size must be a power of two.
159 */
160 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
161  
162 /*
163 A block structure of huge, non-smart, blocks to divide the input into, to allow
164 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
165 The whole compression algorithm, including the smarter block splitting, will
166 be executed independently on each huge block.
167 Dividing into huge blocks hurts compression, but not much relative to the size.
168 Set it to 0 to disable master blocks.
169 */
170 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
171  
172 /*
173 Used to initialize costs for example
174 */
175 #define ZOPFLI_LARGE_FLOAT 1e30
176  
177 /*
178 For longest match cache. max 256. Uses huge amounts of memory but makes it
179 faster. Uses this many times three bytes per single byte of the input data.
180 This is so because longest match finding has to find the exact distance
181 that belongs to each length for the best lz77 strategy.
182 Good values: e.g. 5, 8.
183 */
184 #define ZOPFLI_CACHE_LENGTH 8
185  
186 /*
187 limit the max hash chain hits for this hash value. This has an effect only
188 on files where the hash value is the same very often. On these files, this
189 gives worse compression (the value should ideally be 32768, which is the
190 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
191 faster on some specific files.
192 Good value: e.g. 8192.
193 */
194 #define ZOPFLI_MAX_CHAIN_HITS 8192
195  
196 /*
197 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
198 consumes a lot of memory but speeds it up. No effect on compression size.
199 */
200 #define ZOPFLI_LONGEST_MATCH_CACHE
201  
202 /*
203 Enable to remember amount of successive identical bytes in the hash chain for
204 finding longest match
205 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
206 This has no effect on the compression result, and enabling it increases speed.
207 */
208 #define ZOPFLI_HASH_SAME
209  
210 /*
211 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
212 best length so far is long enough. This is way faster for files with lots of
213 identical bytes, on which the compressor is otherwise too slow. Regular files
214 are unaffected or maybe a tiny bit slower.
215 This has no effect on the compression result, only on speed.
216 */
217 #define ZOPFLI_HASH_SAME_HASH
218  
219 /*
220 Enable this, to avoid slowness for files which are a repetition of the same
221 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
222 the compression result.
223 */
224 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
225  
226 /*
227 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
228 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
229 varies from file to file.
230 */
231 #define ZOPFLI_LAZY_MATCHING
232  
233 /*
234 Appends value to dynamically allocated memory, doubling its allocation size
235 whenever needed.
236  
237 value: the value to append, type T
238 data: pointer to the dynamic array to append to, type T**
239 size: pointer to the size of the array to append to, type size_t*. This is the
240 size that you consider the array to be, not the internal allocation size.
241 Precondition: allocated size of data is at least a power of two greater than or
242 equal than *size.
243 */
244 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
245 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
246 if (!((*size) & ((*size) - 1))) {\
247 /*double alloc size if it's a power of two*/\
248 void** data_void = reinterpret_cast<void**>(data);\
249 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
250 : realloc((*data), (*size) * 2 * sizeof(**data));\
251 }\
252 (*data)[(*size)] = (value);\
253 (*size)++;\
254 }
255 #else /* C gives problems with strict-aliasing rules for (void**) cast */
256 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
257 if (!((*size) & ((*size) - 1))) {\
258 /*double alloc size if it's a power of two*/\
259 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
260 : realloc((*data), (*size) * 2 * sizeof(**data));\
261 }\
262 (*data)[(*size)] = (value);\
263 (*size)++;\
264 }
265 #endif
266  
267  
268 #endif /* ZOPFLI_UTIL_H_ */
269  
270  
271 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
272  
273 /*
274 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
275 values.
276 This is needed because the squeeze runs will ask these values multiple times for
277 the same position.
278 Uses large amounts of memory, since it has to remember the distance belonging
279 to every possible shorter-than-the-best length (the so called "sublen" array).
280 */
281 typedef struct ZopfliLongestMatchCache {
282 unsigned short* length;
283 unsigned short* dist;
284 unsigned char* sublen;
285 } ZopfliLongestMatchCache;
286  
287 /* Initializes the ZopfliLongestMatchCache. */
288 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
289  
290 /* Frees up the memory of the ZopfliLongestMatchCache. */
291 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
292  
293 /* Stores sublen array in the cache. */
294 void ZopfliSublenToCache(const unsigned short* sublen,
295 size_t pos, size_t length,
296 ZopfliLongestMatchCache* lmc);
297  
298 /* Extracts sublen array from the cache. */
299 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
300 size_t pos, size_t length,
301 unsigned short* sublen);
302 /* Returns the length up to which could be stored in the cache. */
303 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
304 size_t pos, size_t length);
305  
306 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
307  
308 #endif /* ZOPFLI_CACHE_H_ */
309  
310 /* hash.h */
311 /*
312 Copyright 2011 Google Inc. All Rights Reserved.
313  
314 Licensed under the Apache License, Version 2.0 (the "License");
315 you may not use this file except in compliance with the License.
316 You may obtain a copy of the License at
317  
318 http://www.apache.org/licenses/LICENSE-2.0
319  
320 Unless required by applicable law or agreed to in writing, software
321 distributed under the License is distributed on an "AS IS" BASIS,
322 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
323 See the License for the specific language governing permissions and
324 limitations under the License.
325  
326 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
327 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
328 */
329  
330 /*
331 The hash for ZopfliFindLongestMatch of lz77.c.
332 */
333  
334 #ifndef ZOPFLI_HASH_H_
335 #define ZOPFLI_HASH_H_
336  
337 /* util.h */
338 /*
339 Copyright 2011 Google Inc. All Rights Reserved.
340  
341 Licensed under the Apache License, Version 2.0 (the "License");
342 you may not use this file except in compliance with the License.
343 You may obtain a copy of the License at
344  
345 http://www.apache.org/licenses/LICENSE-2.0
346  
347 Unless required by applicable law or agreed to in writing, software
348 distributed under the License is distributed on an "AS IS" BASIS,
349 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
350 See the License for the specific language governing permissions and
351 limitations under the License.
352  
353 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
354 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
355 */
356  
357 /*
358 Several utilities, including: #defines to try different compression results,
359 basic deflate specification values and generic program options.
360 */
361  
362 #ifndef ZOPFLI_UTIL_H_
363 #define ZOPFLI_UTIL_H_
364  
365 #include <string.h>
366 #include <stdlib.h>
367  
368 /* Minimum and maximum length that can be encoded in deflate. */
369 #define ZOPFLI_MAX_MATCH 258
370 #define ZOPFLI_MIN_MATCH 3
371  
372 /* Number of distinct literal/length and distance symbols in DEFLATE */
373 #define ZOPFLI_NUM_LL 288
374 #define ZOPFLI_NUM_D 32
375  
376 /*
377 The window size for deflate. Must be a power of two. This should be 32768, the
378 maximum possible by the deflate spec. Anything less hurts compression more than
379 speed.
380 */
381 #define ZOPFLI_WINDOW_SIZE 32768
382  
383 /*
384 The window mask used to wrap indices into the window. This is why the
385 window size must be a power of two.
386 */
387 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
388  
389 /*
390 A block structure of huge, non-smart, blocks to divide the input into, to allow
391 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
392 The whole compression algorithm, including the smarter block splitting, will
393 be executed independently on each huge block.
394 Dividing into huge blocks hurts compression, but not much relative to the size.
395 Set it to 0 to disable master blocks.
396 */
397 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
398  
399 /*
400 Used to initialize costs for example
401 */
402 #define ZOPFLI_LARGE_FLOAT 1e30
403  
404 /*
405 For longest match cache. max 256. Uses huge amounts of memory but makes it
406 faster. Uses this many times three bytes per single byte of the input data.
407 This is so because longest match finding has to find the exact distance
408 that belongs to each length for the best lz77 strategy.
409 Good values: e.g. 5, 8.
410 */
411 #define ZOPFLI_CACHE_LENGTH 8
412  
413 /*
414 limit the max hash chain hits for this hash value. This has an effect only
415 on files where the hash value is the same very often. On these files, this
416 gives worse compression (the value should ideally be 32768, which is the
417 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
418 faster on some specific files.
419 Good value: e.g. 8192.
420 */
421 #define ZOPFLI_MAX_CHAIN_HITS 8192
422  
423 /*
424 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
425 consumes a lot of memory but speeds it up. No effect on compression size.
426 */
427 #define ZOPFLI_LONGEST_MATCH_CACHE
428  
429 /*
430 Enable to remember amount of successive identical bytes in the hash chain for
431 finding longest match
432 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
433 This has no effect on the compression result, and enabling it increases speed.
434 */
435 #define ZOPFLI_HASH_SAME
436  
437 /*
438 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
439 best length so far is long enough. This is way faster for files with lots of
440 identical bytes, on which the compressor is otherwise too slow. Regular files
441 are unaffected or maybe a tiny bit slower.
442 This has no effect on the compression result, only on speed.
443 */
444 #define ZOPFLI_HASH_SAME_HASH
445  
446 /*
447 Enable this, to avoid slowness for files which are a repetition of the same
448 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
449 the compression result.
450 */
451 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
452  
453 /*
454 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
455 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
456 varies from file to file.
457 */
458 #define ZOPFLI_LAZY_MATCHING
459  
460 /*
461 Appends value to dynamically allocated memory, doubling its allocation size
462 whenever needed.
463  
464 value: the value to append, type T
465 data: pointer to the dynamic array to append to, type T**
466 size: pointer to the size of the array to append to, type size_t*. This is the
467 size that you consider the array to be, not the internal allocation size.
468 Precondition: allocated size of data is at least a power of two greater than or
469 equal than *size.
470 */
471 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
472 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
473 if (!((*size) & ((*size) - 1))) {\
474 /*double alloc size if it's a power of two*/\
475 void** data_void = reinterpret_cast<void**>(data);\
476 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
477 : realloc((*data), (*size) * 2 * sizeof(**data));\
478 }\
479 (*data)[(*size)] = (value);\
480 (*size)++;\
481 }
482 #else /* C gives problems with strict-aliasing rules for (void**) cast */
483 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
484 if (!((*size) & ((*size) - 1))) {\
485 /*double alloc size if it's a power of two*/\
486 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
487 : realloc((*data), (*size) * 2 * sizeof(**data));\
488 }\
489 (*data)[(*size)] = (value);\
490 (*size)++;\
491 }
492 #endif
493  
494  
495 #endif /* ZOPFLI_UTIL_H_ */
496  
497  
498 typedef struct ZopfliHash {
499 int* head; /* Hash value to index of its most recent occurrence. */
500 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
501 int* hashval; /* Index to hash value at this index. */
502 int val; /* Current hash value. */
503  
504 #ifdef ZOPFLI_HASH_SAME_HASH
505 /* Fields with similar purpose as the above hash, but for the second hash with
506 a value that is calculated differently. */
507 int* head2; /* Hash value to index of its most recent occurrence. */
508 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
509 int* hashval2; /* Index to hash value at this index. */
510 int val2; /* Current hash value. */
511 #endif
512  
513 #ifdef ZOPFLI_HASH_SAME
514 unsigned short* same; /* Amount of repetitions of same byte after this .*/
515 #endif
516 } ZopfliHash;
517  
518 /* Allocates ZopfliHash memory. */
519 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
520  
521 /* Resets all fields of ZopfliHash. */
522 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
523  
524 /* Frees ZopfliHash memory. */
525 void ZopfliCleanHash(ZopfliHash* h);
526  
527 /*
528 Updates the hash values based on the current position in the array. All calls
529 to this must be made for consecutive bytes.
530 */
531 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
532 ZopfliHash* h);
533  
534 /*
535 Prepopulates hash:
536 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
537 correctly.
538 */
539 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
540 ZopfliHash* h);
541  
542 #endif /* ZOPFLI_HASH_H_ */
543  
544 /* zopfli.h */
545 /*
546 Copyright 2011 Google Inc. All Rights Reserved.
547  
548 Licensed under the Apache License, Version 2.0 (the "License");
549 you may not use this file except in compliance with the License.
550 You may obtain a copy of the License at
551  
552 http://www.apache.org/licenses/LICENSE-2.0
553  
554 Unless required by applicable law or agreed to in writing, software
555 distributed under the License is distributed on an "AS IS" BASIS,
556 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
557 See the License for the specific language governing permissions and
558 limitations under the License.
559  
560 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
561 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
562 */
563  
564 #ifndef ZOPFLI_ZOPFLI_H_
565 #define ZOPFLI_ZOPFLI_H_
566  
567 #include <stddef.h>
568 #include <stdlib.h> /* for size_t */
569  
570 #ifdef __cplusplus
571 extern "C" {
572 #endif
573  
574 /*
575 Options used throughout the program.
576 */
577 typedef struct ZopfliOptions {
578 /* Whether to print output */
579 int verbose;
580  
581 /* Whether to print more detailed output */
582 int verbose_more;
583  
584 /*
585 Maximum amount of times to rerun forward and backward pass to optimize LZ77
586 compression cost. Good values: 10, 15 for small files, 5 for files over
587 several MB in size or it will be too slow.
588 */
589 int numiterations;
590  
591 /*
592 If true, splits the data in multiple deflate blocks with optimal choice
593 for the block boundaries. Block splitting gives better compression. Default:
594 true (1).
595 */
596 int blocksplitting;
597  
598 /*
599 No longer used, left for compatibility.
600 */
601 int blocksplittinglast;
602  
603 /*
604 Maximum amount of blocks to split into (0 for unlimited, but this can give
605 extreme results that hurt compression on some files). Default value: 15.
606 */
607 int blocksplittingmax;
608 } ZopfliOptions;
609  
610 /* Initializes options with default values. */
611 void ZopfliInitOptions(ZopfliOptions* options);
612  
613 /* Output format */
614 typedef enum {
615 ZOPFLI_FORMAT_GZIP,
616 ZOPFLI_FORMAT_ZLIB,
617 ZOPFLI_FORMAT_DEFLATE
618 } ZopfliFormat;
619  
620 /*
621 Compresses according to the given output format and appends the result to the
622 output.
623  
624 options: global program options
625 output_type: the output format to use
626 out: pointer to the dynamic output array to which the result is appended. Must
627 be freed after use
628 outsize: pointer to the dynamic output array size
629 */
630 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
631 const unsigned char* in, size_t insize,
632 unsigned char** out, size_t* outsize);
633  
634 #ifdef __cplusplus
635 } // extern "C"
636 #endif
637  
638 #endif /* ZOPFLI_ZOPFLI_H_ */
639  
640  
641 /*
642 Stores lit/length and dist pairs for LZ77.
643 Parameter litlens: Contains the literal symbols or length values.
644 Parameter dists: Contains the distances. A value is 0 to indicate that there is
645 no dist and the corresponding litlens value is a literal instead of a length.
646 Parameter size: The size of both the litlens and dists arrays.
647 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
648 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
649  
650 */
651 typedef struct ZopfliLZ77Store {
652 unsigned short* litlens; /* Lit or len. */
653 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
654 if > 0: length in corresponding litlens, this is the distance. */
655 size_t size;
656  
657 const unsigned char* data; /* original data */
658 size_t* pos; /* position in data where this LZ77 command begins */
659  
660 unsigned short* ll_symbol;
661 unsigned short* d_symbol;
662  
663 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
664 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
665 precise histogram at every N symbols, and the rest can be calculated by
666 looping through the actual symbols of this chunk. */
667 size_t* ll_counts;
668 size_t* d_counts;
669 } ZopfliLZ77Store;
670  
671 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
672 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
673 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
674 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
675 size_t pos, ZopfliLZ77Store* store);
676 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
677 ZopfliLZ77Store* target);
678 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
679 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
680 size_t lstart, size_t lend);
681 /* Gets the histogram of lit/len and dist symbols in the given range, using the
682 cumulative histograms, so faster than adding one by one for large range. Does
683 not add the one end symbol of value 256. */
684 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
685 size_t lstart, size_t lend,
686 size_t* ll_counts, size_t* d_counts);
687  
688 /*
689 Some state information for compressing a block.
690 This is currently a bit under-used (with mainly only the longest match cache),
691 but is kept for easy future expansion.
692 */
693 typedef struct ZopfliBlockState {
694 const ZopfliOptions* options;
695  
696 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
697 /* Cache for length/distance pairs found so far. */
698 ZopfliLongestMatchCache* lmc;
699 #endif
700  
701 /* The start (inclusive) and end (not inclusive) of the current block. */
702 size_t blockstart;
703 size_t blockend;
704 } ZopfliBlockState;
705  
706 void ZopfliInitBlockState(const ZopfliOptions* options,
707 size_t blockstart, size_t blockend, int add_lmc,
708 ZopfliBlockState* s);
709 void ZopfliCleanBlockState(ZopfliBlockState* s);
710  
711 /*
712 Finds the longest match (length and corresponding distance) for LZ77
713 compression.
714 Even when not using "sublen", it can be more efficient to provide an array,
715 because only then the caching is used.
716 array: the data
717 pos: position in the data to find the match for
718 size: size of the data
719 limit: limit length to maximum this value (default should be 258). This allows
720 finding a shorter dist for that length (= less extra bits). Must be
721 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
722 sublen: output array of 259 elements, or null. Has, for each length, the
723 smallest distance required to reach this length. Only 256 of its 259 values
724 are used, the first 3 are ignored (the shortest length is 3. It is purely
725 for convenience that the array is made 3 longer).
726 */
727 void ZopfliFindLongestMatch(
728 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
729 size_t pos, size_t size, size_t limit,
730 unsigned short* sublen, unsigned short* distance, unsigned short* length);
731  
732 /*
733 Verifies if length and dist are indeed valid, only used for assertion.
734 */
735 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
736 unsigned short dist, unsigned short length);
737  
738 /*
739 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
740 with the slow but better "squeeze" implementation.
741 The result is placed in the ZopfliLZ77Store.
742 If instart is larger than 0, it uses values before instart as starting
743 dictionary.
744 */
745 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
746 size_t instart, size_t inend,
747 ZopfliLZ77Store* store, ZopfliHash* h);
748  
749 #endif /* ZOPFLI_LZ77_H_ */
750  
751 /* zopfli.h */
752 /*
753 Copyright 2011 Google Inc. All Rights Reserved.
754  
755 Licensed under the Apache License, Version 2.0 (the "License");
756 you may not use this file except in compliance with the License.
757 You may obtain a copy of the License at
758  
759 http://www.apache.org/licenses/LICENSE-2.0
760  
761 Unless required by applicable law or agreed to in writing, software
762 distributed under the License is distributed on an "AS IS" BASIS,
763 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
764 See the License for the specific language governing permissions and
765 limitations under the License.
766  
767 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
768 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
769 */
770  
771 #ifndef ZOPFLI_ZOPFLI_H_
772 #define ZOPFLI_ZOPFLI_H_
773  
774 #include <stddef.h>
775 #include <stdlib.h> /* for size_t */
776  
777 #ifdef __cplusplus
778 extern "C" {
779 #endif
780  
781 /*
782 Options used throughout the program.
783 */
784 typedef struct ZopfliOptions {
785 /* Whether to print output */
786 int verbose;
787  
788 /* Whether to print more detailed output */
789 int verbose_more;
790  
791 /*
792 Maximum amount of times to rerun forward and backward pass to optimize LZ77
793 compression cost. Good values: 10, 15 for small files, 5 for files over
794 several MB in size or it will be too slow.
795 */
796 int numiterations;
797  
798 /*
799 If true, splits the data in multiple deflate blocks with optimal choice
800 for the block boundaries. Block splitting gives better compression. Default:
801 true (1).
802 */
803 int blocksplitting;
804  
805 /*
806 No longer used, left for compatibility.
807 */
808 int blocksplittinglast;
809  
810 /*
811 Maximum amount of blocks to split into (0 for unlimited, but this can give
812 extreme results that hurt compression on some files). Default value: 15.
813 */
814 int blocksplittingmax;
815 } ZopfliOptions;
816  
817 /* Initializes options with default values. */
818 void ZopfliInitOptions(ZopfliOptions* options);
819  
820 /* Output format */
821 typedef enum {
822 ZOPFLI_FORMAT_GZIP,
823 ZOPFLI_FORMAT_ZLIB,
824 ZOPFLI_FORMAT_DEFLATE
825 } ZopfliFormat;
826  
827 /*
828 Compresses according to the given output format and appends the result to the
829 output.
830  
831 options: global program options
832 output_type: the output format to use
833 out: pointer to the dynamic output array to which the result is appended. Must
834 be freed after use
835 outsize: pointer to the dynamic output array size
836 */
837 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
838 const unsigned char* in, size_t insize,
839 unsigned char** out, size_t* outsize);
840  
841 #ifdef __cplusplus
842 } // extern "C"
843 #endif
844  
845 #endif /* ZOPFLI_ZOPFLI_H_ */
846  
847  
848  
849 /*
850 Does blocksplitting on LZ77 data.
851 The output splitpoints are indices in the LZ77 data.
852 maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit.
853 */
854 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
855 const ZopfliLZ77Store* lz77, size_t maxblocks,
856 size_t** splitpoints, size_t* npoints);
857  
858 /*
859 Does blocksplitting on uncompressed data.
860 The output splitpoints are indices in the uncompressed bytes.
861  
862 options: general program options.
863 in: uncompressed input data
864 instart: where to start splitting
865 inend: where to end splitting (not inclusive)
866 maxblocks: maximum amount of blocks to split into, or 0 for no limit
867 splitpoints: dynamic array to put the resulting split point coordinates into.
868 The coordinates are indices in the input array.
869 npoints: pointer to amount of splitpoints, for the dynamic array. The amount of
870 blocks is the amount of splitpoitns + 1.
871 */
872 void ZopfliBlockSplit(const ZopfliOptions* options,
873 const unsigned char* in, size_t instart, size_t inend,
874 size_t maxblocks, size_t** splitpoints, size_t* npoints);
875  
876 /*
877 Divides the input into equal blocks, does not even take LZ77 lengths into
878 account.
879 */
880 void ZopfliBlockSplitSimple(const unsigned char* in,
881 size_t instart, size_t inend,
882 size_t blocksize,
883 size_t** splitpoints, size_t* npoints);
884  
885 #endif /* ZOPFLI_BLOCKSPLITTER_H_ */
886  
887  
888 #include <assert.h>
889 #include <stdio.h>
890 #include <stdlib.h>
891  
892 /* deflate.h */
893 /*
894 Copyright 2011 Google Inc. All Rights Reserved.
895  
896 Licensed under the Apache License, Version 2.0 (the "License");
897 you may not use this file except in compliance with the License.
898 You may obtain a copy of the License at
899  
900 http://www.apache.org/licenses/LICENSE-2.0
901  
902 Unless required by applicable law or agreed to in writing, software
903 distributed under the License is distributed on an "AS IS" BASIS,
904 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
905 See the License for the specific language governing permissions and
906 limitations under the License.
907  
908 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
909 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
910 */
911  
912 #ifndef ZOPFLI_DEFLATE_H_
913 #define ZOPFLI_DEFLATE_H_
914  
915 /*
916 Functions to compress according to the DEFLATE specification, using the
917 "squeeze" LZ77 compression backend.
918 */
919  
920 /* lz77.h */
921 /*
922 Copyright 2011 Google Inc. All Rights Reserved.
923  
924 Licensed under the Apache License, Version 2.0 (the "License");
925 you may not use this file except in compliance with the License.
926 You may obtain a copy of the License at
927  
928 http://www.apache.org/licenses/LICENSE-2.0
929  
930 Unless required by applicable law or agreed to in writing, software
931 distributed under the License is distributed on an "AS IS" BASIS,
932 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
933 See the License for the specific language governing permissions and
934 limitations under the License.
935  
936 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
937 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
938 */
939  
940 /*
941 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
942 compression.
943 */
944  
945 #ifndef ZOPFLI_LZ77_H_
946 #define ZOPFLI_LZ77_H_
947  
948 #include <stdlib.h>
949  
950 /* cache.h */
951 /*
952 Copyright 2011 Google Inc. All Rights Reserved.
953  
954 Licensed under the Apache License, Version 2.0 (the "License");
955 you may not use this file except in compliance with the License.
956 You may obtain a copy of the License at
957  
958 http://www.apache.org/licenses/LICENSE-2.0
959  
960 Unless required by applicable law or agreed to in writing, software
961 distributed under the License is distributed on an "AS IS" BASIS,
962 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
963 See the License for the specific language governing permissions and
964 limitations under the License.
965  
966 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
967 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
968 */
969  
970 /*
971 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
972 */
973  
974 #ifndef ZOPFLI_CACHE_H_
975 #define ZOPFLI_CACHE_H_
976  
977 /* util.h */
978 /*
979 Copyright 2011 Google Inc. All Rights Reserved.
980  
981 Licensed under the Apache License, Version 2.0 (the "License");
982 you may not use this file except in compliance with the License.
983 You may obtain a copy of the License at
984  
985 http://www.apache.org/licenses/LICENSE-2.0
986  
987 Unless required by applicable law or agreed to in writing, software
988 distributed under the License is distributed on an "AS IS" BASIS,
989 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
990 See the License for the specific language governing permissions and
991 limitations under the License.
992  
993 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
994 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
995 */
996  
997 /*
998 Several utilities, including: #defines to try different compression results,
999 basic deflate specification values and generic program options.
1000 */
1001  
1002 #ifndef ZOPFLI_UTIL_H_
1003 #define ZOPFLI_UTIL_H_
1004  
1005 #include <string.h>
1006 #include <stdlib.h>
1007  
1008 /* Minimum and maximum length that can be encoded in deflate. */
1009 #define ZOPFLI_MAX_MATCH 258
1010 #define ZOPFLI_MIN_MATCH 3
1011  
1012 /* Number of distinct literal/length and distance symbols in DEFLATE */
1013 #define ZOPFLI_NUM_LL 288
1014 #define ZOPFLI_NUM_D 32
1015  
1016 /*
1017 The window size for deflate. Must be a power of two. This should be 32768, the
1018 maximum possible by the deflate spec. Anything less hurts compression more than
1019 speed.
1020 */
1021 #define ZOPFLI_WINDOW_SIZE 32768
1022  
1023 /*
1024 The window mask used to wrap indices into the window. This is why the
1025 window size must be a power of two.
1026 */
1027 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
1028  
1029 /*
1030 A block structure of huge, non-smart, blocks to divide the input into, to allow
1031 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
1032 The whole compression algorithm, including the smarter block splitting, will
1033 be executed independently on each huge block.
1034 Dividing into huge blocks hurts compression, but not much relative to the size.
1035 Set it to 0 to disable master blocks.
1036 */
1037 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
1038  
1039 /*
1040 Used to initialize costs for example
1041 */
1042 #define ZOPFLI_LARGE_FLOAT 1e30
1043  
1044 /*
1045 For longest match cache. max 256. Uses huge amounts of memory but makes it
1046 faster. Uses this many times three bytes per single byte of the input data.
1047 This is so because longest match finding has to find the exact distance
1048 that belongs to each length for the best lz77 strategy.
1049 Good values: e.g. 5, 8.
1050 */
1051 #define ZOPFLI_CACHE_LENGTH 8
1052  
1053 /*
1054 limit the max hash chain hits for this hash value. This has an effect only
1055 on files where the hash value is the same very often. On these files, this
1056 gives worse compression (the value should ideally be 32768, which is the
1057 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
1058 faster on some specific files.
1059 Good value: e.g. 8192.
1060 */
1061 #define ZOPFLI_MAX_CHAIN_HITS 8192
1062  
1063 /*
1064 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
1065 consumes a lot of memory but speeds it up. No effect on compression size.
1066 */
1067 #define ZOPFLI_LONGEST_MATCH_CACHE
1068  
1069 /*
1070 Enable to remember amount of successive identical bytes in the hash chain for
1071 finding longest match
1072 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
1073 This has no effect on the compression result, and enabling it increases speed.
1074 */
1075 #define ZOPFLI_HASH_SAME
1076  
1077 /*
1078 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
1079 best length so far is long enough. This is way faster for files with lots of
1080 identical bytes, on which the compressor is otherwise too slow. Regular files
1081 are unaffected or maybe a tiny bit slower.
1082 This has no effect on the compression result, only on speed.
1083 */
1084 #define ZOPFLI_HASH_SAME_HASH
1085  
1086 /*
1087 Enable this, to avoid slowness for files which are a repetition of the same
1088 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
1089 the compression result.
1090 */
1091 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
1092  
1093 /*
1094 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
1095 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
1096 varies from file to file.
1097 */
1098 #define ZOPFLI_LAZY_MATCHING
1099  
1100 /*
1101 Appends value to dynamically allocated memory, doubling its allocation size
1102 whenever needed.
1103  
1104 value: the value to append, type T
1105 data: pointer to the dynamic array to append to, type T**
1106 size: pointer to the size of the array to append to, type size_t*. This is the
1107 size that you consider the array to be, not the internal allocation size.
1108 Precondition: allocated size of data is at least a power of two greater than or
1109 equal than *size.
1110 */
1111 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
1112 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
1113 if (!((*size) & ((*size) - 1))) {\
1114 /*double alloc size if it's a power of two*/\
1115 void** data_void = reinterpret_cast<void**>(data);\
1116 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
1117 : realloc((*data), (*size) * 2 * sizeof(**data));\
1118 }\
1119 (*data)[(*size)] = (value);\
1120 (*size)++;\
1121 }
1122 #else /* C gives problems with strict-aliasing rules for (void**) cast */
1123 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
1124 if (!((*size) & ((*size) - 1))) {\
1125 /*double alloc size if it's a power of two*/\
1126 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
1127 : realloc((*data), (*size) * 2 * sizeof(**data));\
1128 }\
1129 (*data)[(*size)] = (value);\
1130 (*size)++;\
1131 }
1132 #endif
1133  
1134  
1135 #endif /* ZOPFLI_UTIL_H_ */
1136  
1137  
1138 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
1139  
1140 /*
1141 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
1142 values.
1143 This is needed because the squeeze runs will ask these values multiple times for
1144 the same position.
1145 Uses large amounts of memory, since it has to remember the distance belonging
1146 to every possible shorter-than-the-best length (the so called "sublen" array).
1147 */
1148 typedef struct ZopfliLongestMatchCache {
1149 unsigned short* length;
1150 unsigned short* dist;
1151 unsigned char* sublen;
1152 } ZopfliLongestMatchCache;
1153  
1154 /* Initializes the ZopfliLongestMatchCache. */
1155 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
1156  
1157 /* Frees up the memory of the ZopfliLongestMatchCache. */
1158 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
1159  
1160 /* Stores sublen array in the cache. */
1161 void ZopfliSublenToCache(const unsigned short* sublen,
1162 size_t pos, size_t length,
1163 ZopfliLongestMatchCache* lmc);
1164  
1165 /* Extracts sublen array from the cache. */
1166 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
1167 size_t pos, size_t length,
1168 unsigned short* sublen);
1169 /* Returns the length up to which could be stored in the cache. */
1170 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
1171 size_t pos, size_t length);
1172  
1173 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
1174  
1175 #endif /* ZOPFLI_CACHE_H_ */
1176  
1177 /* hash.h */
1178 /*
1179 Copyright 2011 Google Inc. All Rights Reserved.
1180  
1181 Licensed under the Apache License, Version 2.0 (the "License");
1182 you may not use this file except in compliance with the License.
1183 You may obtain a copy of the License at
1184  
1185 http://www.apache.org/licenses/LICENSE-2.0
1186  
1187 Unless required by applicable law or agreed to in writing, software
1188 distributed under the License is distributed on an "AS IS" BASIS,
1189 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1190 See the License for the specific language governing permissions and
1191 limitations under the License.
1192  
1193 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1194 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1195 */
1196  
1197 /*
1198 The hash for ZopfliFindLongestMatch of lz77.c.
1199 */
1200  
1201 #ifndef ZOPFLI_HASH_H_
1202 #define ZOPFLI_HASH_H_
1203  
1204 /* util.h */
1205 /*
1206 Copyright 2011 Google Inc. All Rights Reserved.
1207  
1208 Licensed under the Apache License, Version 2.0 (the "License");
1209 you may not use this file except in compliance with the License.
1210 You may obtain a copy of the License at
1211  
1212 http://www.apache.org/licenses/LICENSE-2.0
1213  
1214 Unless required by applicable law or agreed to in writing, software
1215 distributed under the License is distributed on an "AS IS" BASIS,
1216 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1217 See the License for the specific language governing permissions and
1218 limitations under the License.
1219  
1220 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1221 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1222 */
1223  
1224 /*
1225 Several utilities, including: #defines to try different compression results,
1226 basic deflate specification values and generic program options.
1227 */
1228  
1229 #ifndef ZOPFLI_UTIL_H_
1230 #define ZOPFLI_UTIL_H_
1231  
1232 #include <string.h>
1233 #include <stdlib.h>
1234  
1235 /* Minimum and maximum length that can be encoded in deflate. */
1236 #define ZOPFLI_MAX_MATCH 258
1237 #define ZOPFLI_MIN_MATCH 3
1238  
1239 /* Number of distinct literal/length and distance symbols in DEFLATE */
1240 #define ZOPFLI_NUM_LL 288
1241 #define ZOPFLI_NUM_D 32
1242  
1243 /*
1244 The window size for deflate. Must be a power of two. This should be 32768, the
1245 maximum possible by the deflate spec. Anything less hurts compression more than
1246 speed.
1247 */
1248 #define ZOPFLI_WINDOW_SIZE 32768
1249  
1250 /*
1251 The window mask used to wrap indices into the window. This is why the
1252 window size must be a power of two.
1253 */
1254 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
1255  
1256 /*
1257 A block structure of huge, non-smart, blocks to divide the input into, to allow
1258 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
1259 The whole compression algorithm, including the smarter block splitting, will
1260 be executed independently on each huge block.
1261 Dividing into huge blocks hurts compression, but not much relative to the size.
1262 Set it to 0 to disable master blocks.
1263 */
1264 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
1265  
1266 /*
1267 Used to initialize costs for example
1268 */
1269 #define ZOPFLI_LARGE_FLOAT 1e30
1270  
1271 /*
1272 For longest match cache. max 256. Uses huge amounts of memory but makes it
1273 faster. Uses this many times three bytes per single byte of the input data.
1274 This is so because longest match finding has to find the exact distance
1275 that belongs to each length for the best lz77 strategy.
1276 Good values: e.g. 5, 8.
1277 */
1278 #define ZOPFLI_CACHE_LENGTH 8
1279  
1280 /*
1281 limit the max hash chain hits for this hash value. This has an effect only
1282 on files where the hash value is the same very often. On these files, this
1283 gives worse compression (the value should ideally be 32768, which is the
1284 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
1285 faster on some specific files.
1286 Good value: e.g. 8192.
1287 */
1288 #define ZOPFLI_MAX_CHAIN_HITS 8192
1289  
1290 /*
1291 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
1292 consumes a lot of memory but speeds it up. No effect on compression size.
1293 */
1294 #define ZOPFLI_LONGEST_MATCH_CACHE
1295  
1296 /*
1297 Enable to remember amount of successive identical bytes in the hash chain for
1298 finding longest match
1299 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
1300 This has no effect on the compression result, and enabling it increases speed.
1301 */
1302 #define ZOPFLI_HASH_SAME
1303  
1304 /*
1305 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
1306 best length so far is long enough. This is way faster for files with lots of
1307 identical bytes, on which the compressor is otherwise too slow. Regular files
1308 are unaffected or maybe a tiny bit slower.
1309 This has no effect on the compression result, only on speed.
1310 */
1311 #define ZOPFLI_HASH_SAME_HASH
1312  
1313 /*
1314 Enable this, to avoid slowness for files which are a repetition of the same
1315 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
1316 the compression result.
1317 */
1318 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
1319  
1320 /*
1321 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
1322 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
1323 varies from file to file.
1324 */
1325 #define ZOPFLI_LAZY_MATCHING
1326  
1327 /*
1328 Appends value to dynamically allocated memory, doubling its allocation size
1329 whenever needed.
1330  
1331 value: the value to append, type T
1332 data: pointer to the dynamic array to append to, type T**
1333 size: pointer to the size of the array to append to, type size_t*. This is the
1334 size that you consider the array to be, not the internal allocation size.
1335 Precondition: allocated size of data is at least a power of two greater than or
1336 equal than *size.
1337 */
1338 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
1339 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
1340 if (!((*size) & ((*size) - 1))) {\
1341 /*double alloc size if it's a power of two*/\
1342 void** data_void = reinterpret_cast<void**>(data);\
1343 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
1344 : realloc((*data), (*size) * 2 * sizeof(**data));\
1345 }\
1346 (*data)[(*size)] = (value);\
1347 (*size)++;\
1348 }
1349 #else /* C gives problems with strict-aliasing rules for (void**) cast */
1350 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
1351 if (!((*size) & ((*size) - 1))) {\
1352 /*double alloc size if it's a power of two*/\
1353 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
1354 : realloc((*data), (*size) * 2 * sizeof(**data));\
1355 }\
1356 (*data)[(*size)] = (value);\
1357 (*size)++;\
1358 }
1359 #endif
1360  
1361  
1362 #endif /* ZOPFLI_UTIL_H_ */
1363  
1364  
1365 typedef struct ZopfliHash {
1366 int* head; /* Hash value to index of its most recent occurrence. */
1367 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
1368 int* hashval; /* Index to hash value at this index. */
1369 int val; /* Current hash value. */
1370  
1371 #ifdef ZOPFLI_HASH_SAME_HASH
1372 /* Fields with similar purpose as the above hash, but for the second hash with
1373 a value that is calculated differently. */
1374 int* head2; /* Hash value to index of its most recent occurrence. */
1375 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
1376 int* hashval2; /* Index to hash value at this index. */
1377 int val2; /* Current hash value. */
1378 #endif
1379  
1380 #ifdef ZOPFLI_HASH_SAME
1381 unsigned short* same; /* Amount of repetitions of same byte after this .*/
1382 #endif
1383 } ZopfliHash;
1384  
1385 /* Allocates ZopfliHash memory. */
1386 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
1387  
1388 /* Resets all fields of ZopfliHash. */
1389 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
1390  
1391 /* Frees ZopfliHash memory. */
1392 void ZopfliCleanHash(ZopfliHash* h);
1393  
1394 /*
1395 Updates the hash values based on the current position in the array. All calls
1396 to this must be made for consecutive bytes.
1397 */
1398 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
1399 ZopfliHash* h);
1400  
1401 /*
1402 Prepopulates hash:
1403 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
1404 correctly.
1405 */
1406 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
1407 ZopfliHash* h);
1408  
1409 #endif /* ZOPFLI_HASH_H_ */
1410  
1411 /* zopfli.h */
1412 /*
1413 Copyright 2011 Google Inc. All Rights Reserved.
1414  
1415 Licensed under the Apache License, Version 2.0 (the "License");
1416 you may not use this file except in compliance with the License.
1417 You may obtain a copy of the License at
1418  
1419 http://www.apache.org/licenses/LICENSE-2.0
1420  
1421 Unless required by applicable law or agreed to in writing, software
1422 distributed under the License is distributed on an "AS IS" BASIS,
1423 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1424 See the License for the specific language governing permissions and
1425 limitations under the License.
1426  
1427 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1428 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1429 */
1430  
1431 #ifndef ZOPFLI_ZOPFLI_H_
1432 #define ZOPFLI_ZOPFLI_H_
1433  
1434 #include <stddef.h>
1435 #include <stdlib.h> /* for size_t */
1436  
1437 #ifdef __cplusplus
1438 extern "C" {
1439 #endif
1440  
1441 /*
1442 Options used throughout the program.
1443 */
1444 typedef struct ZopfliOptions {
1445 /* Whether to print output */
1446 int verbose;
1447  
1448 /* Whether to print more detailed output */
1449 int verbose_more;
1450  
1451 /*
1452 Maximum amount of times to rerun forward and backward pass to optimize LZ77
1453 compression cost. Good values: 10, 15 for small files, 5 for files over
1454 several MB in size or it will be too slow.
1455 */
1456 int numiterations;
1457  
1458 /*
1459 If true, splits the data in multiple deflate blocks with optimal choice
1460 for the block boundaries. Block splitting gives better compression. Default:
1461 true (1).
1462 */
1463 int blocksplitting;
1464  
1465 /*
1466 No longer used, left for compatibility.
1467 */
1468 int blocksplittinglast;
1469  
1470 /*
1471 Maximum amount of blocks to split into (0 for unlimited, but this can give
1472 extreme results that hurt compression on some files). Default value: 15.
1473 */
1474 int blocksplittingmax;
1475 } ZopfliOptions;
1476  
1477 /* Initializes options with default values. */
1478 void ZopfliInitOptions(ZopfliOptions* options);
1479  
1480 /* Output format */
1481 typedef enum {
1482 ZOPFLI_FORMAT_GZIP,
1483 ZOPFLI_FORMAT_ZLIB,
1484 ZOPFLI_FORMAT_DEFLATE
1485 } ZopfliFormat;
1486  
1487 /*
1488 Compresses according to the given output format and appends the result to the
1489 output.
1490  
1491 options: global program options
1492 output_type: the output format to use
1493 out: pointer to the dynamic output array to which the result is appended. Must
1494 be freed after use
1495 outsize: pointer to the dynamic output array size
1496 */
1497 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
1498 const unsigned char* in, size_t insize,
1499 unsigned char** out, size_t* outsize);
1500  
1501 #ifdef __cplusplus
1502 } // extern "C"
1503 #endif
1504  
1505 #endif /* ZOPFLI_ZOPFLI_H_ */
1506  
1507  
1508 /*
1509 Stores lit/length and dist pairs for LZ77.
1510 Parameter litlens: Contains the literal symbols or length values.
1511 Parameter dists: Contains the distances. A value is 0 to indicate that there is
1512 no dist and the corresponding litlens value is a literal instead of a length.
1513 Parameter size: The size of both the litlens and dists arrays.
1514 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
1515 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
1516  
1517 */
1518 typedef struct ZopfliLZ77Store {
1519 unsigned short* litlens; /* Lit or len. */
1520 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
1521 if > 0: length in corresponding litlens, this is the distance. */
1522 size_t size;
1523  
1524 const unsigned char* data; /* original data */
1525 size_t* pos; /* position in data where this LZ77 command begins */
1526  
1527 unsigned short* ll_symbol;
1528 unsigned short* d_symbol;
1529  
1530 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
1531 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
1532 precise histogram at every N symbols, and the rest can be calculated by
1533 looping through the actual symbols of this chunk. */
1534 size_t* ll_counts;
1535 size_t* d_counts;
1536 } ZopfliLZ77Store;
1537  
1538 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
1539 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
1540 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
1541 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
1542 size_t pos, ZopfliLZ77Store* store);
1543 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
1544 ZopfliLZ77Store* target);
1545 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
1546 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
1547 size_t lstart, size_t lend);
1548 /* Gets the histogram of lit/len and dist symbols in the given range, using the
1549 cumulative histograms, so faster than adding one by one for large range. Does
1550 not add the one end symbol of value 256. */
1551 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
1552 size_t lstart, size_t lend,
1553 size_t* ll_counts, size_t* d_counts);
1554  
1555 /*
1556 Some state information for compressing a block.
1557 This is currently a bit under-used (with mainly only the longest match cache),
1558 but is kept for easy future expansion.
1559 */
1560 typedef struct ZopfliBlockState {
1561 const ZopfliOptions* options;
1562  
1563 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
1564 /* Cache for length/distance pairs found so far. */
1565 ZopfliLongestMatchCache* lmc;
1566 #endif
1567  
1568 /* The start (inclusive) and end (not inclusive) of the current block. */
1569 size_t blockstart;
1570 size_t blockend;
1571 } ZopfliBlockState;
1572  
1573 void ZopfliInitBlockState(const ZopfliOptions* options,
1574 size_t blockstart, size_t blockend, int add_lmc,
1575 ZopfliBlockState* s);
1576 void ZopfliCleanBlockState(ZopfliBlockState* s);
1577  
1578 /*
1579 Finds the longest match (length and corresponding distance) for LZ77
1580 compression.
1581 Even when not using "sublen", it can be more efficient to provide an array,
1582 because only then the caching is used.
1583 array: the data
1584 pos: position in the data to find the match for
1585 size: size of the data
1586 limit: limit length to maximum this value (default should be 258). This allows
1587 finding a shorter dist for that length (= less extra bits). Must be
1588 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
1589 sublen: output array of 259 elements, or null. Has, for each length, the
1590 smallest distance required to reach this length. Only 256 of its 259 values
1591 are used, the first 3 are ignored (the shortest length is 3. It is purely
1592 for convenience that the array is made 3 longer).
1593 */
1594 void ZopfliFindLongestMatch(
1595 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
1596 size_t pos, size_t size, size_t limit,
1597 unsigned short* sublen, unsigned short* distance, unsigned short* length);
1598  
1599 /*
1600 Verifies if length and dist are indeed valid, only used for assertion.
1601 */
1602 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
1603 unsigned short dist, unsigned short length);
1604  
1605 /*
1606 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
1607 with the slow but better "squeeze" implementation.
1608 The result is placed in the ZopfliLZ77Store.
1609 If instart is larger than 0, it uses values before instart as starting
1610 dictionary.
1611 */
1612 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
1613 size_t instart, size_t inend,
1614 ZopfliLZ77Store* store, ZopfliHash* h);
1615  
1616 #endif /* ZOPFLI_LZ77_H_ */
1617  
1618 /* zopfli.h */
1619 /*
1620 Copyright 2011 Google Inc. All Rights Reserved.
1621  
1622 Licensed under the Apache License, Version 2.0 (the "License");
1623 you may not use this file except in compliance with the License.
1624 You may obtain a copy of the License at
1625  
1626 http://www.apache.org/licenses/LICENSE-2.0
1627  
1628 Unless required by applicable law or agreed to in writing, software
1629 distributed under the License is distributed on an "AS IS" BASIS,
1630 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1631 See the License for the specific language governing permissions and
1632 limitations under the License.
1633  
1634 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1635 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1636 */
1637  
1638 #ifndef ZOPFLI_ZOPFLI_H_
1639 #define ZOPFLI_ZOPFLI_H_
1640  
1641 #include <stddef.h>
1642 #include <stdlib.h> /* for size_t */
1643  
1644 #ifdef __cplusplus
1645 extern "C" {
1646 #endif
1647  
1648 /*
1649 Options used throughout the program.
1650 */
1651 typedef struct ZopfliOptions {
1652 /* Whether to print output */
1653 int verbose;
1654  
1655 /* Whether to print more detailed output */
1656 int verbose_more;
1657  
1658 /*
1659 Maximum amount of times to rerun forward and backward pass to optimize LZ77
1660 compression cost. Good values: 10, 15 for small files, 5 for files over
1661 several MB in size or it will be too slow.
1662 */
1663 int numiterations;
1664  
1665 /*
1666 If true, splits the data in multiple deflate blocks with optimal choice
1667 for the block boundaries. Block splitting gives better compression. Default:
1668 true (1).
1669 */
1670 int blocksplitting;
1671  
1672 /*
1673 No longer used, left for compatibility.
1674 */
1675 int blocksplittinglast;
1676  
1677 /*
1678 Maximum amount of blocks to split into (0 for unlimited, but this can give
1679 extreme results that hurt compression on some files). Default value: 15.
1680 */
1681 int blocksplittingmax;
1682 } ZopfliOptions;
1683  
1684 /* Initializes options with default values. */
1685 void ZopfliInitOptions(ZopfliOptions* options);
1686  
1687 /* Output format */
1688 typedef enum {
1689 ZOPFLI_FORMAT_GZIP,
1690 ZOPFLI_FORMAT_ZLIB,
1691 ZOPFLI_FORMAT_DEFLATE
1692 } ZopfliFormat;
1693  
1694 /*
1695 Compresses according to the given output format and appends the result to the
1696 output.
1697  
1698 options: global program options
1699 output_type: the output format to use
1700 out: pointer to the dynamic output array to which the result is appended. Must
1701 be freed after use
1702 outsize: pointer to the dynamic output array size
1703 */
1704 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
1705 const unsigned char* in, size_t insize,
1706 unsigned char** out, size_t* outsize);
1707  
1708 #ifdef __cplusplus
1709 } // extern "C"
1710 #endif
1711  
1712 #endif /* ZOPFLI_ZOPFLI_H_ */
1713  
1714  
1715 #ifdef __cplusplus
1716 extern "C" {
1717 #endif
1718  
1719 /*
1720 Compresses according to the deflate specification and append the compressed
1721 result to the output.
1722 This function will usually output multiple deflate blocks. If final is 1, then
1723 the final bit will be set on the last block.
1724  
1725 options: global program options
1726 btype: the deflate block type. Use 2 for best compression.
1727 -0: non compressed blocks (00)
1728 -1: blocks with fixed tree (01)
1729 -2: blocks with dynamic tree (10)
1730 final: whether this is the last section of the input, sets the final bit to the
1731 last deflate block.
1732 in: the input bytes
1733 insize: number of input bytes
1734 bp: bit pointer for the output array. This must initially be 0, and for
1735 consecutive calls must be reused (it can have values from 0-7). This is
1736 because deflate appends blocks as bit-based data, rather than on byte
1737 boundaries.
1738 out: pointer to the dynamic output array to which the result is appended. Must
1739 be freed after use.
1740 outsize: pointer to the dynamic output array size.
1741 */
1742 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
1743 const unsigned char* in, size_t insize,
1744 unsigned char* bp, unsigned char** out, size_t* outsize);
1745  
1746 /*
1747 Like ZopfliDeflate, but allows to specify start and end byte with instart and
1748 inend. Only that part is compressed, but earlier bytes are still used for the
1749 back window.
1750 */
1751 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
1752 const unsigned char* in, size_t instart, size_t inend,
1753 unsigned char* bp, unsigned char** out,
1754 size_t* outsize);
1755  
1756 /*
1757 Calculates block size in bits.
1758 litlens: lz77 lit/lengths
1759 dists: ll77 distances
1760 lstart: start of block
1761 lend: end of block (not inclusive)
1762 */
1763 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
1764 size_t lstart, size_t lend, int btype);
1765  
1766 /*
1767 Calculates block size in bits, automatically using the best btype.
1768 */
1769 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
1770 size_t lstart, size_t lend);
1771  
1772 #ifdef __cplusplus
1773 } // extern "C"
1774 #endif
1775  
1776 #endif /* ZOPFLI_DEFLATE_H_ */
1777  
1778 /* squeeze.h */
1779 /*
1780 Copyright 2011 Google Inc. All Rights Reserved.
1781  
1782 Licensed under the Apache License, Version 2.0 (the "License");
1783 you may not use this file except in compliance with the License.
1784 You may obtain a copy of the License at
1785  
1786 http://www.apache.org/licenses/LICENSE-2.0
1787  
1788 Unless required by applicable law or agreed to in writing, software
1789 distributed under the License is distributed on an "AS IS" BASIS,
1790 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1791 See the License for the specific language governing permissions and
1792 limitations under the License.
1793  
1794 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1795 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1796 */
1797  
1798 /*
1799 The squeeze functions do enhanced LZ77 compression by optimal parsing with a
1800 cost model, rather than greedily choosing the longest length or using a single
1801 step of lazy matching like regular implementations.
1802  
1803 Since the cost model is based on the Huffman tree that can only be calculated
1804 after the LZ77 data is generated, there is a chicken and egg problem, and
1805 multiple runs are done with updated cost models to converge to a better
1806 solution.
1807 */
1808  
1809 #ifndef ZOPFLI_SQUEEZE_H_
1810 #define ZOPFLI_SQUEEZE_H_
1811  
1812 /* lz77.h */
1813 /*
1814 Copyright 2011 Google Inc. All Rights Reserved.
1815  
1816 Licensed under the Apache License, Version 2.0 (the "License");
1817 you may not use this file except in compliance with the License.
1818 You may obtain a copy of the License at
1819  
1820 http://www.apache.org/licenses/LICENSE-2.0
1821  
1822 Unless required by applicable law or agreed to in writing, software
1823 distributed under the License is distributed on an "AS IS" BASIS,
1824 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1825 See the License for the specific language governing permissions and
1826 limitations under the License.
1827  
1828 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1829 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1830 */
1831  
1832 /*
1833 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
1834 compression.
1835 */
1836  
1837 #ifndef ZOPFLI_LZ77_H_
1838 #define ZOPFLI_LZ77_H_
1839  
1840 #include <stdlib.h>
1841  
1842 /* cache.h */
1843 /*
1844 Copyright 2011 Google Inc. All Rights Reserved.
1845  
1846 Licensed under the Apache License, Version 2.0 (the "License");
1847 you may not use this file except in compliance with the License.
1848 You may obtain a copy of the License at
1849  
1850 http://www.apache.org/licenses/LICENSE-2.0
1851  
1852 Unless required by applicable law or agreed to in writing, software
1853 distributed under the License is distributed on an "AS IS" BASIS,
1854 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1855 See the License for the specific language governing permissions and
1856 limitations under the License.
1857  
1858 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1859 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1860 */
1861  
1862 /*
1863 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
1864 */
1865  
1866 #ifndef ZOPFLI_CACHE_H_
1867 #define ZOPFLI_CACHE_H_
1868  
1869 /* util.h */
1870 /*
1871 Copyright 2011 Google Inc. All Rights Reserved.
1872  
1873 Licensed under the Apache License, Version 2.0 (the "License");
1874 you may not use this file except in compliance with the License.
1875 You may obtain a copy of the License at
1876  
1877 http://www.apache.org/licenses/LICENSE-2.0
1878  
1879 Unless required by applicable law or agreed to in writing, software
1880 distributed under the License is distributed on an "AS IS" BASIS,
1881 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1882 See the License for the specific language governing permissions and
1883 limitations under the License.
1884  
1885 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
1886 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
1887 */
1888  
1889 /*
1890 Several utilities, including: #defines to try different compression results,
1891 basic deflate specification values and generic program options.
1892 */
1893  
1894 #ifndef ZOPFLI_UTIL_H_
1895 #define ZOPFLI_UTIL_H_
1896  
1897 #include <string.h>
1898 #include <stdlib.h>
1899  
1900 /* Minimum and maximum length that can be encoded in deflate. */
1901 #define ZOPFLI_MAX_MATCH 258
1902 #define ZOPFLI_MIN_MATCH 3
1903  
1904 /* Number of distinct literal/length and distance symbols in DEFLATE */
1905 #define ZOPFLI_NUM_LL 288
1906 #define ZOPFLI_NUM_D 32
1907  
1908 /*
1909 The window size for deflate. Must be a power of two. This should be 32768, the
1910 maximum possible by the deflate spec. Anything less hurts compression more than
1911 speed.
1912 */
1913 #define ZOPFLI_WINDOW_SIZE 32768
1914  
1915 /*
1916 The window mask used to wrap indices into the window. This is why the
1917 window size must be a power of two.
1918 */
1919 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
1920  
1921 /*
1922 A block structure of huge, non-smart, blocks to divide the input into, to allow
1923 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
1924 The whole compression algorithm, including the smarter block splitting, will
1925 be executed independently on each huge block.
1926 Dividing into huge blocks hurts compression, but not much relative to the size.
1927 Set it to 0 to disable master blocks.
1928 */
1929 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
1930  
1931 /*
1932 Used to initialize costs for example
1933 */
1934 #define ZOPFLI_LARGE_FLOAT 1e30
1935  
1936 /*
1937 For longest match cache. max 256. Uses huge amounts of memory but makes it
1938 faster. Uses this many times three bytes per single byte of the input data.
1939 This is so because longest match finding has to find the exact distance
1940 that belongs to each length for the best lz77 strategy.
1941 Good values: e.g. 5, 8.
1942 */
1943 #define ZOPFLI_CACHE_LENGTH 8
1944  
1945 /*
1946 limit the max hash chain hits for this hash value. This has an effect only
1947 on files where the hash value is the same very often. On these files, this
1948 gives worse compression (the value should ideally be 32768, which is the
1949 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
1950 faster on some specific files.
1951 Good value: e.g. 8192.
1952 */
1953 #define ZOPFLI_MAX_CHAIN_HITS 8192
1954  
1955 /*
1956 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
1957 consumes a lot of memory but speeds it up. No effect on compression size.
1958 */
1959 #define ZOPFLI_LONGEST_MATCH_CACHE
1960  
1961 /*
1962 Enable to remember amount of successive identical bytes in the hash chain for
1963 finding longest match
1964 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
1965 This has no effect on the compression result, and enabling it increases speed.
1966 */
1967 #define ZOPFLI_HASH_SAME
1968  
1969 /*
1970 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
1971 best length so far is long enough. This is way faster for files with lots of
1972 identical bytes, on which the compressor is otherwise too slow. Regular files
1973 are unaffected or maybe a tiny bit slower.
1974 This has no effect on the compression result, only on speed.
1975 */
1976 #define ZOPFLI_HASH_SAME_HASH
1977  
1978 /*
1979 Enable this, to avoid slowness for files which are a repetition of the same
1980 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
1981 the compression result.
1982 */
1983 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
1984  
1985 /*
1986 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
1987 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
1988 varies from file to file.
1989 */
1990 #define ZOPFLI_LAZY_MATCHING
1991  
1992 /*
1993 Appends value to dynamically allocated memory, doubling its allocation size
1994 whenever needed.
1995  
1996 value: the value to append, type T
1997 data: pointer to the dynamic array to append to, type T**
1998 size: pointer to the size of the array to append to, type size_t*. This is the
1999 size that you consider the array to be, not the internal allocation size.
2000 Precondition: allocated size of data is at least a power of two greater than or
2001 equal than *size.
2002 */
2003 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
2004 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2005 if (!((*size) & ((*size) - 1))) {\
2006 /*double alloc size if it's a power of two*/\
2007 void** data_void = reinterpret_cast<void**>(data);\
2008 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
2009 : realloc((*data), (*size) * 2 * sizeof(**data));\
2010 }\
2011 (*data)[(*size)] = (value);\
2012 (*size)++;\
2013 }
2014 #else /* C gives problems with strict-aliasing rules for (void**) cast */
2015 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2016 if (!((*size) & ((*size) - 1))) {\
2017 /*double alloc size if it's a power of two*/\
2018 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
2019 : realloc((*data), (*size) * 2 * sizeof(**data));\
2020 }\
2021 (*data)[(*size)] = (value);\
2022 (*size)++;\
2023 }
2024 #endif
2025  
2026  
2027 #endif /* ZOPFLI_UTIL_H_ */
2028  
2029  
2030 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
2031  
2032 /*
2033 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
2034 values.
2035 This is needed because the squeeze runs will ask these values multiple times for
2036 the same position.
2037 Uses large amounts of memory, since it has to remember the distance belonging
2038 to every possible shorter-than-the-best length (the so called "sublen" array).
2039 */
2040 typedef struct ZopfliLongestMatchCache {
2041 unsigned short* length;
2042 unsigned short* dist;
2043 unsigned char* sublen;
2044 } ZopfliLongestMatchCache;
2045  
2046 /* Initializes the ZopfliLongestMatchCache. */
2047 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
2048  
2049 /* Frees up the memory of the ZopfliLongestMatchCache. */
2050 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
2051  
2052 /* Stores sublen array in the cache. */
2053 void ZopfliSublenToCache(const unsigned short* sublen,
2054 size_t pos, size_t length,
2055 ZopfliLongestMatchCache* lmc);
2056  
2057 /* Extracts sublen array from the cache. */
2058 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
2059 size_t pos, size_t length,
2060 unsigned short* sublen);
2061 /* Returns the length up to which could be stored in the cache. */
2062 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
2063 size_t pos, size_t length);
2064  
2065 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
2066  
2067 #endif /* ZOPFLI_CACHE_H_ */
2068  
2069 /* hash.h */
2070 /*
2071 Copyright 2011 Google Inc. All Rights Reserved.
2072  
2073 Licensed under the Apache License, Version 2.0 (the "License");
2074 you may not use this file except in compliance with the License.
2075 You may obtain a copy of the License at
2076  
2077 http://www.apache.org/licenses/LICENSE-2.0
2078  
2079 Unless required by applicable law or agreed to in writing, software
2080 distributed under the License is distributed on an "AS IS" BASIS,
2081 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2082 See the License for the specific language governing permissions and
2083 limitations under the License.
2084  
2085 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
2086 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
2087 */
2088  
2089 /*
2090 The hash for ZopfliFindLongestMatch of lz77.c.
2091 */
2092  
2093 #ifndef ZOPFLI_HASH_H_
2094 #define ZOPFLI_HASH_H_
2095  
2096 /* util.h */
2097 /*
2098 Copyright 2011 Google Inc. All Rights Reserved.
2099  
2100 Licensed under the Apache License, Version 2.0 (the "License");
2101 you may not use this file except in compliance with the License.
2102 You may obtain a copy of the License at
2103  
2104 http://www.apache.org/licenses/LICENSE-2.0
2105  
2106 Unless required by applicable law or agreed to in writing, software
2107 distributed under the License is distributed on an "AS IS" BASIS,
2108 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2109 See the License for the specific language governing permissions and
2110 limitations under the License.
2111  
2112 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
2113 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
2114 */
2115  
2116 /*
2117 Several utilities, including: #defines to try different compression results,
2118 basic deflate specification values and generic program options.
2119 */
2120  
2121 #ifndef ZOPFLI_UTIL_H_
2122 #define ZOPFLI_UTIL_H_
2123  
2124 #include <string.h>
2125 #include <stdlib.h>
2126  
2127 /* Minimum and maximum length that can be encoded in deflate. */
2128 #define ZOPFLI_MAX_MATCH 258
2129 #define ZOPFLI_MIN_MATCH 3
2130  
2131 /* Number of distinct literal/length and distance symbols in DEFLATE */
2132 #define ZOPFLI_NUM_LL 288
2133 #define ZOPFLI_NUM_D 32
2134  
2135 /*
2136 The window size for deflate. Must be a power of two. This should be 32768, the
2137 maximum possible by the deflate spec. Anything less hurts compression more than
2138 speed.
2139 */
2140 #define ZOPFLI_WINDOW_SIZE 32768
2141  
2142 /*
2143 The window mask used to wrap indices into the window. This is why the
2144 window size must be a power of two.
2145 */
2146 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
2147  
2148 /*
2149 A block structure of huge, non-smart, blocks to divide the input into, to allow
2150 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
2151 The whole compression algorithm, including the smarter block splitting, will
2152 be executed independently on each huge block.
2153 Dividing into huge blocks hurts compression, but not much relative to the size.
2154 Set it to 0 to disable master blocks.
2155 */
2156 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
2157  
2158 /*
2159 Used to initialize costs for example
2160 */
2161 #define ZOPFLI_LARGE_FLOAT 1e30
2162  
2163 /*
2164 For longest match cache. max 256. Uses huge amounts of memory but makes it
2165 faster. Uses this many times three bytes per single byte of the input data.
2166 This is so because longest match finding has to find the exact distance
2167 that belongs to each length for the best lz77 strategy.
2168 Good values: e.g. 5, 8.
2169 */
2170 #define ZOPFLI_CACHE_LENGTH 8
2171  
2172 /*
2173 limit the max hash chain hits for this hash value. This has an effect only
2174 on files where the hash value is the same very often. On these files, this
2175 gives worse compression (the value should ideally be 32768, which is the
2176 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
2177 faster on some specific files.
2178 Good value: e.g. 8192.
2179 */
2180 #define ZOPFLI_MAX_CHAIN_HITS 8192
2181  
2182 /*
2183 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
2184 consumes a lot of memory but speeds it up. No effect on compression size.
2185 */
2186 #define ZOPFLI_LONGEST_MATCH_CACHE
2187  
2188 /*
2189 Enable to remember amount of successive identical bytes in the hash chain for
2190 finding longest match
2191 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
2192 This has no effect on the compression result, and enabling it increases speed.
2193 */
2194 #define ZOPFLI_HASH_SAME
2195  
2196 /*
2197 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
2198 best length so far is long enough. This is way faster for files with lots of
2199 identical bytes, on which the compressor is otherwise too slow. Regular files
2200 are unaffected or maybe a tiny bit slower.
2201 This has no effect on the compression result, only on speed.
2202 */
2203 #define ZOPFLI_HASH_SAME_HASH
2204  
2205 /*
2206 Enable this, to avoid slowness for files which are a repetition of the same
2207 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
2208 the compression result.
2209 */
2210 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
2211  
2212 /*
2213 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
2214 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
2215 varies from file to file.
2216 */
2217 #define ZOPFLI_LAZY_MATCHING
2218  
2219 /*
2220 Appends value to dynamically allocated memory, doubling its allocation size
2221 whenever needed.
2222  
2223 value: the value to append, type T
2224 data: pointer to the dynamic array to append to, type T**
2225 size: pointer to the size of the array to append to, type size_t*. This is the
2226 size that you consider the array to be, not the internal allocation size.
2227 Precondition: allocated size of data is at least a power of two greater than or
2228 equal than *size.
2229 */
2230 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
2231 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2232 if (!((*size) & ((*size) - 1))) {\
2233 /*double alloc size if it's a power of two*/\
2234 void** data_void = reinterpret_cast<void**>(data);\
2235 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
2236 : realloc((*data), (*size) * 2 * sizeof(**data));\
2237 }\
2238 (*data)[(*size)] = (value);\
2239 (*size)++;\
2240 }
2241 #else /* C gives problems with strict-aliasing rules for (void**) cast */
2242 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2243 if (!((*size) & ((*size) - 1))) {\
2244 /*double alloc size if it's a power of two*/\
2245 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
2246 : realloc((*data), (*size) * 2 * sizeof(**data));\
2247 }\
2248 (*data)[(*size)] = (value);\
2249 (*size)++;\
2250 }
2251 #endif
2252  
2253  
2254 #endif /* ZOPFLI_UTIL_H_ */
2255  
2256  
2257 typedef struct ZopfliHash {
2258 int* head; /* Hash value to index of its most recent occurrence. */
2259 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
2260 int* hashval; /* Index to hash value at this index. */
2261 int val; /* Current hash value. */
2262  
2263 #ifdef ZOPFLI_HASH_SAME_HASH
2264 /* Fields with similar purpose as the above hash, but for the second hash with
2265 a value that is calculated differently. */
2266 int* head2; /* Hash value to index of its most recent occurrence. */
2267 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
2268 int* hashval2; /* Index to hash value at this index. */
2269 int val2; /* Current hash value. */
2270 #endif
2271  
2272 #ifdef ZOPFLI_HASH_SAME
2273 unsigned short* same; /* Amount of repetitions of same byte after this .*/
2274 #endif
2275 } ZopfliHash;
2276  
2277 /* Allocates ZopfliHash memory. */
2278 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
2279  
2280 /* Resets all fields of ZopfliHash. */
2281 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
2282  
2283 /* Frees ZopfliHash memory. */
2284 void ZopfliCleanHash(ZopfliHash* h);
2285  
2286 /*
2287 Updates the hash values based on the current position in the array. All calls
2288 to this must be made for consecutive bytes.
2289 */
2290 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
2291 ZopfliHash* h);
2292  
2293 /*
2294 Prepopulates hash:
2295 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
2296 correctly.
2297 */
2298 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
2299 ZopfliHash* h);
2300  
2301 #endif /* ZOPFLI_HASH_H_ */
2302  
2303 /* zopfli.h */
2304 /*
2305 Copyright 2011 Google Inc. All Rights Reserved.
2306  
2307 Licensed under the Apache License, Version 2.0 (the "License");
2308 you may not use this file except in compliance with the License.
2309 You may obtain a copy of the License at
2310  
2311 http://www.apache.org/licenses/LICENSE-2.0
2312  
2313 Unless required by applicable law or agreed to in writing, software
2314 distributed under the License is distributed on an "AS IS" BASIS,
2315 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2316 See the License for the specific language governing permissions and
2317 limitations under the License.
2318  
2319 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
2320 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
2321 */
2322  
2323 #ifndef ZOPFLI_ZOPFLI_H_
2324 #define ZOPFLI_ZOPFLI_H_
2325  
2326 #include <stddef.h>
2327 #include <stdlib.h> /* for size_t */
2328  
2329 #ifdef __cplusplus
2330 extern "C" {
2331 #endif
2332  
2333 /*
2334 Options used throughout the program.
2335 */
2336 typedef struct ZopfliOptions {
2337 /* Whether to print output */
2338 int verbose;
2339  
2340 /* Whether to print more detailed output */
2341 int verbose_more;
2342  
2343 /*
2344 Maximum amount of times to rerun forward and backward pass to optimize LZ77
2345 compression cost. Good values: 10, 15 for small files, 5 for files over
2346 several MB in size or it will be too slow.
2347 */
2348 int numiterations;
2349  
2350 /*
2351 If true, splits the data in multiple deflate blocks with optimal choice
2352 for the block boundaries. Block splitting gives better compression. Default:
2353 true (1).
2354 */
2355 int blocksplitting;
2356  
2357 /*
2358 No longer used, left for compatibility.
2359 */
2360 int blocksplittinglast;
2361  
2362 /*
2363 Maximum amount of blocks to split into (0 for unlimited, but this can give
2364 extreme results that hurt compression on some files). Default value: 15.
2365 */
2366 int blocksplittingmax;
2367 } ZopfliOptions;
2368  
2369 /* Initializes options with default values. */
2370 void ZopfliInitOptions(ZopfliOptions* options);
2371  
2372 /* Output format */
2373 typedef enum {
2374 ZOPFLI_FORMAT_GZIP,
2375 ZOPFLI_FORMAT_ZLIB,
2376 ZOPFLI_FORMAT_DEFLATE
2377 } ZopfliFormat;
2378  
2379 /*
2380 Compresses according to the given output format and appends the result to the
2381 output.
2382  
2383 options: global program options
2384 output_type: the output format to use
2385 out: pointer to the dynamic output array to which the result is appended. Must
2386 be freed after use
2387 outsize: pointer to the dynamic output array size
2388 */
2389 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
2390 const unsigned char* in, size_t insize,
2391 unsigned char** out, size_t* outsize);
2392  
2393 #ifdef __cplusplus
2394 } // extern "C"
2395 #endif
2396  
2397 #endif /* ZOPFLI_ZOPFLI_H_ */
2398  
2399  
2400 /*
2401 Stores lit/length and dist pairs for LZ77.
2402 Parameter litlens: Contains the literal symbols or length values.
2403 Parameter dists: Contains the distances. A value is 0 to indicate that there is
2404 no dist and the corresponding litlens value is a literal instead of a length.
2405 Parameter size: The size of both the litlens and dists arrays.
2406 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
2407 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
2408  
2409 */
2410 typedef struct ZopfliLZ77Store {
2411 unsigned short* litlens; /* Lit or len. */
2412 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
2413 if > 0: length in corresponding litlens, this is the distance. */
2414 size_t size;
2415  
2416 const unsigned char* data; /* original data */
2417 size_t* pos; /* position in data where this LZ77 command begins */
2418  
2419 unsigned short* ll_symbol;
2420 unsigned short* d_symbol;
2421  
2422 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
2423 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
2424 precise histogram at every N symbols, and the rest can be calculated by
2425 looping through the actual symbols of this chunk. */
2426 size_t* ll_counts;
2427 size_t* d_counts;
2428 } ZopfliLZ77Store;
2429  
2430 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
2431 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
2432 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
2433 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
2434 size_t pos, ZopfliLZ77Store* store);
2435 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
2436 ZopfliLZ77Store* target);
2437 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
2438 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
2439 size_t lstart, size_t lend);
2440 /* Gets the histogram of lit/len and dist symbols in the given range, using the
2441 cumulative histograms, so faster than adding one by one for large range. Does
2442 not add the one end symbol of value 256. */
2443 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
2444 size_t lstart, size_t lend,
2445 size_t* ll_counts, size_t* d_counts);
2446  
2447 /*
2448 Some state information for compressing a block.
2449 This is currently a bit under-used (with mainly only the longest match cache),
2450 but is kept for easy future expansion.
2451 */
2452 typedef struct ZopfliBlockState {
2453 const ZopfliOptions* options;
2454  
2455 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
2456 /* Cache for length/distance pairs found so far. */
2457 ZopfliLongestMatchCache* lmc;
2458 #endif
2459  
2460 /* The start (inclusive) and end (not inclusive) of the current block. */
2461 size_t blockstart;
2462 size_t blockend;
2463 } ZopfliBlockState;
2464  
2465 void ZopfliInitBlockState(const ZopfliOptions* options,
2466 size_t blockstart, size_t blockend, int add_lmc,
2467 ZopfliBlockState* s);
2468 void ZopfliCleanBlockState(ZopfliBlockState* s);
2469  
2470 /*
2471 Finds the longest match (length and corresponding distance) for LZ77
2472 compression.
2473 Even when not using "sublen", it can be more efficient to provide an array,
2474 because only then the caching is used.
2475 array: the data
2476 pos: position in the data to find the match for
2477 size: size of the data
2478 limit: limit length to maximum this value (default should be 258). This allows
2479 finding a shorter dist for that length (= less extra bits). Must be
2480 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
2481 sublen: output array of 259 elements, or null. Has, for each length, the
2482 smallest distance required to reach this length. Only 256 of its 259 values
2483 are used, the first 3 are ignored (the shortest length is 3. It is purely
2484 for convenience that the array is made 3 longer).
2485 */
2486 void ZopfliFindLongestMatch(
2487 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
2488 size_t pos, size_t size, size_t limit,
2489 unsigned short* sublen, unsigned short* distance, unsigned short* length);
2490  
2491 /*
2492 Verifies if length and dist are indeed valid, only used for assertion.
2493 */
2494 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
2495 unsigned short dist, unsigned short length);
2496  
2497 /*
2498 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
2499 with the slow but better "squeeze" implementation.
2500 The result is placed in the ZopfliLZ77Store.
2501 If instart is larger than 0, it uses values before instart as starting
2502 dictionary.
2503 */
2504 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
2505 size_t instart, size_t inend,
2506 ZopfliLZ77Store* store, ZopfliHash* h);
2507  
2508 #endif /* ZOPFLI_LZ77_H_ */
2509  
2510  
2511 /*
2512 Calculates lit/len and dist pairs for given data.
2513 If instart is larger than 0, it uses values before instart as starting
2514 dictionary.
2515 */
2516 void ZopfliLZ77Optimal(ZopfliBlockState *s,
2517 const unsigned char* in, size_t instart, size_t inend,
2518 int numiterations,
2519 ZopfliLZ77Store* store);
2520  
2521 /*
2522 Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the
2523 deflate standard.
2524 The fixed tree never gives the best compression. But this gives the best
2525 possible LZ77 encoding possible with the fixed tree.
2526 This does not create or output any fixed tree, only LZ77 data optimized for
2527 using with a fixed tree.
2528 If instart is larger than 0, it uses values before instart as starting
2529 dictionary.
2530 */
2531 void ZopfliLZ77OptimalFixed(ZopfliBlockState *s,
2532 const unsigned char* in,
2533 size_t instart, size_t inend,
2534 ZopfliLZ77Store* store);
2535  
2536 #endif /* ZOPFLI_SQUEEZE_H_ */
2537  
2538 /* tree.h */
2539 /*
2540 Copyright 2011 Google Inc. All Rights Reserved.
2541  
2542 Licensed under the Apache License, Version 2.0 (the "License");
2543 you may not use this file except in compliance with the License.
2544 You may obtain a copy of the License at
2545  
2546 http://www.apache.org/licenses/LICENSE-2.0
2547  
2548 Unless required by applicable law or agreed to in writing, software
2549 distributed under the License is distributed on an "AS IS" BASIS,
2550 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2551 See the License for the specific language governing permissions and
2552 limitations under the License.
2553  
2554 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
2555 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
2556 */
2557  
2558 /*
2559 Utilities for creating and using Huffman trees.
2560 */
2561  
2562 #ifndef ZOPFLI_TREE_H_
2563 #define ZOPFLI_TREE_H_
2564  
2565 #include <string.h>
2566  
2567 /*
2568 Calculates the bitlengths for the Huffman tree, based on the counts of each
2569 symbol.
2570 */
2571 void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits,
2572 unsigned *bitlengths);
2573  
2574 /*
2575 Converts a series of Huffman tree bitlengths, to the bit values of the symbols.
2576 */
2577 void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits,
2578 unsigned* symbols);
2579  
2580 /*
2581 Calculates the entropy of each symbol, based on the counts of each symbol. The
2582 result is similar to the result of ZopfliCalculateBitLengths, but with the
2583 actual theoritical bit lengths according to the entropy. Since the resulting
2584 values are fractional, they cannot be used to encode the tree specified by
2585 DEFLATE.
2586 */
2587 void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths);
2588  
2589 #endif /* ZOPFLI_TREE_H_ */
2590  
2591 /* util.h */
2592 /*
2593 Copyright 2011 Google Inc. All Rights Reserved.
2594  
2595 Licensed under the Apache License, Version 2.0 (the "License");
2596 you may not use this file except in compliance with the License.
2597 You may obtain a copy of the License at
2598  
2599 http://www.apache.org/licenses/LICENSE-2.0
2600  
2601 Unless required by applicable law or agreed to in writing, software
2602 distributed under the License is distributed on an "AS IS" BASIS,
2603 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2604 See the License for the specific language governing permissions and
2605 limitations under the License.
2606  
2607 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
2608 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
2609 */
2610  
2611 /*
2612 Several utilities, including: #defines to try different compression results,
2613 basic deflate specification values and generic program options.
2614 */
2615  
2616 #ifndef ZOPFLI_UTIL_H_
2617 #define ZOPFLI_UTIL_H_
2618  
2619 #include <string.h>
2620 #include <stdlib.h>
2621  
2622 /* Minimum and maximum length that can be encoded in deflate. */
2623 #define ZOPFLI_MAX_MATCH 258
2624 #define ZOPFLI_MIN_MATCH 3
2625  
2626 /* Number of distinct literal/length and distance symbols in DEFLATE */
2627 #define ZOPFLI_NUM_LL 288
2628 #define ZOPFLI_NUM_D 32
2629  
2630 /*
2631 The window size for deflate. Must be a power of two. This should be 32768, the
2632 maximum possible by the deflate spec. Anything less hurts compression more than
2633 speed.
2634 */
2635 #define ZOPFLI_WINDOW_SIZE 32768
2636  
2637 /*
2638 The window mask used to wrap indices into the window. This is why the
2639 window size must be a power of two.
2640 */
2641 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
2642  
2643 /*
2644 A block structure of huge, non-smart, blocks to divide the input into, to allow
2645 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
2646 The whole compression algorithm, including the smarter block splitting, will
2647 be executed independently on each huge block.
2648 Dividing into huge blocks hurts compression, but not much relative to the size.
2649 Set it to 0 to disable master blocks.
2650 */
2651 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
2652  
2653 /*
2654 Used to initialize costs for example
2655 */
2656 #define ZOPFLI_LARGE_FLOAT 1e30
2657  
2658 /*
2659 For longest match cache. max 256. Uses huge amounts of memory but makes it
2660 faster. Uses this many times three bytes per single byte of the input data.
2661 This is so because longest match finding has to find the exact distance
2662 that belongs to each length for the best lz77 strategy.
2663 Good values: e.g. 5, 8.
2664 */
2665 #define ZOPFLI_CACHE_LENGTH 8
2666  
2667 /*
2668 limit the max hash chain hits for this hash value. This has an effect only
2669 on files where the hash value is the same very often. On these files, this
2670 gives worse compression (the value should ideally be 32768, which is the
2671 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
2672 faster on some specific files.
2673 Good value: e.g. 8192.
2674 */
2675 #define ZOPFLI_MAX_CHAIN_HITS 8192
2676  
2677 /*
2678 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
2679 consumes a lot of memory but speeds it up. No effect on compression size.
2680 */
2681 #define ZOPFLI_LONGEST_MATCH_CACHE
2682  
2683 /*
2684 Enable to remember amount of successive identical bytes in the hash chain for
2685 finding longest match
2686 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
2687 This has no effect on the compression result, and enabling it increases speed.
2688 */
2689 #define ZOPFLI_HASH_SAME
2690  
2691 /*
2692 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
2693 best length so far is long enough. This is way faster for files with lots of
2694 identical bytes, on which the compressor is otherwise too slow. Regular files
2695 are unaffected or maybe a tiny bit slower.
2696 This has no effect on the compression result, only on speed.
2697 */
2698 #define ZOPFLI_HASH_SAME_HASH
2699  
2700 /*
2701 Enable this, to avoid slowness for files which are a repetition of the same
2702 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
2703 the compression result.
2704 */
2705 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
2706  
2707 /*
2708 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
2709 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
2710 varies from file to file.
2711 */
2712 #define ZOPFLI_LAZY_MATCHING
2713  
2714 /*
2715 Appends value to dynamically allocated memory, doubling its allocation size
2716 whenever needed.
2717  
2718 value: the value to append, type T
2719 data: pointer to the dynamic array to append to, type T**
2720 size: pointer to the size of the array to append to, type size_t*. This is the
2721 size that you consider the array to be, not the internal allocation size.
2722 Precondition: allocated size of data is at least a power of two greater than or
2723 equal than *size.
2724 */
2725 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
2726 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2727 if (!((*size) & ((*size) - 1))) {\
2728 /*double alloc size if it's a power of two*/\
2729 void** data_void = reinterpret_cast<void**>(data);\
2730 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
2731 : realloc((*data), (*size) * 2 * sizeof(**data));\
2732 }\
2733 (*data)[(*size)] = (value);\
2734 (*size)++;\
2735 }
2736 #else /* C gives problems with strict-aliasing rules for (void**) cast */
2737 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
2738 if (!((*size) & ((*size) - 1))) {\
2739 /*double alloc size if it's a power of two*/\
2740 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
2741 : realloc((*data), (*size) * 2 * sizeof(**data));\
2742 }\
2743 (*data)[(*size)] = (value);\
2744 (*size)++;\
2745 }
2746 #endif
2747  
2748  
2749 #endif /* ZOPFLI_UTIL_H_ */
2750  
2751  
2752 /*
2753 The "f" for the FindMinimum function below.
2754 i: the current parameter of f(i)
2755 context: for your implementation
2756 */
2757 typedef double FindMinimumFun(size_t i, void* context);
2758  
2759 /*
2760 Finds minimum of function f(i) where is is of type size_t, f(i) is of type
2761 double, i is in range start-end (excluding end).
2762 Outputs the minimum value in *smallest and returns the index of this value.
2763 */
2764 4 static size_t FindMinimum(FindMinimumFun f, void* context,
2765 size_t start, size_t end, double* smallest) {
2766 4 ▶ if (end - start < 1024) {
Branch: 1 branch
BranchTrueFalseTotal
if (end - start < 1024) { 4 0 4
2767 4 double best = ZOPFLI_LARGE_FLOAT;
2768 4 size_t result = start;
2769 size_t i;
2770 98 ▶ for (i = start; i < end; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = start; i < end; i++) { 94 4 98
2771 94 double v = f(i, context);
2772 94 ▶ if (v < best) {
Branch: 1 branch
BranchTrueFalseTotal
if (v < best) { 7 87 94
2773 7 best = v;
2774 7 result = i;
2775 }
2776 }
2777 4 *smallest = best;
2778 4 return result;
2779 } else {
2780 /* Try to find minimum faster by recursively checking multiple points. */
2781 #define NUM 9 /* Good value: 9. */
2782 size_t i;
2783 size_t p[NUM];
2784 double vp[NUM];
2785 size_t besti;
2786 double best;
2787 0 double lastbest = ZOPFLI_LARGE_FLOAT;
2788 0 size_t pos = start;
2789  
2790 for (;;) {
2791 0 ▶ if (end - start <= NUM) break;
Branch: 1 branch
BranchTrueFalseTotal
if (end - start <= NUM) break; 0 0 0
2792  
2793 0 ▶ for (i = 0; i < NUM; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM; i++) { 0 0 0
2794 0 p[i] = start + (i + 1) * ((end - start) / (NUM + 1));
2795 0 vp[i] = f(p[i], context);
2796 }
2797 0 besti = 0;
2798 0 best = vp[0];
2799 0 ▶ for (i = 1; i < NUM; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 1; i < NUM; i++) { 0 0 0
2800 0 ▶ if (vp[i] < best) {
Branch: 1 branch
BranchTrueFalseTotal
if (vp[i] < best) { 0 0 0
2801 0 best = vp[i];
2802 0 besti = i;
2803 }
2804 }
2805 0 ▶ if (best > lastbest) break;
Branch: 1 branch
BranchTrueFalseTotal
if (best > lastbest) break; 0 0 0
2806  
2807 0 ▶ start = besti == 0 ? start : p[besti - 1];
Branch: 1 branch
BranchTrueFalseTotal
start = besti == 0 ? start : p[besti - 1]; 0 0 0
2808 0 ▶ end = besti == NUM - 1 ? end : p[besti + 1];
Branch: 1 branch
BranchTrueFalseTotal
end = besti == NUM - 1 ? end : p[besti + 1]; 0 0 0
2809  
2810 0 pos = p[besti];
2811 0 lastbest = best;
2812 }
2813 0 *smallest = lastbest;
2814 0 return pos;
2815 #undef NUM
2816 }
2817 }
2818  
2819 /*
2820 Returns estimated cost of a block in bits. It includes the size to encode the
2821 tree and the size to encode all literal, length and distance symbols and their
2822 extra bits.
2823  
2824 litlens: lz77 lit/lengths
2825 dists: ll77 distances
2826 lstart: start of block
2827 lend: end of block (not inclusive)
2828 */
2829 192 static double EstimateCost(const ZopfliLZ77Store* lz77,
2830 size_t lstart, size_t lend) {
2831 192 return ZopfliCalculateBlockSizeAutoType(lz77, lstart, lend);
2832 }
2833  
2834 typedef struct SplitCostContext {
2835 const ZopfliLZ77Store* lz77;
2836 size_t start;
2837 size_t end;
2838 } SplitCostContext;
2839  
2840  
2841 /*
2842 Gets the cost which is the sum of the cost of the left and the right section
2843 of the data.
2844 type: FindMinimumFun
2845 */
2846 94 static double SplitCost(size_t i, void* context) {
2847 94 SplitCostContext* c = (SplitCostContext*)context;
2848 94 return EstimateCost(c->lz77, c->start, i) + EstimateCost(c->lz77, i, c->end);
2849 }
2850  
2851 1 static void AddSorted(size_t value, size_t** out, size_t* outsize) {
2852 size_t i;
2853 1 ▶ ZOPFLI_APPEND_DATA(value, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(value, out, outsize); 1 0 1
ZOPFLI_APPEND_DATA(value, out, outsize); 1 0 1
2854 1 ▶ for (i = 0; i + 1 < *outsize; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i + 1 < *outsize; i++) { 0 1 1
2855 0 ▶ if ((*out)[i] > value) {
Branch: 1 branch
BranchTrueFalseTotal
if ((*out)[i] > value) { 0 0 0
2856 size_t j;
2857 0 ▶ for (j = *outsize - 1; j > i; j--) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = *outsize - 1; j > i; j--) { 0 0 0
2858 0 (*out)[j] = (*out)[j - 1];
2859 }
2860 0 (*out)[i] = value;
2861 0 break;
2862 }
2863 }
2864 1 }
2865  
2866 /*
2867 Prints the block split points as decimal and hex values in the terminal.
2868 */
2869 0 static void PrintBlockSplitPoints(const ZopfliLZ77Store* lz77,
2870 const size_t* lz77splitpoints,
2871 size_t nlz77points) {
2872 0 size_t* splitpoints = 0;
2873 0 size_t npoints = 0;
2874 size_t i;
2875 /* The input is given as lz77 indices, but we want to see the uncompressed
2876 index values. */
2877 0 size_t pos = 0;
2878 0 ▶ if (nlz77points > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (nlz77points > 0) { 0 0 0
2879 0 ▶ for (i = 0; i < lz77->size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < lz77->size; i++) { 0 0 0
2880 0 ▶ size_t length = lz77->dists[i] == 0 ? 1 : lz77->litlens[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t length = lz77->dists[i] == 0 ? 1 : lz77->litlens[i]; 0 0 0
2881 0 ▶ if (lz77splitpoints[npoints] == i) {
Branch: 1 branch
BranchTrueFalseTotal
if (lz77splitpoints[npoints] == i) { 0 0 0
2882 0 ▶ ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints); 0 0 0
ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints); 0 0 0
2883 0 ▶ if (npoints == nlz77points) break;
Branch: 1 branch
BranchTrueFalseTotal
if (npoints == nlz77points) break; 0 0 0
2884 }
2885 0 pos += length;
2886 }
2887 }
2888 assert(npoints == nlz77points);
2889  
2890 0 fprintf(stderr, "block split points: ");
2891 0 ▶ for (i = 0; i < npoints; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < npoints; i++) { 0 0 0
2892 0 fprintf(stderr, "%d ", (int)splitpoints[i]);
2893 }
2894 0 fprintf(stderr, "(hex:");
2895 0 ▶ for (i = 0; i < npoints; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < npoints; i++) { 0 0 0
2896 0 fprintf(stderr, " %x", (int)splitpoints[i]);
2897 }
2898 0 fprintf(stderr, ")\n");
2899  
2900 0 free(splitpoints);
2901 0 }
2902  
2903 /*
2904 Finds next block to try to split, the largest of the available ones.
2905 The largest is chosen to make sure that if only a limited amount of blocks is
2906 requested, their sizes are spread evenly.
2907 lz77size: the size of the LL77 data, which is the size of the done array here.
2908 done: array indicating which blocks starting at that position are no longer
2909 splittable (splitting them increases rather than decreases cost).
2910 splitpoints: the splitpoints found so far.
2911 npoints: the amount of splitpoints found so far.
2912 lstart: output variable, giving start of block.
2913 lend: output variable, giving end of block.
2914 returns 1 if a block was found, 0 if no block found (all are done).
2915 */
2916 4 static int FindLargestSplittableBlock(
2917 size_t lz77size, const unsigned char* done,
2918 const size_t* splitpoints, size_t npoints,
2919 size_t* lstart, size_t* lend) {
2920 4 size_t longest = 0;
2921 4 int found = 0;
2922 size_t i;
2923 10 ▶ for (i = 0; i <= npoints; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i <= npoints; i++) { 6 4 10
2924 6 ▶ size_t start = i == 0 ? 0 : splitpoints[i - 1];
Branch: 1 branch
BranchTrueFalseTotal
size_t start = i == 0 ? 0 : splitpoints[i - 1]; 2 4 6
2925 6 ▶ size_t end = i == npoints ? lz77size - 1 : splitpoints[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t end = i == npoints ? lz77size - 1 : splitpoints[i]; 4 2 6
2926 6 ▶ if (!done[start] && end - start > longest) {
Branch: 2 branches
BranchTrueFalseTotal
if (!done[start] && end - start > longest) { 3 3 6
if (!done[start] && end - start > longest) { 3 0 3
2927 3 *lstart = start;
2928 3 *lend = end;
2929 3 found = 1;
2930 3 longest = end - start;
2931 }
2932 }
2933 4 return found;
2934 }
2935  
2936 3 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
2937 const ZopfliLZ77Store* lz77, size_t maxblocks,
2938 size_t** splitpoints, size_t* npoints) {
2939 size_t lstart, lend;
2940 size_t i;
2941 3 size_t llpos = 0;
2942 3 size_t numblocks = 1;
2943 unsigned char* done;
2944 double splitcost, origcost;
2945  
2946 3 ▶ if (lz77->size < 10) return; /* This code fails on tiny files. */
Branch: 1 branch
BranchTrueFalseTotal
if (lz77->size < 10) return; /* This code fails on tiny files. */ 0 3 3
2947  
2948 3 done = (unsigned char*)malloc(lz77->size);
2949 3 ▶ if (!done) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!done) exit(-1); /* Allocation failed. */ 0 3 3
2950 67 ▶ for (i = 0; i < lz77->size; i++) done[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < lz77->size; i++) done[i] = 0; 64 3 67
2951  
2952 3 lstart = 0;
2953 3 lend = lz77->size;
2954 1 for (;;) {
2955 SplitCostContext c;
2956  
2957 4 ▶ if (maxblocks > 0 && numblocks >= maxblocks) {
Branch: 2 branches
BranchTrueFalseTotal
if (maxblocks > 0 && numblocks >= maxblocks) { 4 0 4
if (maxblocks > 0 && numblocks >= maxblocks) { 0 4 4
2958 0 break;
2959 }
2960  
2961 4 c.lz77 = lz77;
2962 4 c.start = lstart;
2963 4 c.end = lend;
2964 assert(lstart < lend);
2965 4 llpos = FindMinimum(SplitCost, &c, lstart + 1, lend, &splitcost);
2966  
2967 assert(llpos > lstart);
2968 assert(llpos < lend);
2969  
2970 4 origcost = EstimateCost(lz77, lstart, lend);
2971  
2972 4 ▶ if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) {
Branch: 3 branches
BranchTrueFalseTotal
if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) { 1 3 4
if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) { 1 0 1
if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) { 0 1 1
2973 3 done[lstart] = 1;
2974 } else {
2975 1 AddSorted(llpos, splitpoints, npoints);
2976 1 numblocks++;
2977 }
2978  
2979 4 ▶ if (!FindLargestSplittableBlock(
Branch: 1 branch
BranchTrueFalseTotal
if (!FindLargestSplittableBlock( 2 2 4
2980 4 lz77->size, done, *splitpoints, *npoints, &lstart, &lend)) {
2981 2 break; /* No further split will probably reduce compression. */
2982 }
2983  
2984 2 ▶ if (lend - lstart < 10) {
Branch: 1 branch
BranchTrueFalseTotal
if (lend - lstart < 10) { 1 1 2
2985 1 break;
2986 }
2987 }
2988  
2989 3 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 3 3
2990 0 PrintBlockSplitPoints(lz77, *splitpoints, *npoints);
2991 }
2992  
2993 3 free(done);
2994 }
2995  
2996 3 void ZopfliBlockSplit(const ZopfliOptions* options,
2997 const unsigned char* in, size_t instart, size_t inend,
2998 size_t maxblocks, size_t** splitpoints, size_t* npoints) {
2999 3 size_t pos = 0;
3000 size_t i;
3001 ZopfliBlockState s;
3002 3 size_t* lz77splitpoints = 0;
3003 3 size_t nlz77points = 0;
3004 ZopfliLZ77Store store;
3005 ZopfliHash hash;
3006 3 ZopfliHash* h = &hash;
3007  
3008 3 ZopfliInitLZ77Store(in, &store);
3009 3 ZopfliInitBlockState(options, instart, inend, 0, &s);
3010 3 ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
3011  
3012 3 *npoints = 0;
3013 3 *splitpoints = 0;
3014  
3015 /* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal
3016 results in better blocks. */
3017 3 ZopfliLZ77Greedy(&s, in, instart, inend, &store, h);
3018  
3019 3 ZopfliBlockSplitLZ77(options,
3020 &store, maxblocks,
3021 &lz77splitpoints, &nlz77points);
3022  
3023 /* Convert LZ77 positions to positions in the uncompressed input. */
3024 3 pos = instart;
3025 3 ▶ if (nlz77points > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (nlz77points > 0) { 1 2 3
3026 10 ▶ for (i = 0; i < store.size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < store.size; i++) { 10 0 10
3027 10 ▶ size_t length = store.dists[i] == 0 ? 1 : store.litlens[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t length = store.dists[i] == 0 ? 1 : store.litlens[i]; 1 9 10
3028 10 ▶ if (lz77splitpoints[*npoints] == i) {
Branch: 1 branch
BranchTrueFalseTotal
if (lz77splitpoints[*npoints] == i) { 1 9 10
3029 1 ▶ ZOPFLI_APPEND_DATA(pos, splitpoints, npoints);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(pos, splitpoints, npoints); 1 0 1
ZOPFLI_APPEND_DATA(pos, splitpoints, npoints); 1 0 1
3030 1 ▶ if (*npoints == nlz77points) break;
Branch: 1 branch
BranchTrueFalseTotal
if (*npoints == nlz77points) break; 1 0 1
3031 }
3032 9 pos += length;
3033 }
3034 }
3035 assert(*npoints == nlz77points);
3036  
3037 3 free(lz77splitpoints);
3038 3 ZopfliCleanBlockState(&s);
3039 3 ZopfliCleanLZ77Store(&store);
3040 3 ZopfliCleanHash(h);
3041 3 }
3042  
3043 0 void ZopfliBlockSplitSimple(const unsigned char* in,
3044 size_t instart, size_t inend,
3045 size_t blocksize,
3046 size_t** splitpoints, size_t* npoints) {
3047 0 size_t i = instart;
3048 0 ▶ while (i < inend) {
Branch: 1 branch
BranchTrueFalseTotal
while (i < inend) { 0 0 0
3049 0 ▶ ZOPFLI_APPEND_DATA(i, splitpoints, npoints);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(i, splitpoints, npoints); 0 0 0
ZOPFLI_APPEND_DATA(i, splitpoints, npoints); 0 0 0
3050 0 i += blocksize;
3051 }
3052 (void)in;
3053 0 }
3054 /*
3055 Copyright 2011 Google Inc. All Rights Reserved.
3056  
3057 Licensed under the Apache License, Version 2.0 (the "License");
3058 you may not use this file except in compliance with the License.
3059 You may obtain a copy of the License at
3060  
3061 http://www.apache.org/licenses/LICENSE-2.0
3062  
3063 Unless required by applicable law or agreed to in writing, software
3064 distributed under the License is distributed on an "AS IS" BASIS,
3065 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3066 See the License for the specific language governing permissions and
3067 limitations under the License.
3068  
3069 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3070 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3071 */
3072  
3073 /* cache.h */
3074 /*
3075 Copyright 2011 Google Inc. All Rights Reserved.
3076  
3077 Licensed under the Apache License, Version 2.0 (the "License");
3078 you may not use this file except in compliance with the License.
3079 You may obtain a copy of the License at
3080  
3081 http://www.apache.org/licenses/LICENSE-2.0
3082  
3083 Unless required by applicable law or agreed to in writing, software
3084 distributed under the License is distributed on an "AS IS" BASIS,
3085 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3086 See the License for the specific language governing permissions and
3087 limitations under the License.
3088  
3089 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3090 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3091 */
3092  
3093 /*
3094 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
3095 */
3096  
3097 #ifndef ZOPFLI_CACHE_H_
3098 #define ZOPFLI_CACHE_H_
3099  
3100 /* util.h */
3101 /*
3102 Copyright 2011 Google Inc. All Rights Reserved.
3103  
3104 Licensed under the Apache License, Version 2.0 (the "License");
3105 you may not use this file except in compliance with the License.
3106 You may obtain a copy of the License at
3107  
3108 http://www.apache.org/licenses/LICENSE-2.0
3109  
3110 Unless required by applicable law or agreed to in writing, software
3111 distributed under the License is distributed on an "AS IS" BASIS,
3112 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3113 See the License for the specific language governing permissions and
3114 limitations under the License.
3115  
3116 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3117 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3118 */
3119  
3120 /*
3121 Several utilities, including: #defines to try different compression results,
3122 basic deflate specification values and generic program options.
3123 */
3124  
3125 #ifndef ZOPFLI_UTIL_H_
3126 #define ZOPFLI_UTIL_H_
3127  
3128 #include <string.h>
3129 #include <stdlib.h>
3130  
3131 /* Minimum and maximum length that can be encoded in deflate. */
3132 #define ZOPFLI_MAX_MATCH 258
3133 #define ZOPFLI_MIN_MATCH 3
3134  
3135 /* Number of distinct literal/length and distance symbols in DEFLATE */
3136 #define ZOPFLI_NUM_LL 288
3137 #define ZOPFLI_NUM_D 32
3138  
3139 /*
3140 The window size for deflate. Must be a power of two. This should be 32768, the
3141 maximum possible by the deflate spec. Anything less hurts compression more than
3142 speed.
3143 */
3144 #define ZOPFLI_WINDOW_SIZE 32768
3145  
3146 /*
3147 The window mask used to wrap indices into the window. This is why the
3148 window size must be a power of two.
3149 */
3150 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
3151  
3152 /*
3153 A block structure of huge, non-smart, blocks to divide the input into, to allow
3154 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
3155 The whole compression algorithm, including the smarter block splitting, will
3156 be executed independently on each huge block.
3157 Dividing into huge blocks hurts compression, but not much relative to the size.
3158 Set it to 0 to disable master blocks.
3159 */
3160 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
3161  
3162 /*
3163 Used to initialize costs for example
3164 */
3165 #define ZOPFLI_LARGE_FLOAT 1e30
3166  
3167 /*
3168 For longest match cache. max 256. Uses huge amounts of memory but makes it
3169 faster. Uses this many times three bytes per single byte of the input data.
3170 This is so because longest match finding has to find the exact distance
3171 that belongs to each length for the best lz77 strategy.
3172 Good values: e.g. 5, 8.
3173 */
3174 #define ZOPFLI_CACHE_LENGTH 8
3175  
3176 /*
3177 limit the max hash chain hits for this hash value. This has an effect only
3178 on files where the hash value is the same very often. On these files, this
3179 gives worse compression (the value should ideally be 32768, which is the
3180 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
3181 faster on some specific files.
3182 Good value: e.g. 8192.
3183 */
3184 #define ZOPFLI_MAX_CHAIN_HITS 8192
3185  
3186 /*
3187 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
3188 consumes a lot of memory but speeds it up. No effect on compression size.
3189 */
3190 #define ZOPFLI_LONGEST_MATCH_CACHE
3191  
3192 /*
3193 Enable to remember amount of successive identical bytes in the hash chain for
3194 finding longest match
3195 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
3196 This has no effect on the compression result, and enabling it increases speed.
3197 */
3198 #define ZOPFLI_HASH_SAME
3199  
3200 /*
3201 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
3202 best length so far is long enough. This is way faster for files with lots of
3203 identical bytes, on which the compressor is otherwise too slow. Regular files
3204 are unaffected or maybe a tiny bit slower.
3205 This has no effect on the compression result, only on speed.
3206 */
3207 #define ZOPFLI_HASH_SAME_HASH
3208  
3209 /*
3210 Enable this, to avoid slowness for files which are a repetition of the same
3211 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
3212 the compression result.
3213 */
3214 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
3215  
3216 /*
3217 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
3218 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
3219 varies from file to file.
3220 */
3221 #define ZOPFLI_LAZY_MATCHING
3222  
3223 /*
3224 Appends value to dynamically allocated memory, doubling its allocation size
3225 whenever needed.
3226  
3227 value: the value to append, type T
3228 data: pointer to the dynamic array to append to, type T**
3229 size: pointer to the size of the array to append to, type size_t*. This is the
3230 size that you consider the array to be, not the internal allocation size.
3231 Precondition: allocated size of data is at least a power of two greater than or
3232 equal than *size.
3233 */
3234 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
3235 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3236 if (!((*size) & ((*size) - 1))) {\
3237 /*double alloc size if it's a power of two*/\
3238 void** data_void = reinterpret_cast<void**>(data);\
3239 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
3240 : realloc((*data), (*size) * 2 * sizeof(**data));\
3241 }\
3242 (*data)[(*size)] = (value);\
3243 (*size)++;\
3244 }
3245 #else /* C gives problems with strict-aliasing rules for (void**) cast */
3246 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3247 if (!((*size) & ((*size) - 1))) {\
3248 /*double alloc size if it's a power of two*/\
3249 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
3250 : realloc((*data), (*size) * 2 * sizeof(**data));\
3251 }\
3252 (*data)[(*size)] = (value);\
3253 (*size)++;\
3254 }
3255 #endif
3256  
3257  
3258 #endif /* ZOPFLI_UTIL_H_ */
3259  
3260  
3261 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
3262  
3263 /*
3264 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
3265 values.
3266 This is needed because the squeeze runs will ask these values multiple times for
3267 the same position.
3268 Uses large amounts of memory, since it has to remember the distance belonging
3269 to every possible shorter-than-the-best length (the so called "sublen" array).
3270 */
3271 typedef struct ZopfliLongestMatchCache {
3272 unsigned short* length;
3273 unsigned short* dist;
3274 unsigned char* sublen;
3275 } ZopfliLongestMatchCache;
3276  
3277 /* Initializes the ZopfliLongestMatchCache. */
3278 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
3279  
3280 /* Frees up the memory of the ZopfliLongestMatchCache. */
3281 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
3282  
3283 /* Stores sublen array in the cache. */
3284 void ZopfliSublenToCache(const unsigned short* sublen,
3285 size_t pos, size_t length,
3286 ZopfliLongestMatchCache* lmc);
3287  
3288 /* Extracts sublen array from the cache. */
3289 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
3290 size_t pos, size_t length,
3291 unsigned short* sublen);
3292 /* Returns the length up to which could be stored in the cache. */
3293 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
3294 size_t pos, size_t length);
3295  
3296 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
3297  
3298 #endif /* ZOPFLI_CACHE_H_ */
3299  
3300  
3301 #include <assert.h>
3302 #include <stdio.h>
3303 #include <stdlib.h>
3304  
3305 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
3306  
3307 8 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc) {
3308 size_t i;
3309 8 lmc->length = (unsigned short*)malloc(sizeof(unsigned short) * blocksize);
3310 8 lmc->dist = (unsigned short*)malloc(sizeof(unsigned short) * blocksize);
3311 /* Rather large amount of memory. */
3312 8 lmc->sublen = (unsigned char*)malloc((unsigned long) (ZOPFLI_CACHE_LENGTH * 3 * blocksize));
3313 8 ▶ if(lmc->sublen == NULL) {
Branch: 1 branch
BranchTrueFalseTotal
if(lmc->sublen == NULL) { 0 8 8
3314 0 fprintf(stderr,
3315 "Error: Out of memory. Tried allocating %lu bytes of memory.\n",
3316 0 (unsigned long) (ZOPFLI_CACHE_LENGTH * 3 * blocksize));
3317 0 exit (EXIT_FAILURE);
3318 }
3319  
3320 /* length > 0 and dist 0 is invalid combination, which indicates on purpose
3321 that this cache value is not filled in yet. */
3322 18368 ▶ for (i = 0; i < blocksize; i++) lmc->length[i] = 1;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < blocksize; i++) lmc->length[i] = 1; 18360 8 18368
3323 18368 ▶ for (i = 0; i < blocksize; i++) lmc->dist[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < blocksize; i++) lmc->dist[i] = 0; 18360 8 18368
3324 440648 ▶ for (i = 0; i < ZOPFLI_CACHE_LENGTH * blocksize * 3; i++) lmc->sublen[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_CACHE_LENGTH * blocksize * 3; i++) lmc->sublen[i] = 0; 440640 8 440648
3325 8 }
3326  
3327 8 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc) {
3328 8 free(lmc->length);
3329 8 free(lmc->dist);
3330 8 free(lmc->sublen);
3331 8 }
3332  
3333 17468 void ZopfliSublenToCache(const unsigned short* sublen,
3334 size_t pos, size_t length,
3335 ZopfliLongestMatchCache* lmc) {
3336 size_t i;
3337 17468 size_t j = 0;
3338 17468 unsigned bestlength = 0;
3339 unsigned char* cache;
3340  
3341 #if ZOPFLI_CACHE_LENGTH == 0
3342 return;
3343 #endif
3344  
3345 17468 cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3];
3346 17468 ▶ if (length < 3) return;
Branch: 1 branch
BranchTrueFalseTotal
if (length < 3) return; 0 17468 17468
3347 4489276 ▶ for (i = 3; i <= length; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 3; i <= length; i++) { 4471808 17468 4489276
3348 4471808 ▶ if (i == length || sublen[i] != sublen[i + 1]) {
Branch: 2 branches
BranchTrueFalseTotal
if (i == length || sublen[i] != sublen[i + 1]) { 4454340 17468 4471808
if (i == length || sublen[i] != sublen[i + 1]) { 0 4454340 4454340
3349 17468 cache[j * 3] = i - 3;
3350 17468 cache[j * 3 + 1] = sublen[i] % 256;
3351 17468 cache[j * 3 + 2] = (sublen[i] >> 8) % 256;
3352 17468 bestlength = i;
3353 17468 j++;
3354 17468 ▶ if (j >= ZOPFLI_CACHE_LENGTH) break;
Branch: 1 branch
BranchTrueFalseTotal
if (j >= ZOPFLI_CACHE_LENGTH) break; 0 17468 17468
3355 }
3356 }
3357 17468 ▶ if (j < ZOPFLI_CACHE_LENGTH) {
Branch: 1 branch
BranchTrueFalseTotal
if (j < ZOPFLI_CACHE_LENGTH) { 17468 0 17468
3358 assert(bestlength == length);
3359 17468 cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] = bestlength - 3;
3360 } else {
3361 assert(bestlength <= length);
3362 }
3363 assert(bestlength == ZopfliMaxCachedSublen(lmc, pos, length));
3364 }
3365  
3366 122310 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
3367 size_t pos, size_t length,
3368 unsigned short* sublen) {
3369 size_t i, j;
3370 122310 unsigned maxlength = ZopfliMaxCachedSublen(lmc, pos, length);
3371 122310 unsigned prevlength = 0;
3372 unsigned char* cache;
3373 #if ZOPFLI_CACHE_LENGTH == 0
3374 return;
3375 #endif
3376 122310 ▶ if (length < 3) return;
Branch: 1 branch
BranchTrueFalseTotal
if (length < 3) return; 0 122310 122310
3377 122310 cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3];
3378 122310 ▶ for (j = 0; j < ZOPFLI_CACHE_LENGTH; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 0; j < ZOPFLI_CACHE_LENGTH; j++) { 122310 0 122310
3379 122310 unsigned length = cache[j * 3] + 3;
3380 122310 unsigned dist = cache[j * 3 + 1] + 256 * cache[j * 3 + 2];
3381 31800600 ▶ for (i = prevlength; i <= length; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = prevlength; i <= length; i++) { 31678290 122310 31800600
3382 31678290 sublen[i] = dist;
3383 }
3384 122310 ▶ if (length == maxlength) break;
Branch: 1 branch
BranchTrueFalseTotal
if (length == maxlength) break; 122310 0 122310
3385 0 prevlength = length + 1;
3386 }
3387 }
3388  
3389 /*
3390 Returns the length up to which could be stored in the cache.
3391 */
3392 244620 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
3393 size_t pos, size_t length) {
3394 unsigned char* cache;
3395 #if ZOPFLI_CACHE_LENGTH == 0
3396 return 0;
3397 #endif
3398 244620 cache = &lmc->sublen[ZOPFLI_CACHE_LENGTH * pos * 3];
3399 (void)length;
3400 244620 ▶ if (cache[1] == 0 && cache[2] == 0) return 0; /* No sublen cached. */
Branch: 2 branches
BranchTrueFalseTotal
if (cache[1] == 0 && cache[2] == 0) return 0; /* No sublen cached. */ 0 244620 244620
if (cache[1] == 0 && cache[2] == 0) return 0; /* No sublen cached. */ 0 0 0
3401 244620 return cache[(ZOPFLI_CACHE_LENGTH - 1) * 3] + 3;
3402 }
3403  
3404 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
3405 /*
3406 Copyright 2011 Google Inc. All Rights Reserved.
3407  
3408 Licensed under the Apache License, Version 2.0 (the "License");
3409 you may not use this file except in compliance with the License.
3410 You may obtain a copy of the License at
3411  
3412 http://www.apache.org/licenses/LICENSE-2.0
3413  
3414 Unless required by applicable law or agreed to in writing, software
3415 distributed under the License is distributed on an "AS IS" BASIS,
3416 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3417 See the License for the specific language governing permissions and
3418 limitations under the License.
3419  
3420 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3421 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3422 */
3423  
3424 /* deflate.h */
3425 /*
3426 Copyright 2011 Google Inc. All Rights Reserved.
3427  
3428 Licensed under the Apache License, Version 2.0 (the "License");
3429 you may not use this file except in compliance with the License.
3430 You may obtain a copy of the License at
3431  
3432 http://www.apache.org/licenses/LICENSE-2.0
3433  
3434 Unless required by applicable law or agreed to in writing, software
3435 distributed under the License is distributed on an "AS IS" BASIS,
3436 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3437 See the License for the specific language governing permissions and
3438 limitations under the License.
3439  
3440 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3441 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3442 */
3443  
3444 #ifndef ZOPFLI_DEFLATE_H_
3445 #define ZOPFLI_DEFLATE_H_
3446  
3447 /*
3448 Functions to compress according to the DEFLATE specification, using the
3449 "squeeze" LZ77 compression backend.
3450 */
3451  
3452 /* lz77.h */
3453 /*
3454 Copyright 2011 Google Inc. All Rights Reserved.
3455  
3456 Licensed under the Apache License, Version 2.0 (the "License");
3457 you may not use this file except in compliance with the License.
3458 You may obtain a copy of the License at
3459  
3460 http://www.apache.org/licenses/LICENSE-2.0
3461  
3462 Unless required by applicable law or agreed to in writing, software
3463 distributed under the License is distributed on an "AS IS" BASIS,
3464 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3465 See the License for the specific language governing permissions and
3466 limitations under the License.
3467  
3468 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3469 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3470 */
3471  
3472 /*
3473 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
3474 compression.
3475 */
3476  
3477 #ifndef ZOPFLI_LZ77_H_
3478 #define ZOPFLI_LZ77_H_
3479  
3480 #include <stdlib.h>
3481  
3482 /* cache.h */
3483 /*
3484 Copyright 2011 Google Inc. All Rights Reserved.
3485  
3486 Licensed under the Apache License, Version 2.0 (the "License");
3487 you may not use this file except in compliance with the License.
3488 You may obtain a copy of the License at
3489  
3490 http://www.apache.org/licenses/LICENSE-2.0
3491  
3492 Unless required by applicable law or agreed to in writing, software
3493 distributed under the License is distributed on an "AS IS" BASIS,
3494 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3495 See the License for the specific language governing permissions and
3496 limitations under the License.
3497  
3498 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3499 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3500 */
3501  
3502 /*
3503 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
3504 */
3505  
3506 #ifndef ZOPFLI_CACHE_H_
3507 #define ZOPFLI_CACHE_H_
3508  
3509 /* util.h */
3510 /*
3511 Copyright 2011 Google Inc. All Rights Reserved.
3512  
3513 Licensed under the Apache License, Version 2.0 (the "License");
3514 you may not use this file except in compliance with the License.
3515 You may obtain a copy of the License at
3516  
3517 http://www.apache.org/licenses/LICENSE-2.0
3518  
3519 Unless required by applicable law or agreed to in writing, software
3520 distributed under the License is distributed on an "AS IS" BASIS,
3521 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3522 See the License for the specific language governing permissions and
3523 limitations under the License.
3524  
3525 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3526 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3527 */
3528  
3529 /*
3530 Several utilities, including: #defines to try different compression results,
3531 basic deflate specification values and generic program options.
3532 */
3533  
3534 #ifndef ZOPFLI_UTIL_H_
3535 #define ZOPFLI_UTIL_H_
3536  
3537 #include <string.h>
3538 #include <stdlib.h>
3539  
3540 /* Minimum and maximum length that can be encoded in deflate. */
3541 #define ZOPFLI_MAX_MATCH 258
3542 #define ZOPFLI_MIN_MATCH 3
3543  
3544 /* Number of distinct literal/length and distance symbols in DEFLATE */
3545 #define ZOPFLI_NUM_LL 288
3546 #define ZOPFLI_NUM_D 32
3547  
3548 /*
3549 The window size for deflate. Must be a power of two. This should be 32768, the
3550 maximum possible by the deflate spec. Anything less hurts compression more than
3551 speed.
3552 */
3553 #define ZOPFLI_WINDOW_SIZE 32768
3554  
3555 /*
3556 The window mask used to wrap indices into the window. This is why the
3557 window size must be a power of two.
3558 */
3559 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
3560  
3561 /*
3562 A block structure of huge, non-smart, blocks to divide the input into, to allow
3563 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
3564 The whole compression algorithm, including the smarter block splitting, will
3565 be executed independently on each huge block.
3566 Dividing into huge blocks hurts compression, but not much relative to the size.
3567 Set it to 0 to disable master blocks.
3568 */
3569 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
3570  
3571 /*
3572 Used to initialize costs for example
3573 */
3574 #define ZOPFLI_LARGE_FLOAT 1e30
3575  
3576 /*
3577 For longest match cache. max 256. Uses huge amounts of memory but makes it
3578 faster. Uses this many times three bytes per single byte of the input data.
3579 This is so because longest match finding has to find the exact distance
3580 that belongs to each length for the best lz77 strategy.
3581 Good values: e.g. 5, 8.
3582 */
3583 #define ZOPFLI_CACHE_LENGTH 8
3584  
3585 /*
3586 limit the max hash chain hits for this hash value. This has an effect only
3587 on files where the hash value is the same very often. On these files, this
3588 gives worse compression (the value should ideally be 32768, which is the
3589 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
3590 faster on some specific files.
3591 Good value: e.g. 8192.
3592 */
3593 #define ZOPFLI_MAX_CHAIN_HITS 8192
3594  
3595 /*
3596 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
3597 consumes a lot of memory but speeds it up. No effect on compression size.
3598 */
3599 #define ZOPFLI_LONGEST_MATCH_CACHE
3600  
3601 /*
3602 Enable to remember amount of successive identical bytes in the hash chain for
3603 finding longest match
3604 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
3605 This has no effect on the compression result, and enabling it increases speed.
3606 */
3607 #define ZOPFLI_HASH_SAME
3608  
3609 /*
3610 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
3611 best length so far is long enough. This is way faster for files with lots of
3612 identical bytes, on which the compressor is otherwise too slow. Regular files
3613 are unaffected or maybe a tiny bit slower.
3614 This has no effect on the compression result, only on speed.
3615 */
3616 #define ZOPFLI_HASH_SAME_HASH
3617  
3618 /*
3619 Enable this, to avoid slowness for files which are a repetition of the same
3620 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
3621 the compression result.
3622 */
3623 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
3624  
3625 /*
3626 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
3627 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
3628 varies from file to file.
3629 */
3630 #define ZOPFLI_LAZY_MATCHING
3631  
3632 /*
3633 Appends value to dynamically allocated memory, doubling its allocation size
3634 whenever needed.
3635  
3636 value: the value to append, type T
3637 data: pointer to the dynamic array to append to, type T**
3638 size: pointer to the size of the array to append to, type size_t*. This is the
3639 size that you consider the array to be, not the internal allocation size.
3640 Precondition: allocated size of data is at least a power of two greater than or
3641 equal than *size.
3642 */
3643 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
3644 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3645 if (!((*size) & ((*size) - 1))) {\
3646 /*double alloc size if it's a power of two*/\
3647 void** data_void = reinterpret_cast<void**>(data);\
3648 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
3649 : realloc((*data), (*size) * 2 * sizeof(**data));\
3650 }\
3651 (*data)[(*size)] = (value);\
3652 (*size)++;\
3653 }
3654 #else /* C gives problems with strict-aliasing rules for (void**) cast */
3655 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3656 if (!((*size) & ((*size) - 1))) {\
3657 /*double alloc size if it's a power of two*/\
3658 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
3659 : realloc((*data), (*size) * 2 * sizeof(**data));\
3660 }\
3661 (*data)[(*size)] = (value);\
3662 (*size)++;\
3663 }
3664 #endif
3665  
3666  
3667 #endif /* ZOPFLI_UTIL_H_ */
3668  
3669  
3670 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
3671  
3672 /*
3673 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
3674 values.
3675 This is needed because the squeeze runs will ask these values multiple times for
3676 the same position.
3677 Uses large amounts of memory, since it has to remember the distance belonging
3678 to every possible shorter-than-the-best length (the so called "sublen" array).
3679 */
3680 typedef struct ZopfliLongestMatchCache {
3681 unsigned short* length;
3682 unsigned short* dist;
3683 unsigned char* sublen;
3684 } ZopfliLongestMatchCache;
3685  
3686 /* Initializes the ZopfliLongestMatchCache. */
3687 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
3688  
3689 /* Frees up the memory of the ZopfliLongestMatchCache. */
3690 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
3691  
3692 /* Stores sublen array in the cache. */
3693 void ZopfliSublenToCache(const unsigned short* sublen,
3694 size_t pos, size_t length,
3695 ZopfliLongestMatchCache* lmc);
3696  
3697 /* Extracts sublen array from the cache. */
3698 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
3699 size_t pos, size_t length,
3700 unsigned short* sublen);
3701 /* Returns the length up to which could be stored in the cache. */
3702 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
3703 size_t pos, size_t length);
3704  
3705 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
3706  
3707 #endif /* ZOPFLI_CACHE_H_ */
3708  
3709 /* hash.h */
3710 /*
3711 Copyright 2011 Google Inc. All Rights Reserved.
3712  
3713 Licensed under the Apache License, Version 2.0 (the "License");
3714 you may not use this file except in compliance with the License.
3715 You may obtain a copy of the License at
3716  
3717 http://www.apache.org/licenses/LICENSE-2.0
3718  
3719 Unless required by applicable law or agreed to in writing, software
3720 distributed under the License is distributed on an "AS IS" BASIS,
3721 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3722 See the License for the specific language governing permissions and
3723 limitations under the License.
3724  
3725 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3726 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3727 */
3728  
3729 /*
3730 The hash for ZopfliFindLongestMatch of lz77.c.
3731 */
3732  
3733 #ifndef ZOPFLI_HASH_H_
3734 #define ZOPFLI_HASH_H_
3735  
3736 /* util.h */
3737 /*
3738 Copyright 2011 Google Inc. All Rights Reserved.
3739  
3740 Licensed under the Apache License, Version 2.0 (the "License");
3741 you may not use this file except in compliance with the License.
3742 You may obtain a copy of the License at
3743  
3744 http://www.apache.org/licenses/LICENSE-2.0
3745  
3746 Unless required by applicable law or agreed to in writing, software
3747 distributed under the License is distributed on an "AS IS" BASIS,
3748 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3749 See the License for the specific language governing permissions and
3750 limitations under the License.
3751  
3752 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3753 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3754 */
3755  
3756 /*
3757 Several utilities, including: #defines to try different compression results,
3758 basic deflate specification values and generic program options.
3759 */
3760  
3761 #ifndef ZOPFLI_UTIL_H_
3762 #define ZOPFLI_UTIL_H_
3763  
3764 #include <string.h>
3765 #include <stdlib.h>
3766  
3767 /* Minimum and maximum length that can be encoded in deflate. */
3768 #define ZOPFLI_MAX_MATCH 258
3769 #define ZOPFLI_MIN_MATCH 3
3770  
3771 /* Number of distinct literal/length and distance symbols in DEFLATE */
3772 #define ZOPFLI_NUM_LL 288
3773 #define ZOPFLI_NUM_D 32
3774  
3775 /*
3776 The window size for deflate. Must be a power of two. This should be 32768, the
3777 maximum possible by the deflate spec. Anything less hurts compression more than
3778 speed.
3779 */
3780 #define ZOPFLI_WINDOW_SIZE 32768
3781  
3782 /*
3783 The window mask used to wrap indices into the window. This is why the
3784 window size must be a power of two.
3785 */
3786 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
3787  
3788 /*
3789 A block structure of huge, non-smart, blocks to divide the input into, to allow
3790 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
3791 The whole compression algorithm, including the smarter block splitting, will
3792 be executed independently on each huge block.
3793 Dividing into huge blocks hurts compression, but not much relative to the size.
3794 Set it to 0 to disable master blocks.
3795 */
3796 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
3797  
3798 /*
3799 Used to initialize costs for example
3800 */
3801 #define ZOPFLI_LARGE_FLOAT 1e30
3802  
3803 /*
3804 For longest match cache. max 256. Uses huge amounts of memory but makes it
3805 faster. Uses this many times three bytes per single byte of the input data.
3806 This is so because longest match finding has to find the exact distance
3807 that belongs to each length for the best lz77 strategy.
3808 Good values: e.g. 5, 8.
3809 */
3810 #define ZOPFLI_CACHE_LENGTH 8
3811  
3812 /*
3813 limit the max hash chain hits for this hash value. This has an effect only
3814 on files where the hash value is the same very often. On these files, this
3815 gives worse compression (the value should ideally be 32768, which is the
3816 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
3817 faster on some specific files.
3818 Good value: e.g. 8192.
3819 */
3820 #define ZOPFLI_MAX_CHAIN_HITS 8192
3821  
3822 /*
3823 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
3824 consumes a lot of memory but speeds it up. No effect on compression size.
3825 */
3826 #define ZOPFLI_LONGEST_MATCH_CACHE
3827  
3828 /*
3829 Enable to remember amount of successive identical bytes in the hash chain for
3830 finding longest match
3831 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
3832 This has no effect on the compression result, and enabling it increases speed.
3833 */
3834 #define ZOPFLI_HASH_SAME
3835  
3836 /*
3837 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
3838 best length so far is long enough. This is way faster for files with lots of
3839 identical bytes, on which the compressor is otherwise too slow. Regular files
3840 are unaffected or maybe a tiny bit slower.
3841 This has no effect on the compression result, only on speed.
3842 */
3843 #define ZOPFLI_HASH_SAME_HASH
3844  
3845 /*
3846 Enable this, to avoid slowness for files which are a repetition of the same
3847 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
3848 the compression result.
3849 */
3850 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
3851  
3852 /*
3853 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
3854 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
3855 varies from file to file.
3856 */
3857 #define ZOPFLI_LAZY_MATCHING
3858  
3859 /*
3860 Appends value to dynamically allocated memory, doubling its allocation size
3861 whenever needed.
3862  
3863 value: the value to append, type T
3864 data: pointer to the dynamic array to append to, type T**
3865 size: pointer to the size of the array to append to, type size_t*. This is the
3866 size that you consider the array to be, not the internal allocation size.
3867 Precondition: allocated size of data is at least a power of two greater than or
3868 equal than *size.
3869 */
3870 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
3871 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3872 if (!((*size) & ((*size) - 1))) {\
3873 /*double alloc size if it's a power of two*/\
3874 void** data_void = reinterpret_cast<void**>(data);\
3875 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
3876 : realloc((*data), (*size) * 2 * sizeof(**data));\
3877 }\
3878 (*data)[(*size)] = (value);\
3879 (*size)++;\
3880 }
3881 #else /* C gives problems with strict-aliasing rules for (void**) cast */
3882 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
3883 if (!((*size) & ((*size) - 1))) {\
3884 /*double alloc size if it's a power of two*/\
3885 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
3886 : realloc((*data), (*size) * 2 * sizeof(**data));\
3887 }\
3888 (*data)[(*size)] = (value);\
3889 (*size)++;\
3890 }
3891 #endif
3892  
3893  
3894 #endif /* ZOPFLI_UTIL_H_ */
3895  
3896  
3897 typedef struct ZopfliHash {
3898 int* head; /* Hash value to index of its most recent occurrence. */
3899 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
3900 int* hashval; /* Index to hash value at this index. */
3901 int val; /* Current hash value. */
3902  
3903 #ifdef ZOPFLI_HASH_SAME_HASH
3904 /* Fields with similar purpose as the above hash, but for the second hash with
3905 a value that is calculated differently. */
3906 int* head2; /* Hash value to index of its most recent occurrence. */
3907 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
3908 int* hashval2; /* Index to hash value at this index. */
3909 int val2; /* Current hash value. */
3910 #endif
3911  
3912 #ifdef ZOPFLI_HASH_SAME
3913 unsigned short* same; /* Amount of repetitions of same byte after this .*/
3914 #endif
3915 } ZopfliHash;
3916  
3917 /* Allocates ZopfliHash memory. */
3918 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
3919  
3920 /* Resets all fields of ZopfliHash. */
3921 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
3922  
3923 /* Frees ZopfliHash memory. */
3924 void ZopfliCleanHash(ZopfliHash* h);
3925  
3926 /*
3927 Updates the hash values based on the current position in the array. All calls
3928 to this must be made for consecutive bytes.
3929 */
3930 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
3931 ZopfliHash* h);
3932  
3933 /*
3934 Prepopulates hash:
3935 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
3936 correctly.
3937 */
3938 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
3939 ZopfliHash* h);
3940  
3941 #endif /* ZOPFLI_HASH_H_ */
3942  
3943 /* zopfli.h */
3944 /*
3945 Copyright 2011 Google Inc. All Rights Reserved.
3946  
3947 Licensed under the Apache License, Version 2.0 (the "License");
3948 you may not use this file except in compliance with the License.
3949 You may obtain a copy of the License at
3950  
3951 http://www.apache.org/licenses/LICENSE-2.0
3952  
3953 Unless required by applicable law or agreed to in writing, software
3954 distributed under the License is distributed on an "AS IS" BASIS,
3955 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3956 See the License for the specific language governing permissions and
3957 limitations under the License.
3958  
3959 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
3960 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
3961 */
3962  
3963 #ifndef ZOPFLI_ZOPFLI_H_
3964 #define ZOPFLI_ZOPFLI_H_
3965  
3966 #include <stddef.h>
3967 #include <stdlib.h> /* for size_t */
3968  
3969 #ifdef __cplusplus
3970 extern "C" {
3971 #endif
3972  
3973 /*
3974 Options used throughout the program.
3975 */
3976 typedef struct ZopfliOptions {
3977 /* Whether to print output */
3978 int verbose;
3979  
3980 /* Whether to print more detailed output */
3981 int verbose_more;
3982  
3983 /*
3984 Maximum amount of times to rerun forward and backward pass to optimize LZ77
3985 compression cost. Good values: 10, 15 for small files, 5 for files over
3986 several MB in size or it will be too slow.
3987 */
3988 int numiterations;
3989  
3990 /*
3991 If true, splits the data in multiple deflate blocks with optimal choice
3992 for the block boundaries. Block splitting gives better compression. Default:
3993 true (1).
3994 */
3995 int blocksplitting;
3996  
3997 /*
3998 No longer used, left for compatibility.
3999 */
4000 int blocksplittinglast;
4001  
4002 /*
4003 Maximum amount of blocks to split into (0 for unlimited, but this can give
4004 extreme results that hurt compression on some files). Default value: 15.
4005 */
4006 int blocksplittingmax;
4007 } ZopfliOptions;
4008  
4009 /* Initializes options with default values. */
4010 void ZopfliInitOptions(ZopfliOptions* options);
4011  
4012 /* Output format */
4013 typedef enum {
4014 ZOPFLI_FORMAT_GZIP,
4015 ZOPFLI_FORMAT_ZLIB,
4016 ZOPFLI_FORMAT_DEFLATE
4017 } ZopfliFormat;
4018  
4019 /*
4020 Compresses according to the given output format and appends the result to the
4021 output.
4022  
4023 options: global program options
4024 output_type: the output format to use
4025 out: pointer to the dynamic output array to which the result is appended. Must
4026 be freed after use
4027 outsize: pointer to the dynamic output array size
4028 */
4029 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
4030 const unsigned char* in, size_t insize,
4031 unsigned char** out, size_t* outsize);
4032  
4033 #ifdef __cplusplus
4034 } // extern "C"
4035 #endif
4036  
4037 #endif /* ZOPFLI_ZOPFLI_H_ */
4038  
4039  
4040 /*
4041 Stores lit/length and dist pairs for LZ77.
4042 Parameter litlens: Contains the literal symbols or length values.
4043 Parameter dists: Contains the distances. A value is 0 to indicate that there is
4044 no dist and the corresponding litlens value is a literal instead of a length.
4045 Parameter size: The size of both the litlens and dists arrays.
4046 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
4047 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
4048  
4049 */
4050 typedef struct ZopfliLZ77Store {
4051 unsigned short* litlens; /* Lit or len. */
4052 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
4053 if > 0: length in corresponding litlens, this is the distance. */
4054 size_t size;
4055  
4056 const unsigned char* data; /* original data */
4057 size_t* pos; /* position in data where this LZ77 command begins */
4058  
4059 unsigned short* ll_symbol;
4060 unsigned short* d_symbol;
4061  
4062 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
4063 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
4064 precise histogram at every N symbols, and the rest can be calculated by
4065 looping through the actual symbols of this chunk. */
4066 size_t* ll_counts;
4067 size_t* d_counts;
4068 } ZopfliLZ77Store;
4069  
4070 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
4071 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
4072 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
4073 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
4074 size_t pos, ZopfliLZ77Store* store);
4075 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
4076 ZopfliLZ77Store* target);
4077 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
4078 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
4079 size_t lstart, size_t lend);
4080 /* Gets the histogram of lit/len and dist symbols in the given range, using the
4081 cumulative histograms, so faster than adding one by one for large range. Does
4082 not add the one end symbol of value 256. */
4083 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
4084 size_t lstart, size_t lend,
4085 size_t* ll_counts, size_t* d_counts);
4086  
4087 /*
4088 Some state information for compressing a block.
4089 This is currently a bit under-used (with mainly only the longest match cache),
4090 but is kept for easy future expansion.
4091 */
4092 typedef struct ZopfliBlockState {
4093 const ZopfliOptions* options;
4094  
4095 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
4096 /* Cache for length/distance pairs found so far. */
4097 ZopfliLongestMatchCache* lmc;
4098 #endif
4099  
4100 /* The start (inclusive) and end (not inclusive) of the current block. */
4101 size_t blockstart;
4102 size_t blockend;
4103 } ZopfliBlockState;
4104  
4105 void ZopfliInitBlockState(const ZopfliOptions* options,
4106 size_t blockstart, size_t blockend, int add_lmc,
4107 ZopfliBlockState* s);
4108 void ZopfliCleanBlockState(ZopfliBlockState* s);
4109  
4110 /*
4111 Finds the longest match (length and corresponding distance) for LZ77
4112 compression.
4113 Even when not using "sublen", it can be more efficient to provide an array,
4114 because only then the caching is used.
4115 array: the data
4116 pos: position in the data to find the match for
4117 size: size of the data
4118 limit: limit length to maximum this value (default should be 258). This allows
4119 finding a shorter dist for that length (= less extra bits). Must be
4120 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
4121 sublen: output array of 259 elements, or null. Has, for each length, the
4122 smallest distance required to reach this length. Only 256 of its 259 values
4123 are used, the first 3 are ignored (the shortest length is 3. It is purely
4124 for convenience that the array is made 3 longer).
4125 */
4126 void ZopfliFindLongestMatch(
4127 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
4128 size_t pos, size_t size, size_t limit,
4129 unsigned short* sublen, unsigned short* distance, unsigned short* length);
4130  
4131 /*
4132 Verifies if length and dist are indeed valid, only used for assertion.
4133 */
4134 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
4135 unsigned short dist, unsigned short length);
4136  
4137 /*
4138 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
4139 with the slow but better "squeeze" implementation.
4140 The result is placed in the ZopfliLZ77Store.
4141 If instart is larger than 0, it uses values before instart as starting
4142 dictionary.
4143 */
4144 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
4145 size_t instart, size_t inend,
4146 ZopfliLZ77Store* store, ZopfliHash* h);
4147  
4148 #endif /* ZOPFLI_LZ77_H_ */
4149  
4150 /* zopfli.h */
4151 /*
4152 Copyright 2011 Google Inc. All Rights Reserved.
4153  
4154 Licensed under the Apache License, Version 2.0 (the "License");
4155 you may not use this file except in compliance with the License.
4156 You may obtain a copy of the License at
4157  
4158 http://www.apache.org/licenses/LICENSE-2.0
4159  
4160 Unless required by applicable law or agreed to in writing, software
4161 distributed under the License is distributed on an "AS IS" BASIS,
4162 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4163 See the License for the specific language governing permissions and
4164 limitations under the License.
4165  
4166 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4167 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4168 */
4169  
4170 #ifndef ZOPFLI_ZOPFLI_H_
4171 #define ZOPFLI_ZOPFLI_H_
4172  
4173 #include <stddef.h>
4174 #include <stdlib.h> /* for size_t */
4175  
4176 #ifdef __cplusplus
4177 extern "C" {
4178 #endif
4179  
4180 /*
4181 Options used throughout the program.
4182 */
4183 typedef struct ZopfliOptions {
4184 /* Whether to print output */
4185 int verbose;
4186  
4187 /* Whether to print more detailed output */
4188 int verbose_more;
4189  
4190 /*
4191 Maximum amount of times to rerun forward and backward pass to optimize LZ77
4192 compression cost. Good values: 10, 15 for small files, 5 for files over
4193 several MB in size or it will be too slow.
4194 */
4195 int numiterations;
4196  
4197 /*
4198 If true, splits the data in multiple deflate blocks with optimal choice
4199 for the block boundaries. Block splitting gives better compression. Default:
4200 true (1).
4201 */
4202 int blocksplitting;
4203  
4204 /*
4205 No longer used, left for compatibility.
4206 */
4207 int blocksplittinglast;
4208  
4209 /*
4210 Maximum amount of blocks to split into (0 for unlimited, but this can give
4211 extreme results that hurt compression on some files). Default value: 15.
4212 */
4213 int blocksplittingmax;
4214 } ZopfliOptions;
4215  
4216 /* Initializes options with default values. */
4217 void ZopfliInitOptions(ZopfliOptions* options);
4218  
4219 /* Output format */
4220 typedef enum {
4221 ZOPFLI_FORMAT_GZIP,
4222 ZOPFLI_FORMAT_ZLIB,
4223 ZOPFLI_FORMAT_DEFLATE
4224 } ZopfliFormat;
4225  
4226 /*
4227 Compresses according to the given output format and appends the result to the
4228 output.
4229  
4230 options: global program options
4231 output_type: the output format to use
4232 out: pointer to the dynamic output array to which the result is appended. Must
4233 be freed after use
4234 outsize: pointer to the dynamic output array size
4235 */
4236 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
4237 const unsigned char* in, size_t insize,
4238 unsigned char** out, size_t* outsize);
4239  
4240 #ifdef __cplusplus
4241 } // extern "C"
4242 #endif
4243  
4244 #endif /* ZOPFLI_ZOPFLI_H_ */
4245  
4246  
4247 #ifdef __cplusplus
4248 extern "C" {
4249 #endif
4250  
4251 /*
4252 Compresses according to the deflate specification and append the compressed
4253 result to the output.
4254 This function will usually output multiple deflate blocks. If final is 1, then
4255 the final bit will be set on the last block.
4256  
4257 options: global program options
4258 btype: the deflate block type. Use 2 for best compression.
4259 -0: non compressed blocks (00)
4260 -1: blocks with fixed tree (01)
4261 -2: blocks with dynamic tree (10)
4262 final: whether this is the last section of the input, sets the final bit to the
4263 last deflate block.
4264 in: the input bytes
4265 insize: number of input bytes
4266 bp: bit pointer for the output array. This must initially be 0, and for
4267 consecutive calls must be reused (it can have values from 0-7). This is
4268 because deflate appends blocks as bit-based data, rather than on byte
4269 boundaries.
4270 out: pointer to the dynamic output array to which the result is appended. Must
4271 be freed after use.
4272 outsize: pointer to the dynamic output array size.
4273 */
4274 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
4275 const unsigned char* in, size_t insize,
4276 unsigned char* bp, unsigned char** out, size_t* outsize);
4277  
4278 /*
4279 Like ZopfliDeflate, but allows to specify start and end byte with instart and
4280 inend. Only that part is compressed, but earlier bytes are still used for the
4281 back window.
4282 */
4283 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
4284 const unsigned char* in, size_t instart, size_t inend,
4285 unsigned char* bp, unsigned char** out,
4286 size_t* outsize);
4287  
4288 /*
4289 Calculates block size in bits.
4290 litlens: lz77 lit/lengths
4291 dists: ll77 distances
4292 lstart: start of block
4293 lend: end of block (not inclusive)
4294 */
4295 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
4296 size_t lstart, size_t lend, int btype);
4297  
4298 /*
4299 Calculates block size in bits, automatically using the best btype.
4300 */
4301 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
4302 size_t lstart, size_t lend);
4303  
4304 #ifdef __cplusplus
4305 } // extern "C"
4306 #endif
4307  
4308 #endif /* ZOPFLI_DEFLATE_H_ */
4309  
4310  
4311 #include <assert.h>
4312 #include <stdio.h>
4313 #include <stdlib.h>
4314  
4315 /* blocksplitter.h */
4316 /*
4317 Copyright 2011 Google Inc. All Rights Reserved.
4318  
4319 Licensed under the Apache License, Version 2.0 (the "License");
4320 you may not use this file except in compliance with the License.
4321 You may obtain a copy of the License at
4322  
4323 http://www.apache.org/licenses/LICENSE-2.0
4324  
4325 Unless required by applicable law or agreed to in writing, software
4326 distributed under the License is distributed on an "AS IS" BASIS,
4327 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4328 See the License for the specific language governing permissions and
4329 limitations under the License.
4330  
4331 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4332 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4333 */
4334  
4335 /*
4336 Functions to choose good boundaries for block splitting. Deflate allows encoding
4337 the data in multiple blocks, with a separate Huffman tree for each block. The
4338 Huffman tree itself requires some bytes to encode, so by choosing certain
4339 blocks, you can either hurt, or enhance compression. These functions choose good
4340 ones that enhance it.
4341 */
4342  
4343 #ifndef ZOPFLI_BLOCKSPLITTER_H_
4344 #define ZOPFLI_BLOCKSPLITTER_H_
4345  
4346 #include <stdlib.h>
4347  
4348 /* lz77.h */
4349 /*
4350 Copyright 2011 Google Inc. All Rights Reserved.
4351  
4352 Licensed under the Apache License, Version 2.0 (the "License");
4353 you may not use this file except in compliance with the License.
4354 You may obtain a copy of the License at
4355  
4356 http://www.apache.org/licenses/LICENSE-2.0
4357  
4358 Unless required by applicable law or agreed to in writing, software
4359 distributed under the License is distributed on an "AS IS" BASIS,
4360 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4361 See the License for the specific language governing permissions and
4362 limitations under the License.
4363  
4364 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4365 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4366 */
4367  
4368 /*
4369 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
4370 compression.
4371 */
4372  
4373 #ifndef ZOPFLI_LZ77_H_
4374 #define ZOPFLI_LZ77_H_
4375  
4376 #include <stdlib.h>
4377  
4378 /* cache.h */
4379 /*
4380 Copyright 2011 Google Inc. All Rights Reserved.
4381  
4382 Licensed under the Apache License, Version 2.0 (the "License");
4383 you may not use this file except in compliance with the License.
4384 You may obtain a copy of the License at
4385  
4386 http://www.apache.org/licenses/LICENSE-2.0
4387  
4388 Unless required by applicable law or agreed to in writing, software
4389 distributed under the License is distributed on an "AS IS" BASIS,
4390 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4391 See the License for the specific language governing permissions and
4392 limitations under the License.
4393  
4394 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4395 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4396 */
4397  
4398 /*
4399 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
4400 */
4401  
4402 #ifndef ZOPFLI_CACHE_H_
4403 #define ZOPFLI_CACHE_H_
4404  
4405 /* util.h */
4406 /*
4407 Copyright 2011 Google Inc. All Rights Reserved.
4408  
4409 Licensed under the Apache License, Version 2.0 (the "License");
4410 you may not use this file except in compliance with the License.
4411 You may obtain a copy of the License at
4412  
4413 http://www.apache.org/licenses/LICENSE-2.0
4414  
4415 Unless required by applicable law or agreed to in writing, software
4416 distributed under the License is distributed on an "AS IS" BASIS,
4417 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4418 See the License for the specific language governing permissions and
4419 limitations under the License.
4420  
4421 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4422 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4423 */
4424  
4425 /*
4426 Several utilities, including: #defines to try different compression results,
4427 basic deflate specification values and generic program options.
4428 */
4429  
4430 #ifndef ZOPFLI_UTIL_H_
4431 #define ZOPFLI_UTIL_H_
4432  
4433 #include <string.h>
4434 #include <stdlib.h>
4435  
4436 /* Minimum and maximum length that can be encoded in deflate. */
4437 #define ZOPFLI_MAX_MATCH 258
4438 #define ZOPFLI_MIN_MATCH 3
4439  
4440 /* Number of distinct literal/length and distance symbols in DEFLATE */
4441 #define ZOPFLI_NUM_LL 288
4442 #define ZOPFLI_NUM_D 32
4443  
4444 /*
4445 The window size for deflate. Must be a power of two. This should be 32768, the
4446 maximum possible by the deflate spec. Anything less hurts compression more than
4447 speed.
4448 */
4449 #define ZOPFLI_WINDOW_SIZE 32768
4450  
4451 /*
4452 The window mask used to wrap indices into the window. This is why the
4453 window size must be a power of two.
4454 */
4455 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
4456  
4457 /*
4458 A block structure of huge, non-smart, blocks to divide the input into, to allow
4459 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
4460 The whole compression algorithm, including the smarter block splitting, will
4461 be executed independently on each huge block.
4462 Dividing into huge blocks hurts compression, but not much relative to the size.
4463 Set it to 0 to disable master blocks.
4464 */
4465 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
4466  
4467 /*
4468 Used to initialize costs for example
4469 */
4470 #define ZOPFLI_LARGE_FLOAT 1e30
4471  
4472 /*
4473 For longest match cache. max 256. Uses huge amounts of memory but makes it
4474 faster. Uses this many times three bytes per single byte of the input data.
4475 This is so because longest match finding has to find the exact distance
4476 that belongs to each length for the best lz77 strategy.
4477 Good values: e.g. 5, 8.
4478 */
4479 #define ZOPFLI_CACHE_LENGTH 8
4480  
4481 /*
4482 limit the max hash chain hits for this hash value. This has an effect only
4483 on files where the hash value is the same very often. On these files, this
4484 gives worse compression (the value should ideally be 32768, which is the
4485 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
4486 faster on some specific files.
4487 Good value: e.g. 8192.
4488 */
4489 #define ZOPFLI_MAX_CHAIN_HITS 8192
4490  
4491 /*
4492 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
4493 consumes a lot of memory but speeds it up. No effect on compression size.
4494 */
4495 #define ZOPFLI_LONGEST_MATCH_CACHE
4496  
4497 /*
4498 Enable to remember amount of successive identical bytes in the hash chain for
4499 finding longest match
4500 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
4501 This has no effect on the compression result, and enabling it increases speed.
4502 */
4503 #define ZOPFLI_HASH_SAME
4504  
4505 /*
4506 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
4507 best length so far is long enough. This is way faster for files with lots of
4508 identical bytes, on which the compressor is otherwise too slow. Regular files
4509 are unaffected or maybe a tiny bit slower.
4510 This has no effect on the compression result, only on speed.
4511 */
4512 #define ZOPFLI_HASH_SAME_HASH
4513  
4514 /*
4515 Enable this, to avoid slowness for files which are a repetition of the same
4516 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
4517 the compression result.
4518 */
4519 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
4520  
4521 /*
4522 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
4523 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
4524 varies from file to file.
4525 */
4526 #define ZOPFLI_LAZY_MATCHING
4527  
4528 /*
4529 Appends value to dynamically allocated memory, doubling its allocation size
4530 whenever needed.
4531  
4532 value: the value to append, type T
4533 data: pointer to the dynamic array to append to, type T**
4534 size: pointer to the size of the array to append to, type size_t*. This is the
4535 size that you consider the array to be, not the internal allocation size.
4536 Precondition: allocated size of data is at least a power of two greater than or
4537 equal than *size.
4538 */
4539 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
4540 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
4541 if (!((*size) & ((*size) - 1))) {\
4542 /*double alloc size if it's a power of two*/\
4543 void** data_void = reinterpret_cast<void**>(data);\
4544 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
4545 : realloc((*data), (*size) * 2 * sizeof(**data));\
4546 }\
4547 (*data)[(*size)] = (value);\
4548 (*size)++;\
4549 }
4550 #else /* C gives problems with strict-aliasing rules for (void**) cast */
4551 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
4552 if (!((*size) & ((*size) - 1))) {\
4553 /*double alloc size if it's a power of two*/\
4554 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
4555 : realloc((*data), (*size) * 2 * sizeof(**data));\
4556 }\
4557 (*data)[(*size)] = (value);\
4558 (*size)++;\
4559 }
4560 #endif
4561  
4562  
4563 #endif /* ZOPFLI_UTIL_H_ */
4564  
4565  
4566 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
4567  
4568 /*
4569 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
4570 values.
4571 This is needed because the squeeze runs will ask these values multiple times for
4572 the same position.
4573 Uses large amounts of memory, since it has to remember the distance belonging
4574 to every possible shorter-than-the-best length (the so called "sublen" array).
4575 */
4576 typedef struct ZopfliLongestMatchCache {
4577 unsigned short* length;
4578 unsigned short* dist;
4579 unsigned char* sublen;
4580 } ZopfliLongestMatchCache;
4581  
4582 /* Initializes the ZopfliLongestMatchCache. */
4583 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
4584  
4585 /* Frees up the memory of the ZopfliLongestMatchCache. */
4586 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
4587  
4588 /* Stores sublen array in the cache. */
4589 void ZopfliSublenToCache(const unsigned short* sublen,
4590 size_t pos, size_t length,
4591 ZopfliLongestMatchCache* lmc);
4592  
4593 /* Extracts sublen array from the cache. */
4594 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
4595 size_t pos, size_t length,
4596 unsigned short* sublen);
4597 /* Returns the length up to which could be stored in the cache. */
4598 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
4599 size_t pos, size_t length);
4600  
4601 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
4602  
4603 #endif /* ZOPFLI_CACHE_H_ */
4604  
4605 /* hash.h */
4606 /*
4607 Copyright 2011 Google Inc. All Rights Reserved.
4608  
4609 Licensed under the Apache License, Version 2.0 (the "License");
4610 you may not use this file except in compliance with the License.
4611 You may obtain a copy of the License at
4612  
4613 http://www.apache.org/licenses/LICENSE-2.0
4614  
4615 Unless required by applicable law or agreed to in writing, software
4616 distributed under the License is distributed on an "AS IS" BASIS,
4617 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4618 See the License for the specific language governing permissions and
4619 limitations under the License.
4620  
4621 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4622 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4623 */
4624  
4625 /*
4626 The hash for ZopfliFindLongestMatch of lz77.c.
4627 */
4628  
4629 #ifndef ZOPFLI_HASH_H_
4630 #define ZOPFLI_HASH_H_
4631  
4632 /* util.h */
4633 /*
4634 Copyright 2011 Google Inc. All Rights Reserved.
4635  
4636 Licensed under the Apache License, Version 2.0 (the "License");
4637 you may not use this file except in compliance with the License.
4638 You may obtain a copy of the License at
4639  
4640 http://www.apache.org/licenses/LICENSE-2.0
4641  
4642 Unless required by applicable law or agreed to in writing, software
4643 distributed under the License is distributed on an "AS IS" BASIS,
4644 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4645 See the License for the specific language governing permissions and
4646 limitations under the License.
4647  
4648 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4649 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4650 */
4651  
4652 /*
4653 Several utilities, including: #defines to try different compression results,
4654 basic deflate specification values and generic program options.
4655 */
4656  
4657 #ifndef ZOPFLI_UTIL_H_
4658 #define ZOPFLI_UTIL_H_
4659  
4660 #include <string.h>
4661 #include <stdlib.h>
4662  
4663 /* Minimum and maximum length that can be encoded in deflate. */
4664 #define ZOPFLI_MAX_MATCH 258
4665 #define ZOPFLI_MIN_MATCH 3
4666  
4667 /* Number of distinct literal/length and distance symbols in DEFLATE */
4668 #define ZOPFLI_NUM_LL 288
4669 #define ZOPFLI_NUM_D 32
4670  
4671 /*
4672 The window size for deflate. Must be a power of two. This should be 32768, the
4673 maximum possible by the deflate spec. Anything less hurts compression more than
4674 speed.
4675 */
4676 #define ZOPFLI_WINDOW_SIZE 32768
4677  
4678 /*
4679 The window mask used to wrap indices into the window. This is why the
4680 window size must be a power of two.
4681 */
4682 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
4683  
4684 /*
4685 A block structure of huge, non-smart, blocks to divide the input into, to allow
4686 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
4687 The whole compression algorithm, including the smarter block splitting, will
4688 be executed independently on each huge block.
4689 Dividing into huge blocks hurts compression, but not much relative to the size.
4690 Set it to 0 to disable master blocks.
4691 */
4692 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
4693  
4694 /*
4695 Used to initialize costs for example
4696 */
4697 #define ZOPFLI_LARGE_FLOAT 1e30
4698  
4699 /*
4700 For longest match cache. max 256. Uses huge amounts of memory but makes it
4701 faster. Uses this many times three bytes per single byte of the input data.
4702 This is so because longest match finding has to find the exact distance
4703 that belongs to each length for the best lz77 strategy.
4704 Good values: e.g. 5, 8.
4705 */
4706 #define ZOPFLI_CACHE_LENGTH 8
4707  
4708 /*
4709 limit the max hash chain hits for this hash value. This has an effect only
4710 on files where the hash value is the same very often. On these files, this
4711 gives worse compression (the value should ideally be 32768, which is the
4712 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
4713 faster on some specific files.
4714 Good value: e.g. 8192.
4715 */
4716 #define ZOPFLI_MAX_CHAIN_HITS 8192
4717  
4718 /*
4719 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
4720 consumes a lot of memory but speeds it up. No effect on compression size.
4721 */
4722 #define ZOPFLI_LONGEST_MATCH_CACHE
4723  
4724 /*
4725 Enable to remember amount of successive identical bytes in the hash chain for
4726 finding longest match
4727 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
4728 This has no effect on the compression result, and enabling it increases speed.
4729 */
4730 #define ZOPFLI_HASH_SAME
4731  
4732 /*
4733 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
4734 best length so far is long enough. This is way faster for files with lots of
4735 identical bytes, on which the compressor is otherwise too slow. Regular files
4736 are unaffected or maybe a tiny bit slower.
4737 This has no effect on the compression result, only on speed.
4738 */
4739 #define ZOPFLI_HASH_SAME_HASH
4740  
4741 /*
4742 Enable this, to avoid slowness for files which are a repetition of the same
4743 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
4744 the compression result.
4745 */
4746 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
4747  
4748 /*
4749 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
4750 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
4751 varies from file to file.
4752 */
4753 #define ZOPFLI_LAZY_MATCHING
4754  
4755 /*
4756 Appends value to dynamically allocated memory, doubling its allocation size
4757 whenever needed.
4758  
4759 value: the value to append, type T
4760 data: pointer to the dynamic array to append to, type T**
4761 size: pointer to the size of the array to append to, type size_t*. This is the
4762 size that you consider the array to be, not the internal allocation size.
4763 Precondition: allocated size of data is at least a power of two greater than or
4764 equal than *size.
4765 */
4766 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
4767 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
4768 if (!((*size) & ((*size) - 1))) {\
4769 /*double alloc size if it's a power of two*/\
4770 void** data_void = reinterpret_cast<void**>(data);\
4771 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
4772 : realloc((*data), (*size) * 2 * sizeof(**data));\
4773 }\
4774 (*data)[(*size)] = (value);\
4775 (*size)++;\
4776 }
4777 #else /* C gives problems with strict-aliasing rules for (void**) cast */
4778 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
4779 if (!((*size) & ((*size) - 1))) {\
4780 /*double alloc size if it's a power of two*/\
4781 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
4782 : realloc((*data), (*size) * 2 * sizeof(**data));\
4783 }\
4784 (*data)[(*size)] = (value);\
4785 (*size)++;\
4786 }
4787 #endif
4788  
4789  
4790 #endif /* ZOPFLI_UTIL_H_ */
4791  
4792  
4793 typedef struct ZopfliHash {
4794 int* head; /* Hash value to index of its most recent occurrence. */
4795 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
4796 int* hashval; /* Index to hash value at this index. */
4797 int val; /* Current hash value. */
4798  
4799 #ifdef ZOPFLI_HASH_SAME_HASH
4800 /* Fields with similar purpose as the above hash, but for the second hash with
4801 a value that is calculated differently. */
4802 int* head2; /* Hash value to index of its most recent occurrence. */
4803 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
4804 int* hashval2; /* Index to hash value at this index. */
4805 int val2; /* Current hash value. */
4806 #endif
4807  
4808 #ifdef ZOPFLI_HASH_SAME
4809 unsigned short* same; /* Amount of repetitions of same byte after this .*/
4810 #endif
4811 } ZopfliHash;
4812  
4813 /* Allocates ZopfliHash memory. */
4814 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
4815  
4816 /* Resets all fields of ZopfliHash. */
4817 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
4818  
4819 /* Frees ZopfliHash memory. */
4820 void ZopfliCleanHash(ZopfliHash* h);
4821  
4822 /*
4823 Updates the hash values based on the current position in the array. All calls
4824 to this must be made for consecutive bytes.
4825 */
4826 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
4827 ZopfliHash* h);
4828  
4829 /*
4830 Prepopulates hash:
4831 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
4832 correctly.
4833 */
4834 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
4835 ZopfliHash* h);
4836  
4837 #endif /* ZOPFLI_HASH_H_ */
4838  
4839 /* zopfli.h */
4840 /*
4841 Copyright 2011 Google Inc. All Rights Reserved.
4842  
4843 Licensed under the Apache License, Version 2.0 (the "License");
4844 you may not use this file except in compliance with the License.
4845 You may obtain a copy of the License at
4846  
4847 http://www.apache.org/licenses/LICENSE-2.0
4848  
4849 Unless required by applicable law or agreed to in writing, software
4850 distributed under the License is distributed on an "AS IS" BASIS,
4851 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4852 See the License for the specific language governing permissions and
4853 limitations under the License.
4854  
4855 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
4856 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
4857 */
4858  
4859 #ifndef ZOPFLI_ZOPFLI_H_
4860 #define ZOPFLI_ZOPFLI_H_
4861  
4862 #include <stddef.h>
4863 #include <stdlib.h> /* for size_t */
4864  
4865 #ifdef __cplusplus
4866 extern "C" {
4867 #endif
4868  
4869 /*
4870 Options used throughout the program.
4871 */
4872 typedef struct ZopfliOptions {
4873 /* Whether to print output */
4874 int verbose;
4875  
4876 /* Whether to print more detailed output */
4877 int verbose_more;
4878  
4879 /*
4880 Maximum amount of times to rerun forward and backward pass to optimize LZ77
4881 compression cost. Good values: 10, 15 for small files, 5 for files over
4882 several MB in size or it will be too slow.
4883 */
4884 int numiterations;
4885  
4886 /*
4887 If true, splits the data in multiple deflate blocks with optimal choice
4888 for the block boundaries. Block splitting gives better compression. Default:
4889 true (1).
4890 */
4891 int blocksplitting;
4892  
4893 /*
4894 No longer used, left for compatibility.
4895 */
4896 int blocksplittinglast;
4897  
4898 /*
4899 Maximum amount of blocks to split into (0 for unlimited, but this can give
4900 extreme results that hurt compression on some files). Default value: 15.
4901 */
4902 int blocksplittingmax;
4903 } ZopfliOptions;
4904  
4905 /* Initializes options with default values. */
4906 void ZopfliInitOptions(ZopfliOptions* options);
4907  
4908 /* Output format */
4909 typedef enum {
4910 ZOPFLI_FORMAT_GZIP,
4911 ZOPFLI_FORMAT_ZLIB,
4912 ZOPFLI_FORMAT_DEFLATE
4913 } ZopfliFormat;
4914  
4915 /*
4916 Compresses according to the given output format and appends the result to the
4917 output.
4918  
4919 options: global program options
4920 output_type: the output format to use
4921 out: pointer to the dynamic output array to which the result is appended. Must
4922 be freed after use
4923 outsize: pointer to the dynamic output array size
4924 */
4925 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
4926 const unsigned char* in, size_t insize,
4927 unsigned char** out, size_t* outsize);
4928  
4929 #ifdef __cplusplus
4930 } // extern "C"
4931 #endif
4932  
4933 #endif /* ZOPFLI_ZOPFLI_H_ */
4934  
4935  
4936 /*
4937 Stores lit/length and dist pairs for LZ77.
4938 Parameter litlens: Contains the literal symbols or length values.
4939 Parameter dists: Contains the distances. A value is 0 to indicate that there is
4940 no dist and the corresponding litlens value is a literal instead of a length.
4941 Parameter size: The size of both the litlens and dists arrays.
4942 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
4943 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
4944  
4945 */
4946 typedef struct ZopfliLZ77Store {
4947 unsigned short* litlens; /* Lit or len. */
4948 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
4949 if > 0: length in corresponding litlens, this is the distance. */
4950 size_t size;
4951  
4952 const unsigned char* data; /* original data */
4953 size_t* pos; /* position in data where this LZ77 command begins */
4954  
4955 unsigned short* ll_symbol;
4956 unsigned short* d_symbol;
4957  
4958 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
4959 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
4960 precise histogram at every N symbols, and the rest can be calculated by
4961 looping through the actual symbols of this chunk. */
4962 size_t* ll_counts;
4963 size_t* d_counts;
4964 } ZopfliLZ77Store;
4965  
4966 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
4967 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
4968 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
4969 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
4970 size_t pos, ZopfliLZ77Store* store);
4971 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
4972 ZopfliLZ77Store* target);
4973 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
4974 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
4975 size_t lstart, size_t lend);
4976 /* Gets the histogram of lit/len and dist symbols in the given range, using the
4977 cumulative histograms, so faster than adding one by one for large range. Does
4978 not add the one end symbol of value 256. */
4979 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
4980 size_t lstart, size_t lend,
4981 size_t* ll_counts, size_t* d_counts);
4982  
4983 /*
4984 Some state information for compressing a block.
4985 This is currently a bit under-used (with mainly only the longest match cache),
4986 but is kept for easy future expansion.
4987 */
4988 typedef struct ZopfliBlockState {
4989 const ZopfliOptions* options;
4990  
4991 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
4992 /* Cache for length/distance pairs found so far. */
4993 ZopfliLongestMatchCache* lmc;
4994 #endif
4995  
4996 /* The start (inclusive) and end (not inclusive) of the current block. */
4997 size_t blockstart;
4998 size_t blockend;
4999 } ZopfliBlockState;
5000  
5001 void ZopfliInitBlockState(const ZopfliOptions* options,
5002 size_t blockstart, size_t blockend, int add_lmc,
5003 ZopfliBlockState* s);
5004 void ZopfliCleanBlockState(ZopfliBlockState* s);
5005  
5006 /*
5007 Finds the longest match (length and corresponding distance) for LZ77
5008 compression.
5009 Even when not using "sublen", it can be more efficient to provide an array,
5010 because only then the caching is used.
5011 array: the data
5012 pos: position in the data to find the match for
5013 size: size of the data
5014 limit: limit length to maximum this value (default should be 258). This allows
5015 finding a shorter dist for that length (= less extra bits). Must be
5016 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
5017 sublen: output array of 259 elements, or null. Has, for each length, the
5018 smallest distance required to reach this length. Only 256 of its 259 values
5019 are used, the first 3 are ignored (the shortest length is 3. It is purely
5020 for convenience that the array is made 3 longer).
5021 */
5022 void ZopfliFindLongestMatch(
5023 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
5024 size_t pos, size_t size, size_t limit,
5025 unsigned short* sublen, unsigned short* distance, unsigned short* length);
5026  
5027 /*
5028 Verifies if length and dist are indeed valid, only used for assertion.
5029 */
5030 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
5031 unsigned short dist, unsigned short length);
5032  
5033 /*
5034 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
5035 with the slow but better "squeeze" implementation.
5036 The result is placed in the ZopfliLZ77Store.
5037 If instart is larger than 0, it uses values before instart as starting
5038 dictionary.
5039 */
5040 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
5041 size_t instart, size_t inend,
5042 ZopfliLZ77Store* store, ZopfliHash* h);
5043  
5044 #endif /* ZOPFLI_LZ77_H_ */
5045  
5046 /* zopfli.h */
5047 /*
5048 Copyright 2011 Google Inc. All Rights Reserved.
5049  
5050 Licensed under the Apache License, Version 2.0 (the "License");
5051 you may not use this file except in compliance with the License.
5052 You may obtain a copy of the License at
5053  
5054 http://www.apache.org/licenses/LICENSE-2.0
5055  
5056 Unless required by applicable law or agreed to in writing, software
5057 distributed under the License is distributed on an "AS IS" BASIS,
5058 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5059 See the License for the specific language governing permissions and
5060 limitations under the License.
5061  
5062 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5063 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5064 */
5065  
5066 #ifndef ZOPFLI_ZOPFLI_H_
5067 #define ZOPFLI_ZOPFLI_H_
5068  
5069 #include <stddef.h>
5070 #include <stdlib.h> /* for size_t */
5071  
5072 #ifdef __cplusplus
5073 extern "C" {
5074 #endif
5075  
5076 /*
5077 Options used throughout the program.
5078 */
5079 typedef struct ZopfliOptions {
5080 /* Whether to print output */
5081 int verbose;
5082  
5083 /* Whether to print more detailed output */
5084 int verbose_more;
5085  
5086 /*
5087 Maximum amount of times to rerun forward and backward pass to optimize LZ77
5088 compression cost. Good values: 10, 15 for small files, 5 for files over
5089 several MB in size or it will be too slow.
5090 */
5091 int numiterations;
5092  
5093 /*
5094 If true, splits the data in multiple deflate blocks with optimal choice
5095 for the block boundaries. Block splitting gives better compression. Default:
5096 true (1).
5097 */
5098 int blocksplitting;
5099  
5100 /*
5101 No longer used, left for compatibility.
5102 */
5103 int blocksplittinglast;
5104  
5105 /*
5106 Maximum amount of blocks to split into (0 for unlimited, but this can give
5107 extreme results that hurt compression on some files). Default value: 15.
5108 */
5109 int blocksplittingmax;
5110 } ZopfliOptions;
5111  
5112 /* Initializes options with default values. */
5113 void ZopfliInitOptions(ZopfliOptions* options);
5114  
5115 /* Output format */
5116 typedef enum {
5117 ZOPFLI_FORMAT_GZIP,
5118 ZOPFLI_FORMAT_ZLIB,
5119 ZOPFLI_FORMAT_DEFLATE
5120 } ZopfliFormat;
5121  
5122 /*
5123 Compresses according to the given output format and appends the result to the
5124 output.
5125  
5126 options: global program options
5127 output_type: the output format to use
5128 out: pointer to the dynamic output array to which the result is appended. Must
5129 be freed after use
5130 outsize: pointer to the dynamic output array size
5131 */
5132 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
5133 const unsigned char* in, size_t insize,
5134 unsigned char** out, size_t* outsize);
5135  
5136 #ifdef __cplusplus
5137 } // extern "C"
5138 #endif
5139  
5140 #endif /* ZOPFLI_ZOPFLI_H_ */
5141  
5142  
5143  
5144 /*
5145 Does blocksplitting on LZ77 data.
5146 The output splitpoints are indices in the LZ77 data.
5147 maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit.
5148 */
5149 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
5150 const ZopfliLZ77Store* lz77, size_t maxblocks,
5151 size_t** splitpoints, size_t* npoints);
5152  
5153 /*
5154 Does blocksplitting on uncompressed data.
5155 The output splitpoints are indices in the uncompressed bytes.
5156  
5157 options: general program options.
5158 in: uncompressed input data
5159 instart: where to start splitting
5160 inend: where to end splitting (not inclusive)
5161 maxblocks: maximum amount of blocks to split into, or 0 for no limit
5162 splitpoints: dynamic array to put the resulting split point coordinates into.
5163 The coordinates are indices in the input array.
5164 npoints: pointer to amount of splitpoints, for the dynamic array. The amount of
5165 blocks is the amount of splitpoitns + 1.
5166 */
5167 void ZopfliBlockSplit(const ZopfliOptions* options,
5168 const unsigned char* in, size_t instart, size_t inend,
5169 size_t maxblocks, size_t** splitpoints, size_t* npoints);
5170  
5171 /*
5172 Divides the input into equal blocks, does not even take LZ77 lengths into
5173 account.
5174 */
5175 void ZopfliBlockSplitSimple(const unsigned char* in,
5176 size_t instart, size_t inend,
5177 size_t blocksize,
5178 size_t** splitpoints, size_t* npoints);
5179  
5180 #endif /* ZOPFLI_BLOCKSPLITTER_H_ */
5181  
5182 /* squeeze.h */
5183 /*
5184 Copyright 2011 Google Inc. All Rights Reserved.
5185  
5186 Licensed under the Apache License, Version 2.0 (the "License");
5187 you may not use this file except in compliance with the License.
5188 You may obtain a copy of the License at
5189  
5190 http://www.apache.org/licenses/LICENSE-2.0
5191  
5192 Unless required by applicable law or agreed to in writing, software
5193 distributed under the License is distributed on an "AS IS" BASIS,
5194 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5195 See the License for the specific language governing permissions and
5196 limitations under the License.
5197  
5198 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5199 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5200 */
5201  
5202 /*
5203 The squeeze functions do enhanced LZ77 compression by optimal parsing with a
5204 cost model, rather than greedily choosing the longest length or using a single
5205 step of lazy matching like regular implementations.
5206  
5207 Since the cost model is based on the Huffman tree that can only be calculated
5208 after the LZ77 data is generated, there is a chicken and egg problem, and
5209 multiple runs are done with updated cost models to converge to a better
5210 solution.
5211 */
5212  
5213 #ifndef ZOPFLI_SQUEEZE_H_
5214 #define ZOPFLI_SQUEEZE_H_
5215  
5216 /* lz77.h */
5217 /*
5218 Copyright 2011 Google Inc. All Rights Reserved.
5219  
5220 Licensed under the Apache License, Version 2.0 (the "License");
5221 you may not use this file except in compliance with the License.
5222 You may obtain a copy of the License at
5223  
5224 http://www.apache.org/licenses/LICENSE-2.0
5225  
5226 Unless required by applicable law or agreed to in writing, software
5227 distributed under the License is distributed on an "AS IS" BASIS,
5228 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5229 See the License for the specific language governing permissions and
5230 limitations under the License.
5231  
5232 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5233 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5234 */
5235  
5236 /*
5237 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
5238 compression.
5239 */
5240  
5241 #ifndef ZOPFLI_LZ77_H_
5242 #define ZOPFLI_LZ77_H_
5243  
5244 #include <stdlib.h>
5245  
5246 /* cache.h */
5247 /*
5248 Copyright 2011 Google Inc. All Rights Reserved.
5249  
5250 Licensed under the Apache License, Version 2.0 (the "License");
5251 you may not use this file except in compliance with the License.
5252 You may obtain a copy of the License at
5253  
5254 http://www.apache.org/licenses/LICENSE-2.0
5255  
5256 Unless required by applicable law or agreed to in writing, software
5257 distributed under the License is distributed on an "AS IS" BASIS,
5258 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5259 See the License for the specific language governing permissions and
5260 limitations under the License.
5261  
5262 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5263 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5264 */
5265  
5266 /*
5267 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
5268 */
5269  
5270 #ifndef ZOPFLI_CACHE_H_
5271 #define ZOPFLI_CACHE_H_
5272  
5273 /* util.h */
5274 /*
5275 Copyright 2011 Google Inc. All Rights Reserved.
5276  
5277 Licensed under the Apache License, Version 2.0 (the "License");
5278 you may not use this file except in compliance with the License.
5279 You may obtain a copy of the License at
5280  
5281 http://www.apache.org/licenses/LICENSE-2.0
5282  
5283 Unless required by applicable law or agreed to in writing, software
5284 distributed under the License is distributed on an "AS IS" BASIS,
5285 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5286 See the License for the specific language governing permissions and
5287 limitations under the License.
5288  
5289 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5290 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5291 */
5292  
5293 /*
5294 Several utilities, including: #defines to try different compression results,
5295 basic deflate specification values and generic program options.
5296 */
5297  
5298 #ifndef ZOPFLI_UTIL_H_
5299 #define ZOPFLI_UTIL_H_
5300  
5301 #include <string.h>
5302 #include <stdlib.h>
5303  
5304 /* Minimum and maximum length that can be encoded in deflate. */
5305 #define ZOPFLI_MAX_MATCH 258
5306 #define ZOPFLI_MIN_MATCH 3
5307  
5308 /* Number of distinct literal/length and distance symbols in DEFLATE */
5309 #define ZOPFLI_NUM_LL 288
5310 #define ZOPFLI_NUM_D 32
5311  
5312 /*
5313 The window size for deflate. Must be a power of two. This should be 32768, the
5314 maximum possible by the deflate spec. Anything less hurts compression more than
5315 speed.
5316 */
5317 #define ZOPFLI_WINDOW_SIZE 32768
5318  
5319 /*
5320 The window mask used to wrap indices into the window. This is why the
5321 window size must be a power of two.
5322 */
5323 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
5324  
5325 /*
5326 A block structure of huge, non-smart, blocks to divide the input into, to allow
5327 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
5328 The whole compression algorithm, including the smarter block splitting, will
5329 be executed independently on each huge block.
5330 Dividing into huge blocks hurts compression, but not much relative to the size.
5331 Set it to 0 to disable master blocks.
5332 */
5333 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
5334  
5335 /*
5336 Used to initialize costs for example
5337 */
5338 #define ZOPFLI_LARGE_FLOAT 1e30
5339  
5340 /*
5341 For longest match cache. max 256. Uses huge amounts of memory but makes it
5342 faster. Uses this many times three bytes per single byte of the input data.
5343 This is so because longest match finding has to find the exact distance
5344 that belongs to each length for the best lz77 strategy.
5345 Good values: e.g. 5, 8.
5346 */
5347 #define ZOPFLI_CACHE_LENGTH 8
5348  
5349 /*
5350 limit the max hash chain hits for this hash value. This has an effect only
5351 on files where the hash value is the same very often. On these files, this
5352 gives worse compression (the value should ideally be 32768, which is the
5353 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
5354 faster on some specific files.
5355 Good value: e.g. 8192.
5356 */
5357 #define ZOPFLI_MAX_CHAIN_HITS 8192
5358  
5359 /*
5360 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
5361 consumes a lot of memory but speeds it up. No effect on compression size.
5362 */
5363 #define ZOPFLI_LONGEST_MATCH_CACHE
5364  
5365 /*
5366 Enable to remember amount of successive identical bytes in the hash chain for
5367 finding longest match
5368 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
5369 This has no effect on the compression result, and enabling it increases speed.
5370 */
5371 #define ZOPFLI_HASH_SAME
5372  
5373 /*
5374 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
5375 best length so far is long enough. This is way faster for files with lots of
5376 identical bytes, on which the compressor is otherwise too slow. Regular files
5377 are unaffected or maybe a tiny bit slower.
5378 This has no effect on the compression result, only on speed.
5379 */
5380 #define ZOPFLI_HASH_SAME_HASH
5381  
5382 /*
5383 Enable this, to avoid slowness for files which are a repetition of the same
5384 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
5385 the compression result.
5386 */
5387 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
5388  
5389 /*
5390 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
5391 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
5392 varies from file to file.
5393 */
5394 #define ZOPFLI_LAZY_MATCHING
5395  
5396 /*
5397 Appends value to dynamically allocated memory, doubling its allocation size
5398 whenever needed.
5399  
5400 value: the value to append, type T
5401 data: pointer to the dynamic array to append to, type T**
5402 size: pointer to the size of the array to append to, type size_t*. This is the
5403 size that you consider the array to be, not the internal allocation size.
5404 Precondition: allocated size of data is at least a power of two greater than or
5405 equal than *size.
5406 */
5407 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
5408 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
5409 if (!((*size) & ((*size) - 1))) {\
5410 /*double alloc size if it's a power of two*/\
5411 void** data_void = reinterpret_cast<void**>(data);\
5412 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
5413 : realloc((*data), (*size) * 2 * sizeof(**data));\
5414 }\
5415 (*data)[(*size)] = (value);\
5416 (*size)++;\
5417 }
5418 #else /* C gives problems with strict-aliasing rules for (void**) cast */
5419 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
5420 if (!((*size) & ((*size) - 1))) {\
5421 /*double alloc size if it's a power of two*/\
5422 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
5423 : realloc((*data), (*size) * 2 * sizeof(**data));\
5424 }\
5425 (*data)[(*size)] = (value);\
5426 (*size)++;\
5427 }
5428 #endif
5429  
5430  
5431 #endif /* ZOPFLI_UTIL_H_ */
5432  
5433  
5434 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
5435  
5436 /*
5437 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
5438 values.
5439 This is needed because the squeeze runs will ask these values multiple times for
5440 the same position.
5441 Uses large amounts of memory, since it has to remember the distance belonging
5442 to every possible shorter-than-the-best length (the so called "sublen" array).
5443 */
5444 typedef struct ZopfliLongestMatchCache {
5445 unsigned short* length;
5446 unsigned short* dist;
5447 unsigned char* sublen;
5448 } ZopfliLongestMatchCache;
5449  
5450 /* Initializes the ZopfliLongestMatchCache. */
5451 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
5452  
5453 /* Frees up the memory of the ZopfliLongestMatchCache. */
5454 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
5455  
5456 /* Stores sublen array in the cache. */
5457 void ZopfliSublenToCache(const unsigned short* sublen,
5458 size_t pos, size_t length,
5459 ZopfliLongestMatchCache* lmc);
5460  
5461 /* Extracts sublen array from the cache. */
5462 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
5463 size_t pos, size_t length,
5464 unsigned short* sublen);
5465 /* Returns the length up to which could be stored in the cache. */
5466 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
5467 size_t pos, size_t length);
5468  
5469 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
5470  
5471 #endif /* ZOPFLI_CACHE_H_ */
5472  
5473 /* hash.h */
5474 /*
5475 Copyright 2011 Google Inc. All Rights Reserved.
5476  
5477 Licensed under the Apache License, Version 2.0 (the "License");
5478 you may not use this file except in compliance with the License.
5479 You may obtain a copy of the License at
5480  
5481 http://www.apache.org/licenses/LICENSE-2.0
5482  
5483 Unless required by applicable law or agreed to in writing, software
5484 distributed under the License is distributed on an "AS IS" BASIS,
5485 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5486 See the License for the specific language governing permissions and
5487 limitations under the License.
5488  
5489 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5490 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5491 */
5492  
5493 /*
5494 The hash for ZopfliFindLongestMatch of lz77.c.
5495 */
5496  
5497 #ifndef ZOPFLI_HASH_H_
5498 #define ZOPFLI_HASH_H_
5499  
5500 /* util.h */
5501 /*
5502 Copyright 2011 Google Inc. All Rights Reserved.
5503  
5504 Licensed under the Apache License, Version 2.0 (the "License");
5505 you may not use this file except in compliance with the License.
5506 You may obtain a copy of the License at
5507  
5508 http://www.apache.org/licenses/LICENSE-2.0
5509  
5510 Unless required by applicable law or agreed to in writing, software
5511 distributed under the License is distributed on an "AS IS" BASIS,
5512 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5513 See the License for the specific language governing permissions and
5514 limitations under the License.
5515  
5516 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5517 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5518 */
5519  
5520 /*
5521 Several utilities, including: #defines to try different compression results,
5522 basic deflate specification values and generic program options.
5523 */
5524  
5525 #ifndef ZOPFLI_UTIL_H_
5526 #define ZOPFLI_UTIL_H_
5527  
5528 #include <string.h>
5529 #include <stdlib.h>
5530  
5531 /* Minimum and maximum length that can be encoded in deflate. */
5532 #define ZOPFLI_MAX_MATCH 258
5533 #define ZOPFLI_MIN_MATCH 3
5534  
5535 /* Number of distinct literal/length and distance symbols in DEFLATE */
5536 #define ZOPFLI_NUM_LL 288
5537 #define ZOPFLI_NUM_D 32
5538  
5539 /*
5540 The window size for deflate. Must be a power of two. This should be 32768, the
5541 maximum possible by the deflate spec. Anything less hurts compression more than
5542 speed.
5543 */
5544 #define ZOPFLI_WINDOW_SIZE 32768
5545  
5546 /*
5547 The window mask used to wrap indices into the window. This is why the
5548 window size must be a power of two.
5549 */
5550 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
5551  
5552 /*
5553 A block structure of huge, non-smart, blocks to divide the input into, to allow
5554 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
5555 The whole compression algorithm, including the smarter block splitting, will
5556 be executed independently on each huge block.
5557 Dividing into huge blocks hurts compression, but not much relative to the size.
5558 Set it to 0 to disable master blocks.
5559 */
5560 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
5561  
5562 /*
5563 Used to initialize costs for example
5564 */
5565 #define ZOPFLI_LARGE_FLOAT 1e30
5566  
5567 /*
5568 For longest match cache. max 256. Uses huge amounts of memory but makes it
5569 faster. Uses this many times three bytes per single byte of the input data.
5570 This is so because longest match finding has to find the exact distance
5571 that belongs to each length for the best lz77 strategy.
5572 Good values: e.g. 5, 8.
5573 */
5574 #define ZOPFLI_CACHE_LENGTH 8
5575  
5576 /*
5577 limit the max hash chain hits for this hash value. This has an effect only
5578 on files where the hash value is the same very often. On these files, this
5579 gives worse compression (the value should ideally be 32768, which is the
5580 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
5581 faster on some specific files.
5582 Good value: e.g. 8192.
5583 */
5584 #define ZOPFLI_MAX_CHAIN_HITS 8192
5585  
5586 /*
5587 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
5588 consumes a lot of memory but speeds it up. No effect on compression size.
5589 */
5590 #define ZOPFLI_LONGEST_MATCH_CACHE
5591  
5592 /*
5593 Enable to remember amount of successive identical bytes in the hash chain for
5594 finding longest match
5595 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
5596 This has no effect on the compression result, and enabling it increases speed.
5597 */
5598 #define ZOPFLI_HASH_SAME
5599  
5600 /*
5601 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
5602 best length so far is long enough. This is way faster for files with lots of
5603 identical bytes, on which the compressor is otherwise too slow. Regular files
5604 are unaffected or maybe a tiny bit slower.
5605 This has no effect on the compression result, only on speed.
5606 */
5607 #define ZOPFLI_HASH_SAME_HASH
5608  
5609 /*
5610 Enable this, to avoid slowness for files which are a repetition of the same
5611 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
5612 the compression result.
5613 */
5614 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
5615  
5616 /*
5617 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
5618 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
5619 varies from file to file.
5620 */
5621 #define ZOPFLI_LAZY_MATCHING
5622  
5623 /*
5624 Appends value to dynamically allocated memory, doubling its allocation size
5625 whenever needed.
5626  
5627 value: the value to append, type T
5628 data: pointer to the dynamic array to append to, type T**
5629 size: pointer to the size of the array to append to, type size_t*. This is the
5630 size that you consider the array to be, not the internal allocation size.
5631 Precondition: allocated size of data is at least a power of two greater than or
5632 equal than *size.
5633 */
5634 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
5635 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
5636 if (!((*size) & ((*size) - 1))) {\
5637 /*double alloc size if it's a power of two*/\
5638 void** data_void = reinterpret_cast<void**>(data);\
5639 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
5640 : realloc((*data), (*size) * 2 * sizeof(**data));\
5641 }\
5642 (*data)[(*size)] = (value);\
5643 (*size)++;\
5644 }
5645 #else /* C gives problems with strict-aliasing rules for (void**) cast */
5646 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
5647 if (!((*size) & ((*size) - 1))) {\
5648 /*double alloc size if it's a power of two*/\
5649 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
5650 : realloc((*data), (*size) * 2 * sizeof(**data));\
5651 }\
5652 (*data)[(*size)] = (value);\
5653 (*size)++;\
5654 }
5655 #endif
5656  
5657  
5658 #endif /* ZOPFLI_UTIL_H_ */
5659  
5660  
5661 typedef struct ZopfliHash {
5662 int* head; /* Hash value to index of its most recent occurrence. */
5663 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
5664 int* hashval; /* Index to hash value at this index. */
5665 int val; /* Current hash value. */
5666  
5667 #ifdef ZOPFLI_HASH_SAME_HASH
5668 /* Fields with similar purpose as the above hash, but for the second hash with
5669 a value that is calculated differently. */
5670 int* head2; /* Hash value to index of its most recent occurrence. */
5671 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
5672 int* hashval2; /* Index to hash value at this index. */
5673 int val2; /* Current hash value. */
5674 #endif
5675  
5676 #ifdef ZOPFLI_HASH_SAME
5677 unsigned short* same; /* Amount of repetitions of same byte after this .*/
5678 #endif
5679 } ZopfliHash;
5680  
5681 /* Allocates ZopfliHash memory. */
5682 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
5683  
5684 /* Resets all fields of ZopfliHash. */
5685 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
5686  
5687 /* Frees ZopfliHash memory. */
5688 void ZopfliCleanHash(ZopfliHash* h);
5689  
5690 /*
5691 Updates the hash values based on the current position in the array. All calls
5692 to this must be made for consecutive bytes.
5693 */
5694 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
5695 ZopfliHash* h);
5696  
5697 /*
5698 Prepopulates hash:
5699 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
5700 correctly.
5701 */
5702 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
5703 ZopfliHash* h);
5704  
5705 #endif /* ZOPFLI_HASH_H_ */
5706  
5707 /* zopfli.h */
5708 /*
5709 Copyright 2011 Google Inc. All Rights Reserved.
5710  
5711 Licensed under the Apache License, Version 2.0 (the "License");
5712 you may not use this file except in compliance with the License.
5713 You may obtain a copy of the License at
5714  
5715 http://www.apache.org/licenses/LICENSE-2.0
5716  
5717 Unless required by applicable law or agreed to in writing, software
5718 distributed under the License is distributed on an "AS IS" BASIS,
5719 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5720 See the License for the specific language governing permissions and
5721 limitations under the License.
5722  
5723 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5724 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5725 */
5726  
5727 #ifndef ZOPFLI_ZOPFLI_H_
5728 #define ZOPFLI_ZOPFLI_H_
5729  
5730 #include <stddef.h>
5731 #include <stdlib.h> /* for size_t */
5732  
5733 #ifdef __cplusplus
5734 extern "C" {
5735 #endif
5736  
5737 /*
5738 Options used throughout the program.
5739 */
5740 typedef struct ZopfliOptions {
5741 /* Whether to print output */
5742 int verbose;
5743  
5744 /* Whether to print more detailed output */
5745 int verbose_more;
5746  
5747 /*
5748 Maximum amount of times to rerun forward and backward pass to optimize LZ77
5749 compression cost. Good values: 10, 15 for small files, 5 for files over
5750 several MB in size or it will be too slow.
5751 */
5752 int numiterations;
5753  
5754 /*
5755 If true, splits the data in multiple deflate blocks with optimal choice
5756 for the block boundaries. Block splitting gives better compression. Default:
5757 true (1).
5758 */
5759 int blocksplitting;
5760  
5761 /*
5762 No longer used, left for compatibility.
5763 */
5764 int blocksplittinglast;
5765  
5766 /*
5767 Maximum amount of blocks to split into (0 for unlimited, but this can give
5768 extreme results that hurt compression on some files). Default value: 15.
5769 */
5770 int blocksplittingmax;
5771 } ZopfliOptions;
5772  
5773 /* Initializes options with default values. */
5774 void ZopfliInitOptions(ZopfliOptions* options);
5775  
5776 /* Output format */
5777 typedef enum {
5778 ZOPFLI_FORMAT_GZIP,
5779 ZOPFLI_FORMAT_ZLIB,
5780 ZOPFLI_FORMAT_DEFLATE
5781 } ZopfliFormat;
5782  
5783 /*
5784 Compresses according to the given output format and appends the result to the
5785 output.
5786  
5787 options: global program options
5788 output_type: the output format to use
5789 out: pointer to the dynamic output array to which the result is appended. Must
5790 be freed after use
5791 outsize: pointer to the dynamic output array size
5792 */
5793 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
5794 const unsigned char* in, size_t insize,
5795 unsigned char** out, size_t* outsize);
5796  
5797 #ifdef __cplusplus
5798 } // extern "C"
5799 #endif
5800  
5801 #endif /* ZOPFLI_ZOPFLI_H_ */
5802  
5803  
5804 /*
5805 Stores lit/length and dist pairs for LZ77.
5806 Parameter litlens: Contains the literal symbols or length values.
5807 Parameter dists: Contains the distances. A value is 0 to indicate that there is
5808 no dist and the corresponding litlens value is a literal instead of a length.
5809 Parameter size: The size of both the litlens and dists arrays.
5810 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
5811 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
5812  
5813 */
5814 typedef struct ZopfliLZ77Store {
5815 unsigned short* litlens; /* Lit or len. */
5816 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
5817 if > 0: length in corresponding litlens, this is the distance. */
5818 size_t size;
5819  
5820 const unsigned char* data; /* original data */
5821 size_t* pos; /* position in data where this LZ77 command begins */
5822  
5823 unsigned short* ll_symbol;
5824 unsigned short* d_symbol;
5825  
5826 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
5827 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
5828 precise histogram at every N symbols, and the rest can be calculated by
5829 looping through the actual symbols of this chunk. */
5830 size_t* ll_counts;
5831 size_t* d_counts;
5832 } ZopfliLZ77Store;
5833  
5834 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
5835 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
5836 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
5837 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
5838 size_t pos, ZopfliLZ77Store* store);
5839 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
5840 ZopfliLZ77Store* target);
5841 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
5842 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
5843 size_t lstart, size_t lend);
5844 /* Gets the histogram of lit/len and dist symbols in the given range, using the
5845 cumulative histograms, so faster than adding one by one for large range. Does
5846 not add the one end symbol of value 256. */
5847 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
5848 size_t lstart, size_t lend,
5849 size_t* ll_counts, size_t* d_counts);
5850  
5851 /*
5852 Some state information for compressing a block.
5853 This is currently a bit under-used (with mainly only the longest match cache),
5854 but is kept for easy future expansion.
5855 */
5856 typedef struct ZopfliBlockState {
5857 const ZopfliOptions* options;
5858  
5859 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
5860 /* Cache for length/distance pairs found so far. */
5861 ZopfliLongestMatchCache* lmc;
5862 #endif
5863  
5864 /* The start (inclusive) and end (not inclusive) of the current block. */
5865 size_t blockstart;
5866 size_t blockend;
5867 } ZopfliBlockState;
5868  
5869 void ZopfliInitBlockState(const ZopfliOptions* options,
5870 size_t blockstart, size_t blockend, int add_lmc,
5871 ZopfliBlockState* s);
5872 void ZopfliCleanBlockState(ZopfliBlockState* s);
5873  
5874 /*
5875 Finds the longest match (length and corresponding distance) for LZ77
5876 compression.
5877 Even when not using "sublen", it can be more efficient to provide an array,
5878 because only then the caching is used.
5879 array: the data
5880 pos: position in the data to find the match for
5881 size: size of the data
5882 limit: limit length to maximum this value (default should be 258). This allows
5883 finding a shorter dist for that length (= less extra bits). Must be
5884 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
5885 sublen: output array of 259 elements, or null. Has, for each length, the
5886 smallest distance required to reach this length. Only 256 of its 259 values
5887 are used, the first 3 are ignored (the shortest length is 3. It is purely
5888 for convenience that the array is made 3 longer).
5889 */
5890 void ZopfliFindLongestMatch(
5891 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
5892 size_t pos, size_t size, size_t limit,
5893 unsigned short* sublen, unsigned short* distance, unsigned short* length);
5894  
5895 /*
5896 Verifies if length and dist are indeed valid, only used for assertion.
5897 */
5898 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
5899 unsigned short dist, unsigned short length);
5900  
5901 /*
5902 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
5903 with the slow but better "squeeze" implementation.
5904 The result is placed in the ZopfliLZ77Store.
5905 If instart is larger than 0, it uses values before instart as starting
5906 dictionary.
5907 */
5908 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
5909 size_t instart, size_t inend,
5910 ZopfliLZ77Store* store, ZopfliHash* h);
5911  
5912 #endif /* ZOPFLI_LZ77_H_ */
5913  
5914  
5915 /*
5916 Calculates lit/len and dist pairs for given data.
5917 If instart is larger than 0, it uses values before instart as starting
5918 dictionary.
5919 */
5920 void ZopfliLZ77Optimal(ZopfliBlockState *s,
5921 const unsigned char* in, size_t instart, size_t inend,
5922 int numiterations,
5923 ZopfliLZ77Store* store);
5924  
5925 /*
5926 Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the
5927 deflate standard.
5928 The fixed tree never gives the best compression. But this gives the best
5929 possible LZ77 encoding possible with the fixed tree.
5930 This does not create or output any fixed tree, only LZ77 data optimized for
5931 using with a fixed tree.
5932 If instart is larger than 0, it uses values before instart as starting
5933 dictionary.
5934 */
5935 void ZopfliLZ77OptimalFixed(ZopfliBlockState *s,
5936 const unsigned char* in,
5937 size_t instart, size_t inend,
5938 ZopfliLZ77Store* store);
5939  
5940 #endif /* ZOPFLI_SQUEEZE_H_ */
5941  
5942 /* symbols.h */
5943 /*
5944 Copyright 2016 Google Inc. All Rights Reserved.
5945  
5946 Licensed under the Apache License, Version 2.0 (the "License");
5947 you may not use this file except in compliance with the License.
5948 You may obtain a copy of the License at
5949  
5950 http://www.apache.org/licenses/LICENSE-2.0
5951  
5952 Unless required by applicable law or agreed to in writing, software
5953 distributed under the License is distributed on an "AS IS" BASIS,
5954 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
5955 See the License for the specific language governing permissions and
5956 limitations under the License.
5957  
5958 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
5959 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
5960 */
5961  
5962 /*
5963 Utilities for using the lz77 symbols of the deflate spec.
5964 */
5965  
5966 #ifndef ZOPFLI_SYMBOLS_H_
5967 #define ZOPFLI_SYMBOLS_H_
5968  
5969 /* __has_builtin available in clang */
5970 #ifdef __has_builtin
5971 # if __has_builtin(__builtin_clz)
5972 # define ZOPFLI_HAS_BUILTIN_CLZ
5973 # endif
5974 /* __builtin_clz available beginning with GCC 3.4 */
5975 #elif __GNUC__ * 100 + __GNUC_MINOR__ >= 304
5976 # define ZOPFLI_HAS_BUILTIN_CLZ
5977 #endif
5978  
5979 /* Gets the amount of extra bits for the given dist, cfr. the DEFLATE spec. */
5980 8504032 static int ZopfliGetDistExtraBits(int dist) {
5981 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
5982 8504032 ▶ if (dist < 5) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (dist < 5) return 0; 16692 8487340 8504032
5983 8487340 return (31 ^ __builtin_clz(dist - 1)) - 1; /* log2(dist - 1) - 1 */
5984 #else
5985 if (dist < 5) return 0;
5986 else if (dist < 9) return 1;
5987 else if (dist < 17) return 2;
5988 else if (dist < 33) return 3;
5989 else if (dist < 65) return 4;
5990 else if (dist < 129) return 5;
5991 else if (dist < 257) return 6;
5992 else if (dist < 513) return 7;
5993 else if (dist < 1025) return 8;
5994 else if (dist < 2049) return 9;
5995 else if (dist < 4097) return 10;
5996 else if (dist < 8193) return 11;
5997 else if (dist < 16385) return 12;
5998 else return 13;
5999 #endif
6000 }
6001  
6002 /* Gets value of the extra bits for the given dist, cfr. the DEFLATE spec. */
6003 37 static int ZopfliGetDistExtraBitsValue(int dist) {
6004 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
6005 37 ▶ if (dist < 5) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist < 5) { 0 37 37
6006 0 return 0;
6007 } else {
6008 37 int l = 31 ^ __builtin_clz(dist - 1); /* log2(dist - 1) */
6009 37 return (dist - (1 + (1 << l))) & ((1 << (l - 1)) - 1);
6010 }
6011 #else
6012 if (dist < 5) return 0;
6013 else if (dist < 9) return (dist - 5) & 1;
6014 else if (dist < 17) return (dist - 9) & 3;
6015 else if (dist < 33) return (dist - 17) & 7;
6016 else if (dist < 65) return (dist - 33) & 15;
6017 else if (dist < 129) return (dist - 65) & 31;
6018 else if (dist < 257) return (dist - 129) & 63;
6019 else if (dist < 513) return (dist - 257) & 127;
6020 else if (dist < 1025) return (dist - 513) & 255;
6021 else if (dist < 2049) return (dist - 1025) & 511;
6022 else if (dist < 4097) return (dist - 2049) & 1023;
6023 else if (dist < 8193) return (dist - 4097) & 2047;
6024 else if (dist < 16385) return (dist - 8193) & 4095;
6025 else return (dist - 16385) & 8191;
6026 #endif
6027 }
6028  
6029 /* Gets the symbol for the given dist, cfr. the DEFLATE spec. */
6030 7721071 static int ZopfliGetDistSymbol(int dist) {
6031 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
6032 7721071 ▶ if (dist < 5) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist < 5) { 15648 7705423 7721071
6033 15648 return dist - 1;
6034 } else {
6035 7705423 int l = (31 ^ __builtin_clz(dist - 1)); /* log2(dist - 1) */
6036 7705423 int r = ((dist - 1) >> (l - 1)) & 1;
6037 7705423 return l * 2 + r;
6038 }
6039 #else
6040 if (dist < 193) {
6041 if (dist < 13) { /* dist 0..13. */
6042 if (dist < 5) return dist - 1;
6043 else if (dist < 7) return 4;
6044 else if (dist < 9) return 5;
6045 else return 6;
6046 } else { /* dist 13..193. */
6047 if (dist < 17) return 7;
6048 else if (dist < 25) return 8;
6049 else if (dist < 33) return 9;
6050 else if (dist < 49) return 10;
6051 else if (dist < 65) return 11;
6052 else if (dist < 97) return 12;
6053 else if (dist < 129) return 13;
6054 else return 14;
6055 }
6056 } else {
6057 if (dist < 2049) { /* dist 193..2049. */
6058 if (dist < 257) return 15;
6059 else if (dist < 385) return 16;
6060 else if (dist < 513) return 17;
6061 else if (dist < 769) return 18;
6062 else if (dist < 1025) return 19;
6063 else if (dist < 1537) return 20;
6064 else return 21;
6065 } else { /* dist 2049..32768. */
6066 if (dist < 3073) return 22;
6067 else if (dist < 4097) return 23;
6068 else if (dist < 6145) return 24;
6069 else if (dist < 8193) return 25;
6070 else if (dist < 12289) return 26;
6071 else if (dist < 16385) return 27;
6072 else if (dist < 24577) return 28;
6073 else return 29;
6074 }
6075 }
6076 #endif
6077 }
6078  
6079 /* Gets the amount of extra bits for the given length, cfr. the DEFLATE spec. */
6080 8504032 static int ZopfliGetLengthExtraBits(int l) {
6081 static const int table[259] = {
6082 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
6083 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
6084 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
6085 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
6086 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
6087 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
6088 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
6089 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
6090 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6091 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6092 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6093 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6094 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6095 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6096 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6097 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0
6098 };
6099 8504032 return table[l];
6100 }
6101  
6102 /* Gets value of the extra bits for the given length, cfr. the DEFLATE spec. */
6103 37 static int ZopfliGetLengthExtraBitsValue(int l) {
6104 static const int table[259] = {
6105 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0,
6106 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5,
6107 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6,
6108 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
6109 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2,
6110 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
6111 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
6112 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
6113 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6,
6114 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
6115 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
6116 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0
6117 };
6118 37 return table[l];
6119 }
6120  
6121 /*
6122 Gets the symbol for the given length, cfr. the DEFLATE spec.
6123 Returns the symbol in the range [257-285] (inclusive)
6124 */
6125 8515617 static int ZopfliGetLengthSymbol(int l) {
6126 static const int table[259] = {
6127 0, 0, 0, 257, 258, 259, 260, 261, 262, 263, 264,
6128 265, 265, 266, 266, 267, 267, 268, 268,
6129 269, 269, 269, 269, 270, 270, 270, 270,
6130 271, 271, 271, 271, 272, 272, 272, 272,
6131 273, 273, 273, 273, 273, 273, 273, 273,
6132 274, 274, 274, 274, 274, 274, 274, 274,
6133 275, 275, 275, 275, 275, 275, 275, 275,
6134 276, 276, 276, 276, 276, 276, 276, 276,
6135 277, 277, 277, 277, 277, 277, 277, 277,
6136 277, 277, 277, 277, 277, 277, 277, 277,
6137 278, 278, 278, 278, 278, 278, 278, 278,
6138 278, 278, 278, 278, 278, 278, 278, 278,
6139 279, 279, 279, 279, 279, 279, 279, 279,
6140 279, 279, 279, 279, 279, 279, 279, 279,
6141 280, 280, 280, 280, 280, 280, 280, 280,
6142 280, 280, 280, 280, 280, 280, 280, 280,
6143 281, 281, 281, 281, 281, 281, 281, 281,
6144 281, 281, 281, 281, 281, 281, 281, 281,
6145 281, 281, 281, 281, 281, 281, 281, 281,
6146 281, 281, 281, 281, 281, 281, 281, 281,
6147 282, 282, 282, 282, 282, 282, 282, 282,
6148 282, 282, 282, 282, 282, 282, 282, 282,
6149 282, 282, 282, 282, 282, 282, 282, 282,
6150 282, 282, 282, 282, 282, 282, 282, 282,
6151 283, 283, 283, 283, 283, 283, 283, 283,
6152 283, 283, 283, 283, 283, 283, 283, 283,
6153 283, 283, 283, 283, 283, 283, 283, 283,
6154 283, 283, 283, 283, 283, 283, 283, 283,
6155 284, 284, 284, 284, 284, 284, 284, 284,
6156 284, 284, 284, 284, 284, 284, 284, 284,
6157 284, 284, 284, 284, 284, 284, 284, 284,
6158 284, 284, 284, 284, 284, 284, 284, 285
6159 };
6160 8515617 return table[l];
6161 }
6162  
6163 /* Gets the amount of extra bits for the given length symbol. */
6164 9587 static int ZopfliGetLengthSymbolExtraBits(int s) {
6165 static const int table[29] = {
6166 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
6167 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0
6168 };
6169 9587 return table[s - 257];
6170 }
6171  
6172 /* Gets the amount of extra bits for the given distance symbol. */
6173 9587 static int ZopfliGetDistSymbolExtraBits(int s) {
6174 static const int table[30] = {
6175 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
6176 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
6177 };
6178 9587 return table[s];
6179 }
6180  
6181 #endif /* ZOPFLI_SYMBOLS_H_ */
6182  
6183 /* tree.h */
6184 /*
6185 Copyright 2011 Google Inc. All Rights Reserved.
6186  
6187 Licensed under the Apache License, Version 2.0 (the "License");
6188 you may not use this file except in compliance with the License.
6189 You may obtain a copy of the License at
6190  
6191 http://www.apache.org/licenses/LICENSE-2.0
6192  
6193 Unless required by applicable law or agreed to in writing, software
6194 distributed under the License is distributed on an "AS IS" BASIS,
6195 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
6196 See the License for the specific language governing permissions and
6197 limitations under the License.
6198  
6199 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
6200 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
6201 */
6202  
6203 /*
6204 Utilities for creating and using Huffman trees.
6205 */
6206  
6207 #ifndef ZOPFLI_TREE_H_
6208 #define ZOPFLI_TREE_H_
6209  
6210 #include <string.h>
6211  
6212 /*
6213 Calculates the bitlengths for the Huffman tree, based on the counts of each
6214 symbol.
6215 */
6216 void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits,
6217 unsigned *bitlengths);
6218  
6219 /*
6220 Converts a series of Huffman tree bitlengths, to the bit values of the symbols.
6221 */
6222 void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits,
6223 unsigned* symbols);
6224  
6225 /*
6226 Calculates the entropy of each symbol, based on the counts of each symbol. The
6227 result is similar to the result of ZopfliCalculateBitLengths, but with the
6228 actual theoritical bit lengths according to the entropy. Since the resulting
6229 values are fractional, they cannot be used to encode the tree specified by
6230 DEFLATE.
6231 */
6232 void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths);
6233  
6234 #endif /* ZOPFLI_TREE_H_ */
6235  
6236  
6237 /*
6238 bp = bitpointer, always in range [0, 7].
6239 The outsize is number of necessary bytes to encode the bits.
6240 Given the value of bp and the amount of bytes, the amount of bits represented
6241 is not simply bytesize * 8 + bp because even representing one bit requires a
6242 whole byte. It is: (bp == 0) ? (bytesize * 8) : ((bytesize - 1) * 8 + bp)
6243 */
6244 12 static void AddBit(int bit,
6245 unsigned char* bp, unsigned char** out, size_t* outsize) {
6246 12 ▶ if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 3 branches
BranchTrueFalseTotal
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 3 9 12
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 0 0 0
6247 12 (*out)[*outsize - 1] |= bit << *bp;
6248 12 *bp = (*bp + 1) & 7;
6249 12 }
6250  
6251 99 static void AddBits(unsigned symbol, unsigned length,
6252 unsigned char* bp, unsigned char** out, size_t* outsize) {
6253 /* TODO(lode): make more efficient (add more bits at once). */
6254 unsigned i;
6255 278 ▶ for (i = 0; i < length; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length; i++) { 179 99 278
6256 179 unsigned bit = (symbol >> i) & 1;
6257 179 ▶ if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 3 branches
BranchTrueFalseTotal
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 18 161 179
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 1 17 18
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 0 1 1
6258 179 (*out)[*outsize - 1] |= bit << *bp;
6259 179 *bp = (*bp + 1) & 7;
6260 }
6261 99 }
6262  
6263 /*
6264 Adds bits, like AddBits, but the order is inverted. The deflate specification
6265 uses both orders in one standard.
6266 */
6267 115 static void AddHuffmanBits(unsigned symbol, unsigned length,
6268 unsigned char* bp, unsigned char** out,
6269 size_t* outsize) {
6270 /* TODO(lode): make more efficient (add more bits at once). */
6271 unsigned i;
6272 471 ▶ for (i = 0; i < length; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length; i++) { 356 115 471
6273 356 unsigned bit = (symbol >> (length - i - 1)) & 1;
6274 356 ▶ if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 3 branches
BranchTrueFalseTotal
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 49 307 356
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 3 46 49
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
6275 356 (*out)[*outsize - 1] |= bit << *bp;
6276 356 *bp = (*bp + 1) & 7;
6277 }
6278 115 }
6279  
6280 /*
6281 Ensures there are at least 2 distance codes to support buggy decoders.
6282 Zlib 1.2.1 and below have a bug where it fails if there isn't at least 1
6283 distance code (with length > 0), even though it's valid according to the
6284 deflate spec to have 0 distance codes. On top of that, some mobile phones
6285 require at least two distance codes. To support these decoders too (but
6286 potentially at the cost of a few bytes), add dummy code lengths of 1.
6287 References to this bug can be found in the changelog of
6288 Zlib 1.2.2 and here: http://www.jonof.id.au/forum/index.php?topic=515.0.
6289  
6290 d_lengths: the 32 lengths of the distance codes.
6291 */
6292 522 static void PatchDistanceCodesForBuggyDecoders(unsigned* d_lengths) {
6293 522 int num_dist_codes = 0; /* Amount of non-zero distance codes */
6294 int i;
6295 16182 ▶ for (i = 0; i < 30 /* Ignore the two unused codes from the spec */; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 30 /* Ignore the two unused codes from the spec */; i++) { 15660 522 16182
6296 15660 ▶ if (d_lengths[i]) num_dist_codes++;
Branch: 1 branch
BranchTrueFalseTotal
if (d_lengths[i]) num_dist_codes++; 434 15226 15660
6297 15660 ▶ if (num_dist_codes >= 2) return; /* Two or more codes is fine. */
Branch: 1 branch
BranchTrueFalseTotal
if (num_dist_codes >= 2) return; /* Two or more codes is fine. */ 0 15660 15660
6298 }
6299  
6300 522 ▶ if (num_dist_codes == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (num_dist_codes == 0) { 88 434 522
6301 88 d_lengths[0] = d_lengths[1] = 1;
6302 434 ▶ } else if (num_dist_codes == 1) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (num_dist_codes == 1) { 434 0 434
6303 434 ▶ d_lengths[d_lengths[0] ? 1 : 0] = 1;
Branch: 1 branch
BranchTrueFalseTotal
d_lengths[d_lengths[0] ? 1 : 0] = 1; 0 434 434
6304 }
6305 }
6306  
6307 /*
6308 Encodes the Huffman tree and returns how many bits its encoding takes. If out
6309 is a null pointer, only returns the size and runs faster.
6310 */
6311 4185 static size_t EncodeTree(const unsigned* ll_lengths,
6312 const unsigned* d_lengths,
6313 int use_16, int use_17, int use_18,
6314 unsigned char* bp,
6315 unsigned char** out, size_t* outsize) {
6316 unsigned lld_total; /* Total amount of literal, length, distance codes. */
6317 /* Runlength encoded version of lengths of litlen and dist trees. */
6318 4185 unsigned* rle = 0;
6319 4185 unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
6320 4185 size_t rle_size = 0; /* Size of rle array. */
6321 4185 size_t rle_bits_size = 0; /* Should have same value as rle_size. */
6322 4185 unsigned hlit = 29; /* 286 - 257 */
6323 4185 unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
6324 unsigned hclen;
6325 unsigned hlit2;
6326 size_t i, j;
6327 size_t clcounts[19];
6328 unsigned clcl[19]; /* Code length code lengths. */
6329 unsigned clsymbols[19];
6330 /* The order in which code length code lengths are encoded as per deflate. */
6331 static const unsigned order[19] = {
6332 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
6333 };
6334 4185 int size_only = !out;
6335 4185 size_t result_size = 0;
6336  
6337 83700 ▶ for(i = 0; i < 19; i++) clcounts[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for(i = 0; i < 19; i++) clcounts[i] = 0; 79515 4185 83700
6338  
6339 /* Trim zeros. */
6340 31545 ▶ while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
Branch: 2 branches
BranchTrueFalseTotal
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--; 30841 704 31545
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--; 27360 3481 30841
6341 103960 ▶ while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
Branch: 2 branches
BranchTrueFalseTotal
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--; 103960 0 103960
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--; 99775 4185 103960
6342 4185 hlit2 = hlit + 257;
6343  
6344 4185 lld_total = hlit2 + hdist + 1;
6345  
6346 194678 ▶ for (i = 0; i < lld_total; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < lld_total; i++) { 190493 4185 194678
6347 /* This is an encoding of a huffman tree, so now the length is a symbol */
6348 190493 ▶ unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2];
Branch: 1 branch
BranchTrueFalseTotal
unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2]; 178554 11939 190493
6349 190493 unsigned count = 1;
6350 190493 ▶ if(use_16 || (symbol == 0 && (use_17 || use_18))) {
Branch: 4 branches
BranchTrueFalseTotal
if(use_16 || (symbol == 0 && (use_17 || use_18))) { 171513 18980 190493
if(use_16 || (symbol == 0 && (use_17 || use_18))) { 150748 20765 171513
if(use_16 || (symbol == 0 && (use_17 || use_18))) { 146374 4374 150748
if(use_16 || (symbol == 0 && (use_17 || use_18))) { 2185 144189 146374
6351 2058650 ▶ for (j = i + 1; j < lld_total && symbol ==
Branch: 2 branches
BranchTrueFalseTotal
for (j = i + 1; j < lld_total && symbol == 1028279 2092 1030371
for (j = i + 1; j < lld_total && symbol == 1004832 23447 1028279
6352 1028279 ▶ (j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) {
Branch: 1 branch
BranchTrueFalseTotal
(j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) { 1008865 19414 1028279
6353 1004832 count++;
6354 }
6355 }
6356 190493 i += count - 1;
6357  
6358 /* Repetitions of zeroes */
6359 190493 ▶ if (symbol == 0 && count >= 3) {
Branch: 2 branches
BranchTrueFalseTotal
if (symbol == 0 && count >= 3) { 159488 31005 190493
if (symbol == 0 && count >= 3) { 13583 145905 159488
6360 13583 ▶ if (use_18) {
Branch: 1 branch
BranchTrueFalseTotal
if (use_18) { 7763 5820 13583
6361 13714 ▶ while (count >= 11) {
Branch: 1 branch
BranchTrueFalseTotal
while (count >= 11) { 5951 7763 13714
6362 5951 unsigned count2 = count > 138 ? 138 : count;
6363 5951 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 3 5948 5951
6364 3 ▶ ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(18, &rle, &rle_size); 2 1 3
ZOPFLI_APPEND_DATA(18, &rle, &rle_size); 1 1 2
6365 3 ▶ ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size); 2 1 3
ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size); 1 1 2
6366 }
6367 5951 clcounts[18]++;
6368 5951 count -= count2;
6369 }
6370 }
6371 13583 ▶ if (use_17) {
Branch: 1 branch
BranchTrueFalseTotal
if (use_17) { 7763 5820 13583
6372 39232 ▶ while (count >= 3) {
Branch: 1 branch
BranchTrueFalseTotal
while (count >= 3) { 31469 7763 39232
6373 31469 unsigned count2 = count > 10 ? 10 : count;
6374 31469 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 1 31468 31469
6375 1 ▶ ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(17, &rle, &rle_size); 1 0 1
ZOPFLI_APPEND_DATA(17, &rle, &rle_size); 0 1 1
6376 1 ▶ ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); 1 0 1
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); 0 1 1
6377 }
6378 31469 clcounts[17]++;
6379 31469 count -= count2;
6380 }
6381 }
6382 }
6383  
6384 /* Repetitions of any symbol */
6385 190493 ▶ if (use_16 && count >= 4) {
Branch: 2 branches
BranchTrueFalseTotal
if (use_16 && count >= 4) { 18980 171513 190493
if (use_16 && count >= 4) { 3176 15804 18980
6386 3176 count--; /* Since the first one is hardcoded. */
6387 3176 clcounts[symbol]++;
6388 3176 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 0 3176 3176
6389 0 ▶ ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); 0 0 0
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); 0 0 0
6390 0 ▶ ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); 0 0 0
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); 0 0 0
6391 }
6392 29087 ▶ while (count >= 3) {
Branch: 1 branch
BranchTrueFalseTotal
while (count >= 3) { 25911 3176 29087
6393 25911 unsigned count2 = count > 6 ? 6 : count;
6394 25911 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 0 25911 25911
6395 0 ▶ ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(16, &rle, &rle_size); 0 0 0
ZOPFLI_APPEND_DATA(16, &rle, &rle_size); 0 0 0
6396 0 ▶ ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); 0 0 0
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size); 0 0 0
6397 }
6398 25911 clcounts[16]++;
6399 25911 count -= count2;
6400 }
6401 }
6402  
6403 /* No or insufficient repetition */
6404 190493 clcounts[symbol] += count;
6405 376401 ▶ while (count > 0) {
Branch: 1 branch
BranchTrueFalseTotal
while (count > 0) { 185908 190493 376401
6406 185908 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 6 185902 185908
6407 6 ▶ ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); 2 4 6
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size); 0 2 2
6408 6 ▶ ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); 2 4 6
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size); 0 2 2
6409 }
6410 185908 count--;
6411 }
6412 }
6413  
6414 4185 ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
6415 4185 ▶ if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols); 1 4184 4185
6416  
6417 4185 hclen = 15;
6418 /* Trim zeros. */
6419 8370 ▶ while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
Branch: 2 branches
BranchTrueFalseTotal
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--; 8370 0 8370
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--; 4185 4185 8370
6420  
6421 4185 ▶ if (!size_only) {
Branch: 1 branch
BranchTrueFalseTotal
if (!size_only) { 1 4184 4185
6422 1 AddBits(hlit, 5, bp, out, outsize);
6423 1 AddBits(hdist, 5, bp, out, outsize);
6424 1 AddBits(hclen, 4, bp, out, outsize);
6425  
6426 19 ▶ for (i = 0; i < hclen + 4; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < hclen + 4; i++) { 18 1 19
6427 18 AddBits(clcl[order[i]], 3, bp, out, outsize);
6428 }
6429  
6430 11 ▶ for (i = 0; i < rle_size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < rle_size; i++) { 10 1 11
6431 10 unsigned symbol = clsymbols[rle[i]];
6432 10 AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
6433 /* Extra bits. */
6434 10 ▶ if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
Branch: 1 branch
BranchTrueFalseTotal
if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize); 0 10 10
6435 10 ▶ else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
Branch: 1 branch
BranchTrueFalseTotal
else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize); 1 9 10
6436 9 ▶ else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
Branch: 1 branch
BranchTrueFalseTotal
else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize); 3 6 9
6437 }
6438 }
6439  
6440 4185 result_size += 14; /* hlit, hdist, hclen bits */
6441 4185 result_size += (hclen + 4) * 3; /* clcl bits */
6442 83700 ▶ for(i = 0; i < 19; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for(i = 0; i < 19; i++) { 79515 4185 83700
6443 79515 result_size += clcl[i] * clcounts[i];
6444 }
6445 /* Extra bits. */
6446 4185 result_size += clcounts[16] * 2;
6447 4185 result_size += clcounts[17] * 3;
6448 4185 result_size += clcounts[18] * 7;
6449  
6450 /* Note: in case of "size_only" these are null pointers so no effect. */
6451 4185 free(rle);
6452 4185 free(rle_bits);
6453  
6454 4185 return result_size;
6455 }
6456  
6457 1 static void AddDynamicTree(const unsigned* ll_lengths,
6458 const unsigned* d_lengths,
6459 unsigned char* bp,
6460 unsigned char** out, size_t* outsize) {
6461 int i;
6462 1 int best = 0;
6463 1 size_t bestsize = 0;
6464  
6465 9 ▶ for(i = 0; i < 8; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for(i = 0; i < 8; i++) { 8 1 9
6466 8 size_t size = EncodeTree(ll_lengths, d_lengths,
6467 i & 1, i & 2, i & 4,
6468 0, 0, 0);
6469 8 ▶ if (bestsize == 0 || size < bestsize) {
Branch: 2 branches
BranchTrueFalseTotal
if (bestsize == 0 || size < bestsize) { 7 1 8
if (bestsize == 0 || size < bestsize) { 4 3 7
6470 5 bestsize = size;
6471 5 best = i;
6472 }
6473 }
6474  
6475 1 EncodeTree(ll_lengths, d_lengths,
6476 best & 1, best & 2, best & 4,
6477 bp, out, outsize);
6478 1 }
6479  
6480 /*
6481 Gives the exact size of the tree, in bits, as it will be encoded in DEFLATE.
6482 */
6483 522 static size_t CalculateTreeSize(const unsigned* ll_lengths,
6484 const unsigned* d_lengths) {
6485 522 size_t result = 0;
6486 int i;
6487  
6488 4698 ▶ for(i = 0; i < 8; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for(i = 0; i < 8; i++) { 4176 522 4698
6489 4176 size_t size = EncodeTree(ll_lengths, d_lengths,
6490 i & 1, i & 2, i & 4,
6491 0, 0, 0);
6492 4176 ▶ if (result == 0 || size < result) result = size;
Branch: 2 branches
BranchTrueFalseTotal
if (result == 0 || size < result) result = size; 3654 522 4176
if (result == 0 || size < result) result = size; 2245 1409 3654
6493 }
6494  
6495 522 return result;
6496 }
6497  
6498 /*
6499 Adds all lit/len and dist codes from the lists as huffman symbols. Does not add
6500 end code 256. expected_data_size is the uncompressed block size, used for
6501 assert, but you can set it to 0 to not do the assertion.
6502 */
6503 4 static void AddLZ77Data(const ZopfliLZ77Store* lz77,
6504 size_t lstart, size_t lend,
6505 size_t expected_data_size,
6506 const unsigned* ll_symbols, const unsigned* ll_lengths,
6507 const unsigned* d_symbols, const unsigned* d_lengths,
6508 unsigned char* bp,
6509 unsigned char** out, size_t* outsize) {
6510 4 size_t testlength = 0;
6511 size_t i;
6512  
6513 68 ▶ for (i = lstart; i < lend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = lstart; i < lend; i++) { 64 4 68
6514 64 unsigned dist = lz77->dists[i];
6515 64 unsigned litlen = lz77->litlens[i];
6516 64 ▶ if (dist == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist == 0) { 27 37 64
6517 assert(litlen < 256);
6518 assert(ll_lengths[litlen] > 0);
6519 27 AddHuffmanBits(ll_symbols[litlen], ll_lengths[litlen], bp, out, outsize);
6520 27 testlength++;
6521 } else {
6522 37 unsigned lls = ZopfliGetLengthSymbol(litlen);
6523 37 unsigned ds = ZopfliGetDistSymbol(dist);
6524 assert(litlen >= 3 && litlen <= 288);
6525 assert(ll_lengths[lls] > 0);
6526 assert(d_lengths[ds] > 0);
6527 37 AddHuffmanBits(ll_symbols[lls], ll_lengths[lls], bp, out, outsize);
6528 37 AddBits(ZopfliGetLengthExtraBitsValue(litlen),
6529 37 ZopfliGetLengthExtraBits(litlen),
6530 bp, out, outsize);
6531 37 AddHuffmanBits(d_symbols[ds], d_lengths[ds], bp, out, outsize);
6532 37 AddBits(ZopfliGetDistExtraBitsValue(dist),
6533 37 ZopfliGetDistExtraBits(dist),
6534 bp, out, outsize);
6535 37 testlength += litlen;
6536 }
6537 }
6538 assert(expected_data_size == 0 || testlength == expected_data_size);
6539 4 }
6540  
6541 207 static void GetFixedTree(unsigned* ll_lengths, unsigned* d_lengths) {
6542 size_t i;
6543 30015 ▶ for (i = 0; i < 144; i++) ll_lengths[i] = 8;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 144; i++) ll_lengths[i] = 8; 29808 207 30015
6544 23391 ▶ for (i = 144; i < 256; i++) ll_lengths[i] = 9;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 144; i < 256; i++) ll_lengths[i] = 9; 23184 207 23391
6545 5175 ▶ for (i = 256; i < 280; i++) ll_lengths[i] = 7;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 256; i < 280; i++) ll_lengths[i] = 7; 4968 207 5175
6546 1863 ▶ for (i = 280; i < 288; i++) ll_lengths[i] = 8;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 280; i < 288; i++) ll_lengths[i] = 8; 1656 207 1863
6547 6831 ▶ for (i = 0; i < 32; i++) d_lengths[i] = 5;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 32; i++) d_lengths[i] = 5; 6624 207 6831
6548 207 }
6549  
6550 /*
6551 Same as CalculateBlockSymbolSize, but for block size smaller than histogram
6552 size.
6553 */
6554 726 static size_t CalculateBlockSymbolSizeSmall(const unsigned* ll_lengths,
6555 const unsigned* d_lengths,
6556 const ZopfliLZ77Store* lz77,
6557 size_t lstart, size_t lend) {
6558 726 size_t result = 0;
6559 size_t i;
6560 13040 ▶ for (i = lstart; i < lend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = lstart; i < lend; i++) { 12314 726 13040
6561 assert(i < lz77->size);
6562 assert(lz77->litlens[i] < 259);
6563 12314 ▶ if (lz77->dists[i] == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (lz77->dists[i] == 0) { 2727 9587 12314
6564 2727 result += ll_lengths[lz77->litlens[i]];
6565 } else {
6566 9587 int ll_symbol = ZopfliGetLengthSymbol(lz77->litlens[i]);
6567 9587 int d_symbol = ZopfliGetDistSymbol(lz77->dists[i]);
6568 9587 result += ll_lengths[ll_symbol];
6569 9587 result += d_lengths[d_symbol];
6570 9587 result += ZopfliGetLengthSymbolExtraBits(ll_symbol);
6571 9587 result += ZopfliGetDistSymbolExtraBits(d_symbol);
6572 }
6573 }
6574 726 result += ll_lengths[256]; /*end symbol*/
6575 726 return result;
6576 }
6577  
6578 /*
6579 Same as CalculateBlockSymbolSize, but with the histogram provided by the caller.
6580 */
6581 522 static size_t CalculateBlockSymbolSizeGivenCounts(const size_t* ll_counts,
6582 const size_t* d_counts,
6583 const unsigned* ll_lengths,
6584 const unsigned* d_lengths,
6585 const ZopfliLZ77Store* lz77,
6586 size_t lstart, size_t lend) {
6587 522 size_t result = 0;
6588 size_t i;
6589 522 ▶ if (lstart + ZOPFLI_NUM_LL * 3 > lend) {
Branch: 1 branch
BranchTrueFalseTotal
if (lstart + ZOPFLI_NUM_LL * 3 > lend) { 522 0 522
6590 522 return CalculateBlockSymbolSizeSmall(
6591 ll_lengths, d_lengths, lz77, lstart, lend);
6592 } else {
6593 0 ▶ for (i = 0; i < 256; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 256; i++) { 0 0 0
6594 0 result += ll_lengths[i] * ll_counts[i];
6595 }
6596 0 ▶ for (i = 257; i < 286; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 257; i < 286; i++) { 0 0 0
6597 0 result += ll_lengths[i] * ll_counts[i];
6598 0 result += ZopfliGetLengthSymbolExtraBits(i) * ll_counts[i];
6599 }
6600 0 ▶ for (i = 0; i < 30; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 30; i++) { 0 0 0
6601 0 result += d_lengths[i] * d_counts[i];
6602 0 result += ZopfliGetDistSymbolExtraBits(i) * d_counts[i];
6603 }
6604 0 result += ll_lengths[256]; /*end symbol*/
6605 0 return result;
6606 }
6607 }
6608  
6609 /*
6610 Calculates size of the part after the header and tree of an LZ77 block, in bits.
6611 */
6612 204 static size_t CalculateBlockSymbolSize(const unsigned* ll_lengths,
6613 const unsigned* d_lengths,
6614 const ZopfliLZ77Store* lz77,
6615 size_t lstart, size_t lend) {
6616 204 ▶ if (lstart + ZOPFLI_NUM_LL * 3 > lend) {
Branch: 1 branch
BranchTrueFalseTotal
if (lstart + ZOPFLI_NUM_LL * 3 > lend) { 204 0 204
6617 204 return CalculateBlockSymbolSizeSmall(
6618 ll_lengths, d_lengths, lz77, lstart, lend);
6619 } else {
6620 size_t ll_counts[ZOPFLI_NUM_LL];
6621 size_t d_counts[ZOPFLI_NUM_D];
6622 0 ZopfliLZ77GetHistogram(lz77, lstart, lend, ll_counts, d_counts);
6623 0 return CalculateBlockSymbolSizeGivenCounts(
6624 ll_counts, d_counts, ll_lengths, d_lengths, lz77, lstart, lend);
6625 }
6626 }
6627  
6628 2705 static size_t AbsDiff(size_t x, size_t y) {
6629 2705 ▶ if (x > y)
Branch: 1 branch
BranchTrueFalseTotal
if (x > y) 790 1915 2705
6630 790 return x - y;
6631 else
6632 1915 return y - x;
6633 }
6634  
6635 /*
6636 Changes the population counts in a way that the consequent Huffman tree
6637 compression, especially its rle-part, will be more likely to compress this data
6638 more efficiently. length contains the size of the histogram.
6639 */
6640 522 void OptimizeHuffmanForRle(int length, size_t* counts) {
6641 int i, k, stride;
6642 size_t symbol, sum, limit;
6643 int* good_for_rle;
6644  
6645 /* 1) We don't want to touch the trailing zeros. We may break the
6646 rules of the format by adding more data in the distance codes. */
6647 9587 ▶ for (; length >= 0; --length) {
Branch: 1 branch
BranchTrueFalseTotal
for (; length >= 0; --length) { 9587 0 9587
6648 9587 ▶ if (length == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (length == 0) { 44 9543 9587
6649 44 return;
6650 }
6651 9543 ▶ if (counts[length - 1] != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (counts[length - 1] != 0) { 478 9065 9543
6652 /* Now counts[0..length - 1] does not have trailing zeros. */
6653 478 break;
6654 }
6655 }
6656 /* 2) Let's mark all population counts that already can be encoded
6657 with an rle code.*/
6658 478 good_for_rle = (int*)malloc((unsigned)length * sizeof(int));
6659 74933 ▶ for (i = 0; i < length; ++i) good_for_rle[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length; ++i) good_for_rle[i] = 0; 74455 478 74933
6660  
6661 /* Let's not spoil any of the existing good rle codes.
6662 Mark any seq of 0's that is longer than 5 as a good_for_rle.
6663 Mark any seq of non-0's that is longer than 7 as a good_for_rle.*/
6664 478 symbol = counts[0];
6665 478 stride = 0;
6666 75411 ▶ for (i = 0; i < length + 1; ++i) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length + 1; ++i) { 74933 478 75411
6667 74933 ▶ if (i == length || counts[i] != symbol) {
Branch: 2 branches
BranchTrueFalseTotal
if (i == length || counts[i] != symbol) { 74455 478 74933
if (i == length || counts[i] != symbol) { 2068 72387 74455
6668 2546 ▶ if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) {
Branch: 4 branches
BranchTrueFalseTotal
if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { 1273 1273 2546
if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { 421 852 1273
if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { 1273 421 1694
if ((symbol == 0 && stride >= 5) || (symbol != 0 && stride >= 7)) { 0 1273 1273
6669 72602 ▶ for (k = 0; k < stride; ++k) {
Branch: 1 branch
BranchTrueFalseTotal
for (k = 0; k < stride; ++k) { 71750 852 72602
6670 71750 good_for_rle[i - k - 1] = 1;
6671 }
6672 }
6673 2546 stride = 1;
6674 2546 ▶ if (i != length) {
Branch: 1 branch
BranchTrueFalseTotal
if (i != length) { 2068 478 2546
6675 2068 symbol = counts[i];
6676 }
6677 } else {
6678 72387 ++stride;
6679 }
6680 }
6681  
6682 /* 3) Let's replace those population counts that lead to more rle codes. */
6683 478 stride = 0;
6684 478 limit = counts[0];
6685 478 sum = 0;
6686 75411 ▶ for (i = 0; i < length + 1; ++i) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length + 1; ++i) { 74933 478 75411
6687 74933 ▶ if (i == length || good_for_rle[i]
Branch: 2 branches
BranchTrueFalseTotal
if (i == length || good_for_rle[i] 74455 478 74933
if (i == length || good_for_rle[i] 2705 71750 74455
6688 /* Heuristic for selecting the stride ranges to collapse. */
6689 2705 ▶ || AbsDiff(counts[i], limit) >= 4) {
Branch: 1 branch
BranchTrueFalseTotal
|| AbsDiff(counts[i], limit) >= 4) { 348 2357 2705
6690 72576 ▶ if (stride >= 4 || (stride >= 3 && sum == 0)) {
Branch: 3 branches
BranchTrueFalseTotal
if (stride >= 4 || (stride >= 3 && sum == 0)) { 72441 135 72576
if (stride >= 4 || (stride >= 3 && sum == 0)) { 7 72434 72441
if (stride >= 4 || (stride >= 3 && sum == 0)) { 0 7 7
6691 /* The stride must end, collapse what we have, if we have enough (4). */
6692 135 int count = (sum + stride / 2) / stride;
6693 135 ▶ if (count < 1) count = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (count < 1) count = 1; 36 99 135
6694 135 ▶ if (sum == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (sum == 0) { 0 135 135
6695 /* Don't make an all zeros stride to be upgraded to ones. */
6696 0 count = 0;
6697 }
6698 2146 ▶ for (k = 0; k < stride; ++k) {
Branch: 1 branch
BranchTrueFalseTotal
for (k = 0; k < stride; ++k) { 2011 135 2146
6699 /* We don't want to change value at counts[i],
6700 that is already belonging to the next stride. Thus - 1. */
6701 2011 counts[i - k - 1] = count;
6702 }
6703 }
6704 72576 stride = 0;
6705 72576 sum = 0;
6706 72576 ▶ if (i < length - 3) {
Branch: 1 branch
BranchTrueFalseTotal
if (i < length - 3) { 70916 1660 72576
6707 /* All interesting strides have a count of at least 4,
6708 at least when non-zeros. */
6709 70916 limit = (counts[i] + counts[i + 1] +
6710 70916 counts[i + 2] + counts[i + 3] + 2) / 4;
6711 1660 ▶ } else if (i < length) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (i < length) { 1182 478 1660
6712 1182 limit = counts[i];
6713 } else {
6714 478 limit = 0;
6715 }
6716 }
6717 74933 ++stride;
6718 74933 ▶ if (i != length) {
Branch: 1 branch
BranchTrueFalseTotal
if (i != length) { 74455 478 74933
6719 74455 sum += counts[i];
6720 }
6721 }
6722  
6723 478 free(good_for_rle);
6724 }
6725  
6726 /*
6727 Tries out OptimizeHuffmanForRle for this block, if the result is smaller,
6728 uses it, otherwise keeps the original. Returns size of encoded tree and data in
6729 bits, not including the 3-bit block header.
6730 */
6731 261 static double TryOptimizeHuffmanForRle(
6732 const ZopfliLZ77Store* lz77, size_t lstart, size_t lend,
6733 const size_t* ll_counts, const size_t* d_counts,
6734 unsigned* ll_lengths, unsigned* d_lengths) {
6735 size_t ll_counts2[ZOPFLI_NUM_LL];
6736 size_t d_counts2[ZOPFLI_NUM_D];
6737 unsigned ll_lengths2[ZOPFLI_NUM_LL];
6738 unsigned d_lengths2[ZOPFLI_NUM_D];
6739 double treesize;
6740 double datasize;
6741 double treesize2;
6742 double datasize2;
6743  
6744 261 treesize = CalculateTreeSize(ll_lengths, d_lengths);
6745 261 datasize = CalculateBlockSymbolSizeGivenCounts(ll_counts, d_counts,
6746 ll_lengths, d_lengths, lz77, lstart, lend);
6747  
6748 261 memcpy(ll_counts2, ll_counts, sizeof(ll_counts2));
6749 261 memcpy(d_counts2, d_counts, sizeof(d_counts2));
6750 261 OptimizeHuffmanForRle(ZOPFLI_NUM_LL, ll_counts2);
6751 261 OptimizeHuffmanForRle(ZOPFLI_NUM_D, d_counts2);
6752 261 ZopfliCalculateBitLengths(ll_counts2, ZOPFLI_NUM_LL, 15, ll_lengths2);
6753 261 ZopfliCalculateBitLengths(d_counts2, ZOPFLI_NUM_D, 15, d_lengths2);
6754 261 PatchDistanceCodesForBuggyDecoders(d_lengths2);
6755  
6756 261 treesize2 = CalculateTreeSize(ll_lengths2, d_lengths2);
6757 261 datasize2 = CalculateBlockSymbolSizeGivenCounts(ll_counts, d_counts,
6758 ll_lengths2, d_lengths2, lz77, lstart, lend);
6759  
6760 261 ▶ if (treesize2 + datasize2 < treesize + datasize) {
Branch: 1 branch
BranchTrueFalseTotal
if (treesize2 + datasize2 < treesize + datasize) { 95 166 261
6761 95 memcpy(ll_lengths, ll_lengths2, sizeof(ll_lengths2));
6762 95 memcpy(d_lengths, d_lengths2, sizeof(d_lengths2));
6763 95 return treesize2 + datasize2;
6764 }
6765 166 return treesize + datasize;
6766 }
6767  
6768 /*
6769 Calculates the bit lengths for the symbols for dynamic blocks. Chooses bit
6770 lengths that give the smallest size of tree encoding + encoding of all the
6771 symbols to have smallest output size. This are not necessarily the ideal Huffman
6772 bit lengths. Returns size of encoded tree and data in bits, not including the
6773 3-bit block header.
6774 */
6775 261 static double GetDynamicLengths(const ZopfliLZ77Store* lz77,
6776 size_t lstart, size_t lend,
6777 unsigned* ll_lengths, unsigned* d_lengths) {
6778 size_t ll_counts[ZOPFLI_NUM_LL];
6779 size_t d_counts[ZOPFLI_NUM_D];
6780  
6781 261 ZopfliLZ77GetHistogram(lz77, lstart, lend, ll_counts, d_counts);
6782 261 ll_counts[256] = 1; /* End symbol. */
6783 261 ZopfliCalculateBitLengths(ll_counts, ZOPFLI_NUM_LL, 15, ll_lengths);
6784 261 ZopfliCalculateBitLengths(d_counts, ZOPFLI_NUM_D, 15, d_lengths);
6785 261 PatchDistanceCodesForBuggyDecoders(d_lengths);
6786 261 return TryOptimizeHuffmanForRle(
6787 lz77, lstart, lend, ll_counts, d_counts, ll_lengths, d_lengths);
6788 }
6789  
6790 664 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
6791 size_t lstart, size_t lend, int btype) {
6792 unsigned ll_lengths[ZOPFLI_NUM_LL];
6793 unsigned d_lengths[ZOPFLI_NUM_D];
6794  
6795 664 double result = 3; /* bfinal and btype bits */
6796  
6797 664 ▶ if (btype == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (btype == 0) { 200 464 664
6798 200 size_t length = ZopfliLZ77GetByteRange(lz77, lstart, lend);
6799 200 size_t rem = length % 65535;
6800 200 ▶ size_t blocks = length / 65535 + (rem ? 1 : 0);
Branch: 1 branch
BranchTrueFalseTotal
size_t blocks = length / 65535 + (rem ? 1 : 0); 200 0 200
6801 /* An uncompressed block must actually be split into multiple blocks if it's
6802 larger than 65535 bytes long. Eeach block header is 5 bytes: 3 bits,
6803 padding, LEN and NLEN (potential less padding for first one ignored). */
6804 200 return blocks * 5 * 8 + length * 8;
6805 464 ▶ } if (btype == 1) {
Branch: 1 branch
BranchTrueFalseTotal
} if (btype == 1) { 204 260 464
6806 204 GetFixedTree(ll_lengths, d_lengths);
6807 204 result += CalculateBlockSymbolSize(
6808 ll_lengths, d_lengths, lz77, lstart, lend);
6809 } else {
6810 260 result += GetDynamicLengths(lz77, lstart, lend, ll_lengths, d_lengths);
6811 }
6812  
6813 464 return result;
6814 }
6815  
6816 196 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
6817 size_t lstart, size_t lend) {
6818 196 double uncompressedcost = ZopfliCalculateBlockSize(lz77, lstart, lend, 0);
6819 /* Don't do the expensive fixed cost calculation for larger blocks that are
6820 unlikely to use it. */
6821 392 double fixedcost = (lz77->size > 1000) ?
6822 196 ▶ uncompressedcost : ZopfliCalculateBlockSize(lz77, lstart, lend, 1);
Branch: 1 branch
BranchTrueFalseTotal
uncompressedcost : ZopfliCalculateBlockSize(lz77, lstart, lend, 1); 196 0 196
6823 196 double dyncost = ZopfliCalculateBlockSize(lz77, lstart, lend, 2);
6824 0 ▶ return (uncompressedcost < fixedcost && uncompressedcost < dyncost)
Branch: 1 branch
BranchTrueFalseTotal
return (uncompressedcost < fixedcost && uncompressedcost < dyncost) 0 0 0
6825 ? uncompressedcost
6826 196 ▶ : (fixedcost < dyncost ? fixedcost : dyncost);
Branch: 2 branches
BranchTrueFalseTotal
: (fixedcost < dyncost ? fixedcost : dyncost); 0 196 196
: (fixedcost < dyncost ? fixedcost : dyncost); 84 112 196
6827 }
6828  
6829 /* Since an uncompressed block can be max 65535 in size, it actually adds
6830 multible blocks if needed. */
6831 0 static void AddNonCompressedBlock(const ZopfliOptions* options, int final,
6832 const unsigned char* in, size_t instart,
6833 size_t inend,
6834 unsigned char* bp,
6835 unsigned char** out, size_t* outsize) {
6836 0 size_t pos = instart;
6837 (void)options;
6838 0 for (;;) {
6839 size_t i;
6840 0 unsigned short blocksize = 65535;
6841 unsigned short nlen;
6842 int currentfinal;
6843  
6844 0 ▶ if (pos + blocksize > inend) blocksize = inend - pos;
Branch: 1 branch
BranchTrueFalseTotal
if (pos + blocksize > inend) blocksize = inend - pos; 0 0 0
6845 0 currentfinal = pos + blocksize >= inend;
6846  
6847 0 nlen = ~blocksize;
6848  
6849 0 ▶ AddBit(final && currentfinal, bp, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
AddBit(final && currentfinal, bp, out, outsize); 0 0 0
AddBit(final && currentfinal, bp, out, outsize); 0 0 0
6850 /* BTYPE 00 */
6851 0 AddBit(0, bp, out, outsize);
6852 0 AddBit(0, bp, out, outsize);
6853  
6854 /* Any bits of input up to the next byte boundary are ignored. */
6855 0 *bp = 0;
6856  
6857 0 ▶ ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize); 0 0 0
6858 0 ▶ ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize); 0 0 0
6859 0 ▶ ZOPFLI_APPEND_DATA(nlen % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(nlen % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(nlen % 256, out, outsize); 0 0 0
6860 0 ▶ ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize); 0 0 0
6861  
6862 0 ▶ for (i = 0; i < blocksize; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < blocksize; i++) { 0 0 0
6863 0 ▶ ZOPFLI_APPEND_DATA(in[pos + i], out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(in[pos + i], out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(in[pos + i], out, outsize); 0 0 0
6864 }
6865  
6866 0 ▶ if (currentfinal) break;
Branch: 1 branch
BranchTrueFalseTotal
if (currentfinal) break; 0 0 0
6867 0 pos += blocksize;
6868 }
6869 0 }
6870  
6871 /*
6872 Adds a deflate block with the given LZ77 data to the output.
6873 options: global program options
6874 btype: the block type, must be 1 or 2
6875 final: whether to set the "final" bit on this block, must be the last block
6876 litlens: literal/length array of the LZ77 data, in the same format as in
6877 ZopfliLZ77Store.
6878 dists: distance array of the LZ77 data, in the same format as in
6879 ZopfliLZ77Store.
6880 lstart: where to start in the LZ77 data
6881 lend: where to end in the LZ77 data (not inclusive)
6882 expected_data_size: the uncompressed block size, used for assert, but you can
6883 set it to 0 to not do the assertion.
6884 bp: output bit pointer
6885 out: dynamic output array to append to
6886 outsize: dynamic output array size
6887 */
6888 4 static void AddLZ77Block(const ZopfliOptions* options, int btype, int final,
6889 const ZopfliLZ77Store* lz77,
6890 size_t lstart, size_t lend,
6891 size_t expected_data_size,
6892 unsigned char* bp,
6893 unsigned char** out, size_t* outsize) {
6894 unsigned ll_lengths[ZOPFLI_NUM_LL];
6895 unsigned d_lengths[ZOPFLI_NUM_D];
6896 unsigned ll_symbols[ZOPFLI_NUM_LL];
6897 unsigned d_symbols[ZOPFLI_NUM_D];
6898 4 size_t detect_block_size = *outsize;
6899 size_t compressed_size;
6900 4 size_t uncompressed_size = 0;
6901 size_t i;
6902 4 ▶ if (btype == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (btype == 0) { 0 4 4
6903 0 size_t length = ZopfliLZ77GetByteRange(lz77, lstart, lend);
6904 0 ▶ size_t pos = lstart == lend ? 0 : lz77->pos[lstart];
Branch: 1 branch
BranchTrueFalseTotal
size_t pos = lstart == lend ? 0 : lz77->pos[lstart]; 0 0 0
6905 0 size_t end = pos + length;
6906 0 AddNonCompressedBlock(options, final,
6907 0 lz77->data, pos, end, bp, out, outsize);
6908 0 return;
6909 }
6910  
6911 4 AddBit(final, bp, out, outsize);
6912 4 AddBit(btype & 1, bp, out, outsize);
6913 4 AddBit((btype & 2) >> 1, bp, out, outsize);
6914  
6915 4 ▶ if (btype == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (btype == 1) { 3 1 4
6916 /* Fixed block. */
6917 3 GetFixedTree(ll_lengths, d_lengths);
6918 } else {
6919 /* Dynamic block. */
6920 unsigned detect_tree_size;
6921 assert(btype == 2);
6922  
6923 1 GetDynamicLengths(lz77, lstart, lend, ll_lengths, d_lengths);
6924  
6925 1 detect_tree_size = *outsize;
6926 1 AddDynamicTree(ll_lengths, d_lengths, bp, out, outsize);
6927 1 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 1 1
6928 0 fprintf(stderr, "treesize: %d\n", (int)(*outsize - detect_tree_size));
6929 }
6930 }
6931  
6932 4 ZopfliLengthsToSymbols(ll_lengths, ZOPFLI_NUM_LL, 15, ll_symbols);
6933 4 ZopfliLengthsToSymbols(d_lengths, ZOPFLI_NUM_D, 15, d_symbols);
6934  
6935 4 detect_block_size = *outsize;
6936 4 AddLZ77Data(lz77, lstart, lend, expected_data_size,
6937 ll_symbols, ll_lengths, d_symbols, d_lengths,
6938 bp, out, outsize);
6939 /* End symbol. */
6940 4 AddHuffmanBits(ll_symbols[256], ll_lengths[256], bp, out, outsize);
6941  
6942 68 ▶ for (i = lstart; i < lend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = lstart; i < lend; i++) { 64 4 68
6943 64 ▶ uncompressed_size += lz77->dists[i] == 0 ? 1 : lz77->litlens[i];
Branch: 1 branch
BranchTrueFalseTotal
uncompressed_size += lz77->dists[i] == 0 ? 1 : lz77->litlens[i]; 37 27 64
6944 }
6945 4 compressed_size = *outsize - detect_block_size;
6946 4 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 4 4
6947 0 fprintf(stderr, "compressed block size: %d (%dk) (unc: %d)\n",
6948 0 (int)compressed_size, (int)(compressed_size / 1024),
6949 (int)(uncompressed_size));
6950 }
6951 }
6952  
6953 4 static void AddLZ77BlockAutoType(const ZopfliOptions* options, int final,
6954 const ZopfliLZ77Store* lz77,
6955 size_t lstart, size_t lend,
6956 size_t expected_data_size,
6957 unsigned char* bp,
6958 unsigned char** out, size_t* outsize) {
6959 4 double uncompressedcost = ZopfliCalculateBlockSize(lz77, lstart, lend, 0);
6960 4 double fixedcost = ZopfliCalculateBlockSize(lz77, lstart, lend, 1);
6961 4 double dyncost = ZopfliCalculateBlockSize(lz77, lstart, lend, 2);
6962  
6963 /* Whether to perform the expensive calculation of creating an optimal block
6964 with fixed huffman tree to check if smaller. Only do this for small blocks or
6965 blocks which already are pretty good with fixed huffman tree. */
6966 4 ▶ int expensivefixed = (lz77->size < 1000) || fixedcost <= dyncost * 1.1;
Branch: 2 branches
BranchTrueFalseTotal
int expensivefixed = (lz77->size < 1000) || fixedcost <= dyncost * 1.1; 0 4 4
int expensivefixed = (lz77->size < 1000) || fixedcost <= dyncost * 1.1; 0 0 0
6967  
6968 ZopfliLZ77Store fixedstore;
6969 4 ▶ if (lstart == lend) {
Branch: 1 branch
BranchTrueFalseTotal
if (lstart == lend) { 0 4 4
6970 /* Smallest empty block is represented by fixed block */
6971 0 AddBits(final, 1, bp, out, outsize);
6972 0 AddBits(1, 2, bp, out, outsize); /* btype 01 */
6973 0 AddBits(0, 7, bp, out, outsize); /* end symbol has code 0000000 */
6974 0 return;
6975 }
6976 4 ZopfliInitLZ77Store(lz77->data, &fixedstore);
6977 4 ▶ if (expensivefixed) {
Branch: 1 branch
BranchTrueFalseTotal
if (expensivefixed) { 4 0 4
6978 /* Recalculate the LZ77 with ZopfliLZ77OptimalFixed */
6979 4 size_t instart = lz77->pos[lstart];
6980 4 size_t inend = instart + ZopfliLZ77GetByteRange(lz77, lstart, lend);
6981  
6982 ZopfliBlockState s;
6983 4 ZopfliInitBlockState(options, instart, inend, 1, &s);
6984 4 ZopfliLZ77OptimalFixed(&s, lz77->data, instart, inend, &fixedstore);
6985 4 fixedcost = ZopfliCalculateBlockSize(&fixedstore, 0, fixedstore.size, 1);
6986 4 ZopfliCleanBlockState(&s);
6987 }
6988  
6989 4 ▶ if (uncompressedcost < fixedcost && uncompressedcost < dyncost) {
Branch: 2 branches
BranchTrueFalseTotal
if (uncompressedcost < fixedcost && uncompressedcost < dyncost) { 0 4 4
if (uncompressedcost < fixedcost && uncompressedcost < dyncost) { 0 0 0
6990 0 AddLZ77Block(options, 0, final, lz77, lstart, lend,
6991 expected_data_size, bp, out, outsize);
6992 4 ▶ } else if (fixedcost < dyncost) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (fixedcost < dyncost) { 3 1 4
6993 3 ▶ if (expensivefixed) {
Branch: 1 branch
BranchTrueFalseTotal
if (expensivefixed) { 3 0 3
6994 3 AddLZ77Block(options, 1, final, &fixedstore, 0, fixedstore.size,
6995 expected_data_size, bp, out, outsize);
6996 } else {
6997 0 AddLZ77Block(options, 1, final, lz77, lstart, lend,
6998 expected_data_size, bp, out, outsize);
6999 }
7000 } else {
7001 1 AddLZ77Block(options, 2, final, lz77, lstart, lend,
7002 expected_data_size, bp, out, outsize);
7003 }
7004  
7005 4 ZopfliCleanLZ77Store(&fixedstore);
7006 }
7007  
7008 /*
7009 Deflate a part, to allow ZopfliDeflate() to use multiple master blocks if
7010 needed.
7011 It is possible to call this function multiple times in a row, shifting
7012 instart and inend to next bytes of the data. If instart is larger than 0, then
7013 previous bytes are used as the initial dictionary for LZ77.
7014 This function will usually output multiple deflate blocks. If final is 1, then
7015 the final bit will be set on the last block.
7016 */
7017 3 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
7018 const unsigned char* in, size_t instart, size_t inend,
7019 unsigned char* bp, unsigned char** out,
7020 size_t* outsize) {
7021 size_t i;
7022 /* byte coordinates rather than lz77 index */
7023 3 size_t* splitpoints_uncompressed = 0;
7024 3 size_t npoints = 0;
7025 3 size_t* splitpoints = 0;
7026 3 double totalcost = 0;
7027 ZopfliLZ77Store lz77;
7028  
7029 /* If btype=2 is specified, it tries all block types. If a lesser btype is
7030 given, then however it forces that one. Neither of the lesser types needs
7031 block splitting as they have no dynamic huffman trees. */
7032 3 ▶ if (btype == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (btype == 0) { 0 3 3
7033 0 AddNonCompressedBlock(options, final, in, instart, inend, bp, out, outsize);
7034 0 return;
7035 3 ▶ } else if (btype == 1) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (btype == 1) { 0 3 3
7036 ZopfliLZ77Store store;
7037 ZopfliBlockState s;
7038 0 ZopfliInitLZ77Store(in, &store);
7039 0 ZopfliInitBlockState(options, instart, inend, 1, &s);
7040  
7041 0 ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store);
7042 0 AddLZ77Block(options, btype, final, &store, 0, store.size, 0,
7043 bp, out, outsize);
7044  
7045 0 ZopfliCleanBlockState(&s);
7046 0 ZopfliCleanLZ77Store(&store);
7047 0 return;
7048 }
7049  
7050  
7051 3 ▶ if (options->blocksplitting) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->blocksplitting) { 3 0 3
7052 3 ZopfliBlockSplit(options, in, instart, inend,
7053 3 options->blocksplittingmax,
7054 &splitpoints_uncompressed, &npoints);
7055 3 splitpoints = (size_t*)malloc(sizeof(*splitpoints) * npoints);
7056 }
7057  
7058 3 ZopfliInitLZ77Store(in, &lz77);
7059  
7060 7 ▶ for (i = 0; i <= npoints; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i <= npoints; i++) { 4 3 7
7061 4 ▶ size_t start = i == 0 ? instart : splitpoints_uncompressed[i - 1];
Branch: 1 branch
BranchTrueFalseTotal
size_t start = i == 0 ? instart : splitpoints_uncompressed[i - 1]; 1 3 4
7062 4 ▶ size_t end = i == npoints ? inend : splitpoints_uncompressed[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t end = i == npoints ? inend : splitpoints_uncompressed[i]; 1 3 4
7063 ZopfliBlockState s;
7064 ZopfliLZ77Store store;
7065 4 ZopfliInitLZ77Store(in, &store);
7066 4 ZopfliInitBlockState(options, start, end, 1, &s);
7067 4 ZopfliLZ77Optimal(&s, in, start, end, options->numiterations, &store);
7068 4 totalcost += ZopfliCalculateBlockSizeAutoType(&store, 0, store.size);
7069  
7070 4 ZopfliAppendLZ77Store(&store, &lz77);
7071 4 ▶ if (i < npoints) splitpoints[i] = lz77.size;
Branch: 1 branch
BranchTrueFalseTotal
if (i < npoints) splitpoints[i] = lz77.size; 1 3 4
7072  
7073 4 ZopfliCleanBlockState(&s);
7074 4 ZopfliCleanLZ77Store(&store);
7075 }
7076  
7077 /* Second block splitting attempt */
7078 3 ▶ if (options->blocksplitting && npoints > 1) {
Branch: 2 branches
BranchTrueFalseTotal
if (options->blocksplitting && npoints > 1) { 3 0 3
if (options->blocksplitting && npoints > 1) { 0 3 3
7079 0 size_t* splitpoints2 = 0;
7080 0 size_t npoints2 = 0;
7081 0 double totalcost2 = 0;
7082  
7083 0 ZopfliBlockSplitLZ77(options, &lz77,
7084 0 options->blocksplittingmax, &splitpoints2, &npoints2);
7085  
7086 0 ▶ for (i = 0; i <= npoints2; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i <= npoints2; i++) { 0 0 0
7087 0 ▶ size_t start = i == 0 ? 0 : splitpoints2[i - 1];
Branch: 1 branch
BranchTrueFalseTotal
size_t start = i == 0 ? 0 : splitpoints2[i - 1]; 0 0 0
7088 0 ▶ size_t end = i == npoints2 ? lz77.size : splitpoints2[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t end = i == npoints2 ? lz77.size : splitpoints2[i]; 0 0 0
7089 0 totalcost2 += ZopfliCalculateBlockSizeAutoType(&lz77, start, end);
7090 }
7091  
7092 0 ▶ if (totalcost2 < totalcost) {
Branch: 1 branch
BranchTrueFalseTotal
if (totalcost2 < totalcost) { 0 0 0
7093 0 free(splitpoints);
7094 0 splitpoints = splitpoints2;
7095 0 npoints = npoints2;
7096 } else {
7097 0 free(splitpoints2);
7098 }
7099 }
7100  
7101 7 ▶ for (i = 0; i <= npoints; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i <= npoints; i++) { 4 3 7
7102 4 ▶ size_t start = i == 0 ? 0 : splitpoints[i - 1];
Branch: 1 branch
BranchTrueFalseTotal
size_t start = i == 0 ? 0 : splitpoints[i - 1]; 1 3 4
7103 4 ▶ size_t end = i == npoints ? lz77.size : splitpoints[i];
Branch: 1 branch
BranchTrueFalseTotal
size_t end = i == npoints ? lz77.size : splitpoints[i]; 3 1 4
7104 4 ▶ AddLZ77BlockAutoType(options, i == npoints && final,
Branch: 2 branches
BranchTrueFalseTotal
AddLZ77BlockAutoType(options, i == npoints && final, 3 1 4
AddLZ77BlockAutoType(options, i == npoints && final, 3 0 3
7105 &lz77, start, end, 0,
7106 bp, out, outsize);
7107 }
7108  
7109 3 ZopfliCleanLZ77Store(&lz77);
7110 3 free(splitpoints);
7111 3 free(splitpoints_uncompressed);
7112 }
7113  
7114 3 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
7115 const unsigned char* in, size_t insize,
7116 unsigned char* bp, unsigned char** out, size_t* outsize) {
7117 3 size_t offset = *outsize;
7118 #if ZOPFLI_MASTER_BLOCK_SIZE == 0
7119 ZopfliDeflatePart(options, btype, final, in, 0, insize, bp, out, outsize);
7120 #else
7121 3 size_t i = 0;
7122 do {
7123 3 int masterfinal = (i + ZOPFLI_MASTER_BLOCK_SIZE >= insize);
7124 3 ▶ int final2 = final && masterfinal;
Branch: 2 branches
BranchTrueFalseTotal
int final2 = final && masterfinal; 3 0 3
int final2 = final && masterfinal; 3 0 3
7125 3 ▶ size_t size = masterfinal ? insize - i : ZOPFLI_MASTER_BLOCK_SIZE;
Branch: 1 branch
BranchTrueFalseTotal
size_t size = masterfinal ? insize - i : ZOPFLI_MASTER_BLOCK_SIZE; 3 0 3
7126 3 ZopfliDeflatePart(options, btype, final2,
7127 in, i, i + size, bp, out, outsize);
7128 3 i += size;
7129 3 ▶ } while (i < insize);
Branch: 1 branch
BranchTrueFalseTotal
} while (i < insize); 0 3 3
7130 #endif
7131 3 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 3 3
7132 0 fprintf(stderr,
7133 "Original Size: %lu, Deflate: %lu, Compression: %f%% Removed\n",
7134 0 (unsigned long)insize, (unsigned long)(*outsize - offset),
7135 0 100.0 * (double)(insize - (*outsize - offset)) / (double)insize);
7136 }
7137 3 }
7138 /*
7139 Copyright 2013 Google Inc. All Rights Reserved.
7140  
7141 Licensed under the Apache License, Version 2.0 (the "License");
7142 you may not use this file except in compliance with the License.
7143 You may obtain a copy of the License at
7144  
7145 http://www.apache.org/licenses/LICENSE-2.0
7146  
7147 Unless required by applicable law or agreed to in writing, software
7148 distributed under the License is distributed on an "AS IS" BASIS,
7149 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7150 See the License for the specific language governing permissions and
7151 limitations under the License.
7152  
7153 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7154 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7155 */
7156  
7157 /* gzip_container.h */
7158 /*
7159 Copyright 2013 Google Inc. All Rights Reserved.
7160  
7161 Licensed under the Apache License, Version 2.0 (the "License");
7162 you may not use this file except in compliance with the License.
7163 You may obtain a copy of the License at
7164  
7165 http://www.apache.org/licenses/LICENSE-2.0
7166  
7167 Unless required by applicable law or agreed to in writing, software
7168 distributed under the License is distributed on an "AS IS" BASIS,
7169 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7170 See the License for the specific language governing permissions and
7171 limitations under the License.
7172  
7173 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7174 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7175 */
7176  
7177 #ifndef ZOPFLI_GZIP_H_
7178 #define ZOPFLI_GZIP_H_
7179  
7180 /*
7181 Functions to compress according to the Gzip specification.
7182 */
7183  
7184 /* zopfli.h */
7185 /*
7186 Copyright 2011 Google Inc. All Rights Reserved.
7187  
7188 Licensed under the Apache License, Version 2.0 (the "License");
7189 you may not use this file except in compliance with the License.
7190 You may obtain a copy of the License at
7191  
7192 http://www.apache.org/licenses/LICENSE-2.0
7193  
7194 Unless required by applicable law or agreed to in writing, software
7195 distributed under the License is distributed on an "AS IS" BASIS,
7196 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7197 See the License for the specific language governing permissions and
7198 limitations under the License.
7199  
7200 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7201 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7202 */
7203  
7204 #ifndef ZOPFLI_ZOPFLI_H_
7205 #define ZOPFLI_ZOPFLI_H_
7206  
7207 #include <stddef.h>
7208 #include <stdlib.h> /* for size_t */
7209  
7210 #ifdef __cplusplus
7211 extern "C" {
7212 #endif
7213  
7214 /*
7215 Options used throughout the program.
7216 */
7217 typedef struct ZopfliOptions {
7218 /* Whether to print output */
7219 int verbose;
7220  
7221 /* Whether to print more detailed output */
7222 int verbose_more;
7223  
7224 /*
7225 Maximum amount of times to rerun forward and backward pass to optimize LZ77
7226 compression cost. Good values: 10, 15 for small files, 5 for files over
7227 several MB in size or it will be too slow.
7228 */
7229 int numiterations;
7230  
7231 /*
7232 If true, splits the data in multiple deflate blocks with optimal choice
7233 for the block boundaries. Block splitting gives better compression. Default:
7234 true (1).
7235 */
7236 int blocksplitting;
7237  
7238 /*
7239 No longer used, left for compatibility.
7240 */
7241 int blocksplittinglast;
7242  
7243 /*
7244 Maximum amount of blocks to split into (0 for unlimited, but this can give
7245 extreme results that hurt compression on some files). Default value: 15.
7246 */
7247 int blocksplittingmax;
7248 } ZopfliOptions;
7249  
7250 /* Initializes options with default values. */
7251 void ZopfliInitOptions(ZopfliOptions* options);
7252  
7253 /* Output format */
7254 typedef enum {
7255 ZOPFLI_FORMAT_GZIP,
7256 ZOPFLI_FORMAT_ZLIB,
7257 ZOPFLI_FORMAT_DEFLATE
7258 } ZopfliFormat;
7259  
7260 /*
7261 Compresses according to the given output format and appends the result to the
7262 output.
7263  
7264 options: global program options
7265 output_type: the output format to use
7266 out: pointer to the dynamic output array to which the result is appended. Must
7267 be freed after use
7268 outsize: pointer to the dynamic output array size
7269 */
7270 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
7271 const unsigned char* in, size_t insize,
7272 unsigned char** out, size_t* outsize);
7273  
7274 #ifdef __cplusplus
7275 } // extern "C"
7276 #endif
7277  
7278 #endif /* ZOPFLI_ZOPFLI_H_ */
7279  
7280  
7281 #ifdef __cplusplus
7282 extern "C" {
7283 #endif
7284  
7285 /*
7286 Compresses according to the gzip specification and append the compressed
7287 result to the output.
7288  
7289 options: global program options
7290 out: pointer to the dynamic output array to which the result is appended. Must
7291 be freed after use.
7292 outsize: pointer to the dynamic output array size.
7293 */
7294 void ZopfliGzipCompress(const ZopfliOptions* options,
7295 const unsigned char* in, size_t insize,
7296 unsigned char** out, size_t* outsize);
7297  
7298 #ifdef __cplusplus
7299 } // extern "C"
7300 #endif
7301  
7302 #endif /* ZOPFLI_GZIP_H_ */
7303  
7304 /* util.h */
7305 /*
7306 Copyright 2011 Google Inc. All Rights Reserved.
7307  
7308 Licensed under the Apache License, Version 2.0 (the "License");
7309 you may not use this file except in compliance with the License.
7310 You may obtain a copy of the License at
7311  
7312 http://www.apache.org/licenses/LICENSE-2.0
7313  
7314 Unless required by applicable law or agreed to in writing, software
7315 distributed under the License is distributed on an "AS IS" BASIS,
7316 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7317 See the License for the specific language governing permissions and
7318 limitations under the License.
7319  
7320 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7321 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7322 */
7323  
7324 /*
7325 Several utilities, including: #defines to try different compression results,
7326 basic deflate specification values and generic program options.
7327 */
7328  
7329 #ifndef ZOPFLI_UTIL_H_
7330 #define ZOPFLI_UTIL_H_
7331  
7332 #include <string.h>
7333 #include <stdlib.h>
7334  
7335 /* Minimum and maximum length that can be encoded in deflate. */
7336 #define ZOPFLI_MAX_MATCH 258
7337 #define ZOPFLI_MIN_MATCH 3
7338  
7339 /* Number of distinct literal/length and distance symbols in DEFLATE */
7340 #define ZOPFLI_NUM_LL 288
7341 #define ZOPFLI_NUM_D 32
7342  
7343 /*
7344 The window size for deflate. Must be a power of two. This should be 32768, the
7345 maximum possible by the deflate spec. Anything less hurts compression more than
7346 speed.
7347 */
7348 #define ZOPFLI_WINDOW_SIZE 32768
7349  
7350 /*
7351 The window mask used to wrap indices into the window. This is why the
7352 window size must be a power of two.
7353 */
7354 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
7355  
7356 /*
7357 A block structure of huge, non-smart, blocks to divide the input into, to allow
7358 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
7359 The whole compression algorithm, including the smarter block splitting, will
7360 be executed independently on each huge block.
7361 Dividing into huge blocks hurts compression, but not much relative to the size.
7362 Set it to 0 to disable master blocks.
7363 */
7364 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
7365  
7366 /*
7367 Used to initialize costs for example
7368 */
7369 #define ZOPFLI_LARGE_FLOAT 1e30
7370  
7371 /*
7372 For longest match cache. max 256. Uses huge amounts of memory but makes it
7373 faster. Uses this many times three bytes per single byte of the input data.
7374 This is so because longest match finding has to find the exact distance
7375 that belongs to each length for the best lz77 strategy.
7376 Good values: e.g. 5, 8.
7377 */
7378 #define ZOPFLI_CACHE_LENGTH 8
7379  
7380 /*
7381 limit the max hash chain hits for this hash value. This has an effect only
7382 on files where the hash value is the same very often. On these files, this
7383 gives worse compression (the value should ideally be 32768, which is the
7384 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
7385 faster on some specific files.
7386 Good value: e.g. 8192.
7387 */
7388 #define ZOPFLI_MAX_CHAIN_HITS 8192
7389  
7390 /*
7391 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
7392 consumes a lot of memory but speeds it up. No effect on compression size.
7393 */
7394 #define ZOPFLI_LONGEST_MATCH_CACHE
7395  
7396 /*
7397 Enable to remember amount of successive identical bytes in the hash chain for
7398 finding longest match
7399 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
7400 This has no effect on the compression result, and enabling it increases speed.
7401 */
7402 #define ZOPFLI_HASH_SAME
7403  
7404 /*
7405 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
7406 best length so far is long enough. This is way faster for files with lots of
7407 identical bytes, on which the compressor is otherwise too slow. Regular files
7408 are unaffected or maybe a tiny bit slower.
7409 This has no effect on the compression result, only on speed.
7410 */
7411 #define ZOPFLI_HASH_SAME_HASH
7412  
7413 /*
7414 Enable this, to avoid slowness for files which are a repetition of the same
7415 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
7416 the compression result.
7417 */
7418 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
7419  
7420 /*
7421 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
7422 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
7423 varies from file to file.
7424 */
7425 #define ZOPFLI_LAZY_MATCHING
7426  
7427 /*
7428 Appends value to dynamically allocated memory, doubling its allocation size
7429 whenever needed.
7430  
7431 value: the value to append, type T
7432 data: pointer to the dynamic array to append to, type T**
7433 size: pointer to the size of the array to append to, type size_t*. This is the
7434 size that you consider the array to be, not the internal allocation size.
7435 Precondition: allocated size of data is at least a power of two greater than or
7436 equal than *size.
7437 */
7438 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
7439 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7440 if (!((*size) & ((*size) - 1))) {\
7441 /*double alloc size if it's a power of two*/\
7442 void** data_void = reinterpret_cast<void**>(data);\
7443 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
7444 : realloc((*data), (*size) * 2 * sizeof(**data));\
7445 }\
7446 (*data)[(*size)] = (value);\
7447 (*size)++;\
7448 }
7449 #else /* C gives problems with strict-aliasing rules for (void**) cast */
7450 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7451 if (!((*size) & ((*size) - 1))) {\
7452 /*double alloc size if it's a power of two*/\
7453 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
7454 : realloc((*data), (*size) * 2 * sizeof(**data));\
7455 }\
7456 (*data)[(*size)] = (value);\
7457 (*size)++;\
7458 }
7459 #endif
7460  
7461  
7462 #endif /* ZOPFLI_UTIL_H_ */
7463  
7464  
7465 #include <stdio.h>
7466  
7467 /* deflate.h */
7468 /*
7469 Copyright 2011 Google Inc. All Rights Reserved.
7470  
7471 Licensed under the Apache License, Version 2.0 (the "License");
7472 you may not use this file except in compliance with the License.
7473 You may obtain a copy of the License at
7474  
7475 http://www.apache.org/licenses/LICENSE-2.0
7476  
7477 Unless required by applicable law or agreed to in writing, software
7478 distributed under the License is distributed on an "AS IS" BASIS,
7479 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7480 See the License for the specific language governing permissions and
7481 limitations under the License.
7482  
7483 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7484 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7485 */
7486  
7487 #ifndef ZOPFLI_DEFLATE_H_
7488 #define ZOPFLI_DEFLATE_H_
7489  
7490 /*
7491 Functions to compress according to the DEFLATE specification, using the
7492 "squeeze" LZ77 compression backend.
7493 */
7494  
7495 /* lz77.h */
7496 /*
7497 Copyright 2011 Google Inc. All Rights Reserved.
7498  
7499 Licensed under the Apache License, Version 2.0 (the "License");
7500 you may not use this file except in compliance with the License.
7501 You may obtain a copy of the License at
7502  
7503 http://www.apache.org/licenses/LICENSE-2.0
7504  
7505 Unless required by applicable law or agreed to in writing, software
7506 distributed under the License is distributed on an "AS IS" BASIS,
7507 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7508 See the License for the specific language governing permissions and
7509 limitations under the License.
7510  
7511 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7512 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7513 */
7514  
7515 /*
7516 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
7517 compression.
7518 */
7519  
7520 #ifndef ZOPFLI_LZ77_H_
7521 #define ZOPFLI_LZ77_H_
7522  
7523 #include <stdlib.h>
7524  
7525 /* cache.h */
7526 /*
7527 Copyright 2011 Google Inc. All Rights Reserved.
7528  
7529 Licensed under the Apache License, Version 2.0 (the "License");
7530 you may not use this file except in compliance with the License.
7531 You may obtain a copy of the License at
7532  
7533 http://www.apache.org/licenses/LICENSE-2.0
7534  
7535 Unless required by applicable law or agreed to in writing, software
7536 distributed under the License is distributed on an "AS IS" BASIS,
7537 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7538 See the License for the specific language governing permissions and
7539 limitations under the License.
7540  
7541 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7542 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7543 */
7544  
7545 /*
7546 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
7547 */
7548  
7549 #ifndef ZOPFLI_CACHE_H_
7550 #define ZOPFLI_CACHE_H_
7551  
7552 /* util.h */
7553 /*
7554 Copyright 2011 Google Inc. All Rights Reserved.
7555  
7556 Licensed under the Apache License, Version 2.0 (the "License");
7557 you may not use this file except in compliance with the License.
7558 You may obtain a copy of the License at
7559  
7560 http://www.apache.org/licenses/LICENSE-2.0
7561  
7562 Unless required by applicable law or agreed to in writing, software
7563 distributed under the License is distributed on an "AS IS" BASIS,
7564 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7565 See the License for the specific language governing permissions and
7566 limitations under the License.
7567  
7568 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7569 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7570 */
7571  
7572 /*
7573 Several utilities, including: #defines to try different compression results,
7574 basic deflate specification values and generic program options.
7575 */
7576  
7577 #ifndef ZOPFLI_UTIL_H_
7578 #define ZOPFLI_UTIL_H_
7579  
7580 #include <string.h>
7581 #include <stdlib.h>
7582  
7583 /* Minimum and maximum length that can be encoded in deflate. */
7584 #define ZOPFLI_MAX_MATCH 258
7585 #define ZOPFLI_MIN_MATCH 3
7586  
7587 /* Number of distinct literal/length and distance symbols in DEFLATE */
7588 #define ZOPFLI_NUM_LL 288
7589 #define ZOPFLI_NUM_D 32
7590  
7591 /*
7592 The window size for deflate. Must be a power of two. This should be 32768, the
7593 maximum possible by the deflate spec. Anything less hurts compression more than
7594 speed.
7595 */
7596 #define ZOPFLI_WINDOW_SIZE 32768
7597  
7598 /*
7599 The window mask used to wrap indices into the window. This is why the
7600 window size must be a power of two.
7601 */
7602 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
7603  
7604 /*
7605 A block structure of huge, non-smart, blocks to divide the input into, to allow
7606 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
7607 The whole compression algorithm, including the smarter block splitting, will
7608 be executed independently on each huge block.
7609 Dividing into huge blocks hurts compression, but not much relative to the size.
7610 Set it to 0 to disable master blocks.
7611 */
7612 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
7613  
7614 /*
7615 Used to initialize costs for example
7616 */
7617 #define ZOPFLI_LARGE_FLOAT 1e30
7618  
7619 /*
7620 For longest match cache. max 256. Uses huge amounts of memory but makes it
7621 faster. Uses this many times three bytes per single byte of the input data.
7622 This is so because longest match finding has to find the exact distance
7623 that belongs to each length for the best lz77 strategy.
7624 Good values: e.g. 5, 8.
7625 */
7626 #define ZOPFLI_CACHE_LENGTH 8
7627  
7628 /*
7629 limit the max hash chain hits for this hash value. This has an effect only
7630 on files where the hash value is the same very often. On these files, this
7631 gives worse compression (the value should ideally be 32768, which is the
7632 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
7633 faster on some specific files.
7634 Good value: e.g. 8192.
7635 */
7636 #define ZOPFLI_MAX_CHAIN_HITS 8192
7637  
7638 /*
7639 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
7640 consumes a lot of memory but speeds it up. No effect on compression size.
7641 */
7642 #define ZOPFLI_LONGEST_MATCH_CACHE
7643  
7644 /*
7645 Enable to remember amount of successive identical bytes in the hash chain for
7646 finding longest match
7647 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
7648 This has no effect on the compression result, and enabling it increases speed.
7649 */
7650 #define ZOPFLI_HASH_SAME
7651  
7652 /*
7653 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
7654 best length so far is long enough. This is way faster for files with lots of
7655 identical bytes, on which the compressor is otherwise too slow. Regular files
7656 are unaffected or maybe a tiny bit slower.
7657 This has no effect on the compression result, only on speed.
7658 */
7659 #define ZOPFLI_HASH_SAME_HASH
7660  
7661 /*
7662 Enable this, to avoid slowness for files which are a repetition of the same
7663 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
7664 the compression result.
7665 */
7666 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
7667  
7668 /*
7669 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
7670 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
7671 varies from file to file.
7672 */
7673 #define ZOPFLI_LAZY_MATCHING
7674  
7675 /*
7676 Appends value to dynamically allocated memory, doubling its allocation size
7677 whenever needed.
7678  
7679 value: the value to append, type T
7680 data: pointer to the dynamic array to append to, type T**
7681 size: pointer to the size of the array to append to, type size_t*. This is the
7682 size that you consider the array to be, not the internal allocation size.
7683 Precondition: allocated size of data is at least a power of two greater than or
7684 equal than *size.
7685 */
7686 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
7687 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7688 if (!((*size) & ((*size) - 1))) {\
7689 /*double alloc size if it's a power of two*/\
7690 void** data_void = reinterpret_cast<void**>(data);\
7691 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
7692 : realloc((*data), (*size) * 2 * sizeof(**data));\
7693 }\
7694 (*data)[(*size)] = (value);\
7695 (*size)++;\
7696 }
7697 #else /* C gives problems with strict-aliasing rules for (void**) cast */
7698 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7699 if (!((*size) & ((*size) - 1))) {\
7700 /*double alloc size if it's a power of two*/\
7701 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
7702 : realloc((*data), (*size) * 2 * sizeof(**data));\
7703 }\
7704 (*data)[(*size)] = (value);\
7705 (*size)++;\
7706 }
7707 #endif
7708  
7709  
7710 #endif /* ZOPFLI_UTIL_H_ */
7711  
7712  
7713 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
7714  
7715 /*
7716 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
7717 values.
7718 This is needed because the squeeze runs will ask these values multiple times for
7719 the same position.
7720 Uses large amounts of memory, since it has to remember the distance belonging
7721 to every possible shorter-than-the-best length (the so called "sublen" array).
7722 */
7723 typedef struct ZopfliLongestMatchCache {
7724 unsigned short* length;
7725 unsigned short* dist;
7726 unsigned char* sublen;
7727 } ZopfliLongestMatchCache;
7728  
7729 /* Initializes the ZopfliLongestMatchCache. */
7730 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
7731  
7732 /* Frees up the memory of the ZopfliLongestMatchCache. */
7733 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
7734  
7735 /* Stores sublen array in the cache. */
7736 void ZopfliSublenToCache(const unsigned short* sublen,
7737 size_t pos, size_t length,
7738 ZopfliLongestMatchCache* lmc);
7739  
7740 /* Extracts sublen array from the cache. */
7741 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
7742 size_t pos, size_t length,
7743 unsigned short* sublen);
7744 /* Returns the length up to which could be stored in the cache. */
7745 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
7746 size_t pos, size_t length);
7747  
7748 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
7749  
7750 #endif /* ZOPFLI_CACHE_H_ */
7751  
7752 /* hash.h */
7753 /*
7754 Copyright 2011 Google Inc. All Rights Reserved.
7755  
7756 Licensed under the Apache License, Version 2.0 (the "License");
7757 you may not use this file except in compliance with the License.
7758 You may obtain a copy of the License at
7759  
7760 http://www.apache.org/licenses/LICENSE-2.0
7761  
7762 Unless required by applicable law or agreed to in writing, software
7763 distributed under the License is distributed on an "AS IS" BASIS,
7764 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7765 See the License for the specific language governing permissions and
7766 limitations under the License.
7767  
7768 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7769 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7770 */
7771  
7772 /*
7773 The hash for ZopfliFindLongestMatch of lz77.c.
7774 */
7775  
7776 #ifndef ZOPFLI_HASH_H_
7777 #define ZOPFLI_HASH_H_
7778  
7779 /* util.h */
7780 /*
7781 Copyright 2011 Google Inc. All Rights Reserved.
7782  
7783 Licensed under the Apache License, Version 2.0 (the "License");
7784 you may not use this file except in compliance with the License.
7785 You may obtain a copy of the License at
7786  
7787 http://www.apache.org/licenses/LICENSE-2.0
7788  
7789 Unless required by applicable law or agreed to in writing, software
7790 distributed under the License is distributed on an "AS IS" BASIS,
7791 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7792 See the License for the specific language governing permissions and
7793 limitations under the License.
7794  
7795 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
7796 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
7797 */
7798  
7799 /*
7800 Several utilities, including: #defines to try different compression results,
7801 basic deflate specification values and generic program options.
7802 */
7803  
7804 #ifndef ZOPFLI_UTIL_H_
7805 #define ZOPFLI_UTIL_H_
7806  
7807 #include <string.h>
7808 #include <stdlib.h>
7809  
7810 /* Minimum and maximum length that can be encoded in deflate. */
7811 #define ZOPFLI_MAX_MATCH 258
7812 #define ZOPFLI_MIN_MATCH 3
7813  
7814 /* Number of distinct literal/length and distance symbols in DEFLATE */
7815 #define ZOPFLI_NUM_LL 288
7816 #define ZOPFLI_NUM_D 32
7817  
7818 /*
7819 The window size for deflate. Must be a power of two. This should be 32768, the
7820 maximum possible by the deflate spec. Anything less hurts compression more than
7821 speed.
7822 */
7823 #define ZOPFLI_WINDOW_SIZE 32768
7824  
7825 /*
7826 The window mask used to wrap indices into the window. This is why the
7827 window size must be a power of two.
7828 */
7829 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
7830  
7831 /*
7832 A block structure of huge, non-smart, blocks to divide the input into, to allow
7833 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
7834 The whole compression algorithm, including the smarter block splitting, will
7835 be executed independently on each huge block.
7836 Dividing into huge blocks hurts compression, but not much relative to the size.
7837 Set it to 0 to disable master blocks.
7838 */
7839 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
7840  
7841 /*
7842 Used to initialize costs for example
7843 */
7844 #define ZOPFLI_LARGE_FLOAT 1e30
7845  
7846 /*
7847 For longest match cache. max 256. Uses huge amounts of memory but makes it
7848 faster. Uses this many times three bytes per single byte of the input data.
7849 This is so because longest match finding has to find the exact distance
7850 that belongs to each length for the best lz77 strategy.
7851 Good values: e.g. 5, 8.
7852 */
7853 #define ZOPFLI_CACHE_LENGTH 8
7854  
7855 /*
7856 limit the max hash chain hits for this hash value. This has an effect only
7857 on files where the hash value is the same very often. On these files, this
7858 gives worse compression (the value should ideally be 32768, which is the
7859 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
7860 faster on some specific files.
7861 Good value: e.g. 8192.
7862 */
7863 #define ZOPFLI_MAX_CHAIN_HITS 8192
7864  
7865 /*
7866 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
7867 consumes a lot of memory but speeds it up. No effect on compression size.
7868 */
7869 #define ZOPFLI_LONGEST_MATCH_CACHE
7870  
7871 /*
7872 Enable to remember amount of successive identical bytes in the hash chain for
7873 finding longest match
7874 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
7875 This has no effect on the compression result, and enabling it increases speed.
7876 */
7877 #define ZOPFLI_HASH_SAME
7878  
7879 /*
7880 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
7881 best length so far is long enough. This is way faster for files with lots of
7882 identical bytes, on which the compressor is otherwise too slow. Regular files
7883 are unaffected or maybe a tiny bit slower.
7884 This has no effect on the compression result, only on speed.
7885 */
7886 #define ZOPFLI_HASH_SAME_HASH
7887  
7888 /*
7889 Enable this, to avoid slowness for files which are a repetition of the same
7890 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
7891 the compression result.
7892 */
7893 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
7894  
7895 /*
7896 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
7897 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
7898 varies from file to file.
7899 */
7900 #define ZOPFLI_LAZY_MATCHING
7901  
7902 /*
7903 Appends value to dynamically allocated memory, doubling its allocation size
7904 whenever needed.
7905  
7906 value: the value to append, type T
7907 data: pointer to the dynamic array to append to, type T**
7908 size: pointer to the size of the array to append to, type size_t*. This is the
7909 size that you consider the array to be, not the internal allocation size.
7910 Precondition: allocated size of data is at least a power of two greater than or
7911 equal than *size.
7912 */
7913 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
7914 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7915 if (!((*size) & ((*size) - 1))) {\
7916 /*double alloc size if it's a power of two*/\
7917 void** data_void = reinterpret_cast<void**>(data);\
7918 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
7919 : realloc((*data), (*size) * 2 * sizeof(**data));\
7920 }\
7921 (*data)[(*size)] = (value);\
7922 (*size)++;\
7923 }
7924 #else /* C gives problems with strict-aliasing rules for (void**) cast */
7925 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
7926 if (!((*size) & ((*size) - 1))) {\
7927 /*double alloc size if it's a power of two*/\
7928 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
7929 : realloc((*data), (*size) * 2 * sizeof(**data));\
7930 }\
7931 (*data)[(*size)] = (value);\
7932 (*size)++;\
7933 }
7934 #endif
7935  
7936  
7937 #endif /* ZOPFLI_UTIL_H_ */
7938  
7939  
7940 typedef struct ZopfliHash {
7941 int* head; /* Hash value to index of its most recent occurrence. */
7942 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
7943 int* hashval; /* Index to hash value at this index. */
7944 int val; /* Current hash value. */
7945  
7946 #ifdef ZOPFLI_HASH_SAME_HASH
7947 /* Fields with similar purpose as the above hash, but for the second hash with
7948 a value that is calculated differently. */
7949 int* head2; /* Hash value to index of its most recent occurrence. */
7950 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
7951 int* hashval2; /* Index to hash value at this index. */
7952 int val2; /* Current hash value. */
7953 #endif
7954  
7955 #ifdef ZOPFLI_HASH_SAME
7956 unsigned short* same; /* Amount of repetitions of same byte after this .*/
7957 #endif
7958 } ZopfliHash;
7959  
7960 /* Allocates ZopfliHash memory. */
7961 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
7962  
7963 /* Resets all fields of ZopfliHash. */
7964 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
7965  
7966 /* Frees ZopfliHash memory. */
7967 void ZopfliCleanHash(ZopfliHash* h);
7968  
7969 /*
7970 Updates the hash values based on the current position in the array. All calls
7971 to this must be made for consecutive bytes.
7972 */
7973 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
7974 ZopfliHash* h);
7975  
7976 /*
7977 Prepopulates hash:
7978 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
7979 correctly.
7980 */
7981 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
7982 ZopfliHash* h);
7983  
7984 #endif /* ZOPFLI_HASH_H_ */
7985  
7986 /* zopfli.h */
7987 /*
7988 Copyright 2011 Google Inc. All Rights Reserved.
7989  
7990 Licensed under the Apache License, Version 2.0 (the "License");
7991 you may not use this file except in compliance with the License.
7992 You may obtain a copy of the License at
7993  
7994 http://www.apache.org/licenses/LICENSE-2.0
7995  
7996 Unless required by applicable law or agreed to in writing, software
7997 distributed under the License is distributed on an "AS IS" BASIS,
7998 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
7999 See the License for the specific language governing permissions and
8000 limitations under the License.
8001  
8002 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8003 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8004 */
8005  
8006 #ifndef ZOPFLI_ZOPFLI_H_
8007 #define ZOPFLI_ZOPFLI_H_
8008  
8009 #include <stddef.h>
8010 #include <stdlib.h> /* for size_t */
8011  
8012 #ifdef __cplusplus
8013 extern "C" {
8014 #endif
8015  
8016 /*
8017 Options used throughout the program.
8018 */
8019 typedef struct ZopfliOptions {
8020 /* Whether to print output */
8021 int verbose;
8022  
8023 /* Whether to print more detailed output */
8024 int verbose_more;
8025  
8026 /*
8027 Maximum amount of times to rerun forward and backward pass to optimize LZ77
8028 compression cost. Good values: 10, 15 for small files, 5 for files over
8029 several MB in size or it will be too slow.
8030 */
8031 int numiterations;
8032  
8033 /*
8034 If true, splits the data in multiple deflate blocks with optimal choice
8035 for the block boundaries. Block splitting gives better compression. Default:
8036 true (1).
8037 */
8038 int blocksplitting;
8039  
8040 /*
8041 No longer used, left for compatibility.
8042 */
8043 int blocksplittinglast;
8044  
8045 /*
8046 Maximum amount of blocks to split into (0 for unlimited, but this can give
8047 extreme results that hurt compression on some files). Default value: 15.
8048 */
8049 int blocksplittingmax;
8050 } ZopfliOptions;
8051  
8052 /* Initializes options with default values. */
8053 void ZopfliInitOptions(ZopfliOptions* options);
8054  
8055 /* Output format */
8056 typedef enum {
8057 ZOPFLI_FORMAT_GZIP,
8058 ZOPFLI_FORMAT_ZLIB,
8059 ZOPFLI_FORMAT_DEFLATE
8060 } ZopfliFormat;
8061  
8062 /*
8063 Compresses according to the given output format and appends the result to the
8064 output.
8065  
8066 options: global program options
8067 output_type: the output format to use
8068 out: pointer to the dynamic output array to which the result is appended. Must
8069 be freed after use
8070 outsize: pointer to the dynamic output array size
8071 */
8072 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
8073 const unsigned char* in, size_t insize,
8074 unsigned char** out, size_t* outsize);
8075  
8076 #ifdef __cplusplus
8077 } // extern "C"
8078 #endif
8079  
8080 #endif /* ZOPFLI_ZOPFLI_H_ */
8081  
8082  
8083 /*
8084 Stores lit/length and dist pairs for LZ77.
8085 Parameter litlens: Contains the literal symbols or length values.
8086 Parameter dists: Contains the distances. A value is 0 to indicate that there is
8087 no dist and the corresponding litlens value is a literal instead of a length.
8088 Parameter size: The size of both the litlens and dists arrays.
8089 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
8090 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
8091  
8092 */
8093 typedef struct ZopfliLZ77Store {
8094 unsigned short* litlens; /* Lit or len. */
8095 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
8096 if > 0: length in corresponding litlens, this is the distance. */
8097 size_t size;
8098  
8099 const unsigned char* data; /* original data */
8100 size_t* pos; /* position in data where this LZ77 command begins */
8101  
8102 unsigned short* ll_symbol;
8103 unsigned short* d_symbol;
8104  
8105 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
8106 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
8107 precise histogram at every N symbols, and the rest can be calculated by
8108 looping through the actual symbols of this chunk. */
8109 size_t* ll_counts;
8110 size_t* d_counts;
8111 } ZopfliLZ77Store;
8112  
8113 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
8114 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
8115 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
8116 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
8117 size_t pos, ZopfliLZ77Store* store);
8118 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
8119 ZopfliLZ77Store* target);
8120 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
8121 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
8122 size_t lstart, size_t lend);
8123 /* Gets the histogram of lit/len and dist symbols in the given range, using the
8124 cumulative histograms, so faster than adding one by one for large range. Does
8125 not add the one end symbol of value 256. */
8126 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
8127 size_t lstart, size_t lend,
8128 size_t* ll_counts, size_t* d_counts);
8129  
8130 /*
8131 Some state information for compressing a block.
8132 This is currently a bit under-used (with mainly only the longest match cache),
8133 but is kept for easy future expansion.
8134 */
8135 typedef struct ZopfliBlockState {
8136 const ZopfliOptions* options;
8137  
8138 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
8139 /* Cache for length/distance pairs found so far. */
8140 ZopfliLongestMatchCache* lmc;
8141 #endif
8142  
8143 /* The start (inclusive) and end (not inclusive) of the current block. */
8144 size_t blockstart;
8145 size_t blockend;
8146 } ZopfliBlockState;
8147  
8148 void ZopfliInitBlockState(const ZopfliOptions* options,
8149 size_t blockstart, size_t blockend, int add_lmc,
8150 ZopfliBlockState* s);
8151 void ZopfliCleanBlockState(ZopfliBlockState* s);
8152  
8153 /*
8154 Finds the longest match (length and corresponding distance) for LZ77
8155 compression.
8156 Even when not using "sublen", it can be more efficient to provide an array,
8157 because only then the caching is used.
8158 array: the data
8159 pos: position in the data to find the match for
8160 size: size of the data
8161 limit: limit length to maximum this value (default should be 258). This allows
8162 finding a shorter dist for that length (= less extra bits). Must be
8163 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
8164 sublen: output array of 259 elements, or null. Has, for each length, the
8165 smallest distance required to reach this length. Only 256 of its 259 values
8166 are used, the first 3 are ignored (the shortest length is 3. It is purely
8167 for convenience that the array is made 3 longer).
8168 */
8169 void ZopfliFindLongestMatch(
8170 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
8171 size_t pos, size_t size, size_t limit,
8172 unsigned short* sublen, unsigned short* distance, unsigned short* length);
8173  
8174 /*
8175 Verifies if length and dist are indeed valid, only used for assertion.
8176 */
8177 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
8178 unsigned short dist, unsigned short length);
8179  
8180 /*
8181 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
8182 with the slow but better "squeeze" implementation.
8183 The result is placed in the ZopfliLZ77Store.
8184 If instart is larger than 0, it uses values before instart as starting
8185 dictionary.
8186 */
8187 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
8188 size_t instart, size_t inend,
8189 ZopfliLZ77Store* store, ZopfliHash* h);
8190  
8191 #endif /* ZOPFLI_LZ77_H_ */
8192  
8193 /* zopfli.h */
8194 /*
8195 Copyright 2011 Google Inc. All Rights Reserved.
8196  
8197 Licensed under the Apache License, Version 2.0 (the "License");
8198 you may not use this file except in compliance with the License.
8199 You may obtain a copy of the License at
8200  
8201 http://www.apache.org/licenses/LICENSE-2.0
8202  
8203 Unless required by applicable law or agreed to in writing, software
8204 distributed under the License is distributed on an "AS IS" BASIS,
8205 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8206 See the License for the specific language governing permissions and
8207 limitations under the License.
8208  
8209 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8210 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8211 */
8212  
8213 #ifndef ZOPFLI_ZOPFLI_H_
8214 #define ZOPFLI_ZOPFLI_H_
8215  
8216 #include <stddef.h>
8217 #include <stdlib.h> /* for size_t */
8218  
8219 #ifdef __cplusplus
8220 extern "C" {
8221 #endif
8222  
8223 /*
8224 Options used throughout the program.
8225 */
8226 typedef struct ZopfliOptions {
8227 /* Whether to print output */
8228 int verbose;
8229  
8230 /* Whether to print more detailed output */
8231 int verbose_more;
8232  
8233 /*
8234 Maximum amount of times to rerun forward and backward pass to optimize LZ77
8235 compression cost. Good values: 10, 15 for small files, 5 for files over
8236 several MB in size or it will be too slow.
8237 */
8238 int numiterations;
8239  
8240 /*
8241 If true, splits the data in multiple deflate blocks with optimal choice
8242 for the block boundaries. Block splitting gives better compression. Default:
8243 true (1).
8244 */
8245 int blocksplitting;
8246  
8247 /*
8248 No longer used, left for compatibility.
8249 */
8250 int blocksplittinglast;
8251  
8252 /*
8253 Maximum amount of blocks to split into (0 for unlimited, but this can give
8254 extreme results that hurt compression on some files). Default value: 15.
8255 */
8256 int blocksplittingmax;
8257 } ZopfliOptions;
8258  
8259 /* Initializes options with default values. */
8260 void ZopfliInitOptions(ZopfliOptions* options);
8261  
8262 /* Output format */
8263 typedef enum {
8264 ZOPFLI_FORMAT_GZIP,
8265 ZOPFLI_FORMAT_ZLIB,
8266 ZOPFLI_FORMAT_DEFLATE
8267 } ZopfliFormat;
8268  
8269 /*
8270 Compresses according to the given output format and appends the result to the
8271 output.
8272  
8273 options: global program options
8274 output_type: the output format to use
8275 out: pointer to the dynamic output array to which the result is appended. Must
8276 be freed after use
8277 outsize: pointer to the dynamic output array size
8278 */
8279 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
8280 const unsigned char* in, size_t insize,
8281 unsigned char** out, size_t* outsize);
8282  
8283 #ifdef __cplusplus
8284 } // extern "C"
8285 #endif
8286  
8287 #endif /* ZOPFLI_ZOPFLI_H_ */
8288  
8289  
8290 #ifdef __cplusplus
8291 extern "C" {
8292 #endif
8293  
8294 /*
8295 Compresses according to the deflate specification and append the compressed
8296 result to the output.
8297 This function will usually output multiple deflate blocks. If final is 1, then
8298 the final bit will be set on the last block.
8299  
8300 options: global program options
8301 btype: the deflate block type. Use 2 for best compression.
8302 -0: non compressed blocks (00)
8303 -1: blocks with fixed tree (01)
8304 -2: blocks with dynamic tree (10)
8305 final: whether this is the last section of the input, sets the final bit to the
8306 last deflate block.
8307 in: the input bytes
8308 insize: number of input bytes
8309 bp: bit pointer for the output array. This must initially be 0, and for
8310 consecutive calls must be reused (it can have values from 0-7). This is
8311 because deflate appends blocks as bit-based data, rather than on byte
8312 boundaries.
8313 out: pointer to the dynamic output array to which the result is appended. Must
8314 be freed after use.
8315 outsize: pointer to the dynamic output array size.
8316 */
8317 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
8318 const unsigned char* in, size_t insize,
8319 unsigned char* bp, unsigned char** out, size_t* outsize);
8320  
8321 /*
8322 Like ZopfliDeflate, but allows to specify start and end byte with instart and
8323 inend. Only that part is compressed, but earlier bytes are still used for the
8324 back window.
8325 */
8326 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
8327 const unsigned char* in, size_t instart, size_t inend,
8328 unsigned char* bp, unsigned char** out,
8329 size_t* outsize);
8330  
8331 /*
8332 Calculates block size in bits.
8333 litlens: lz77 lit/lengths
8334 dists: ll77 distances
8335 lstart: start of block
8336 lend: end of block (not inclusive)
8337 */
8338 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
8339 size_t lstart, size_t lend, int btype);
8340  
8341 /*
8342 Calculates block size in bits, automatically using the best btype.
8343 */
8344 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
8345 size_t lstart, size_t lend);
8346  
8347 #ifdef __cplusplus
8348 } // extern "C"
8349 #endif
8350  
8351 #endif /* ZOPFLI_DEFLATE_H_ */
8352  
8353  
8354 /* CRC polynomial: 0xedb88320 */
8355 static const unsigned long crc32_table[256] = {
8356 0u, 1996959894u, 3993919788u, 2567524794u, 124634137u, 1886057615u,
8357 3915621685u, 2657392035u, 249268274u, 2044508324u, 3772115230u, 2547177864u,
8358 162941995u, 2125561021u, 3887607047u, 2428444049u, 498536548u, 1789927666u,
8359 4089016648u, 2227061214u, 450548861u, 1843258603u, 4107580753u, 2211677639u,
8360 325883990u, 1684777152u, 4251122042u, 2321926636u, 335633487u, 1661365465u,
8361 4195302755u, 2366115317u, 997073096u, 1281953886u, 3579855332u, 2724688242u,
8362 1006888145u, 1258607687u, 3524101629u, 2768942443u, 901097722u, 1119000684u,
8363 3686517206u, 2898065728u, 853044451u, 1172266101u, 3705015759u, 2882616665u,
8364 651767980u, 1373503546u, 3369554304u, 3218104598u, 565507253u, 1454621731u,
8365 3485111705u, 3099436303u, 671266974u, 1594198024u, 3322730930u, 2970347812u,
8366 795835527u, 1483230225u, 3244367275u, 3060149565u, 1994146192u, 31158534u,
8367 2563907772u, 4023717930u, 1907459465u, 112637215u, 2680153253u, 3904427059u,
8368 2013776290u, 251722036u, 2517215374u, 3775830040u, 2137656763u, 141376813u,
8369 2439277719u, 3865271297u, 1802195444u, 476864866u, 2238001368u, 4066508878u,
8370 1812370925u, 453092731u, 2181625025u, 4111451223u, 1706088902u, 314042704u,
8371 2344532202u, 4240017532u, 1658658271u, 366619977u, 2362670323u, 4224994405u,
8372 1303535960u, 984961486u, 2747007092u, 3569037538u, 1256170817u, 1037604311u,
8373 2765210733u, 3554079995u, 1131014506u, 879679996u, 2909243462u, 3663771856u,
8374 1141124467u, 855842277u, 2852801631u, 3708648649u, 1342533948u, 654459306u,
8375 3188396048u, 3373015174u, 1466479909u, 544179635u, 3110523913u, 3462522015u,
8376 1591671054u, 702138776u, 2966460450u, 3352799412u, 1504918807u, 783551873u,
8377 3082640443u, 3233442989u, 3988292384u, 2596254646u, 62317068u, 1957810842u,
8378 3939845945u, 2647816111u, 81470997u, 1943803523u, 3814918930u, 2489596804u,
8379 225274430u, 2053790376u, 3826175755u, 2466906013u, 167816743u, 2097651377u,
8380 4027552580u, 2265490386u, 503444072u, 1762050814u, 4150417245u, 2154129355u,
8381 426522225u, 1852507879u, 4275313526u, 2312317920u, 282753626u, 1742555852u,
8382 4189708143u, 2394877945u, 397917763u, 1622183637u, 3604390888u, 2714866558u,
8383 953729732u, 1340076626u, 3518719985u, 2797360999u, 1068828381u, 1219638859u,
8384 3624741850u, 2936675148u, 906185462u, 1090812512u, 3747672003u, 2825379669u,
8385 829329135u, 1181335161u, 3412177804u, 3160834842u, 628085408u, 1382605366u,
8386 3423369109u, 3138078467u, 570562233u, 1426400815u, 3317316542u, 2998733608u,
8387 733239954u, 1555261956u, 3268935591u, 3050360625u, 752459403u, 1541320221u,
8388 2607071920u, 3965973030u, 1969922972u, 40735498u, 2617837225u, 3943577151u,
8389 1913087877u, 83908371u, 2512341634u, 3803740692u, 2075208622u, 213261112u,
8390 2463272603u, 3855990285u, 2094854071u, 198958881u, 2262029012u, 4057260610u,
8391 1759359992u, 534414190u, 2176718541u, 4139329115u, 1873836001u, 414664567u,
8392 2282248934u, 4279200368u, 1711684554u, 285281116u, 2405801727u, 4167216745u,
8393 1634467795u, 376229701u, 2685067896u, 3608007406u, 1308918612u, 956543938u,
8394 2808555105u, 3495958263u, 1231636301u, 1047427035u, 2932959818u, 3654703836u,
8395 1088359270u, 936918000u, 2847714899u, 3736837829u, 1202900863u, 817233897u,
8396 3183342108u, 3401237130u, 1404277552u, 615818150u, 3134207493u, 3453421203u,
8397 1423857449u, 601450431u, 3009837614u, 3294710456u, 1567103746u, 711928724u,
8398 3020668471u, 3272380065u, 1510334235u, 755167117u
8399 };
8400  
8401 /* Returns the CRC32 */
8402 3 static unsigned long CRC(const unsigned char* data, size_t size) {
8403 3 unsigned long result = 0xffffffffu;
8404 9183 ▶ for (; size > 0; size--) {
Branch: 1 branch
BranchTrueFalseTotal
for (; size > 0; size--) { 9180 3 9183
8405 9180 result = crc32_table[(result ^ *(data++)) & 0xff] ^ (result >> 8);
8406 }
8407 3 return result ^ 0xffffffffu;
8408 }
8409  
8410 /* Compresses the data according to the gzip specification, RFC 1952. */
8411 3 void ZopfliGzipCompress(const ZopfliOptions* options,
8412 const unsigned char* in, size_t insize,
8413 unsigned char** out, size_t* outsize) {
8414 3 unsigned long crcvalue = CRC(in, insize);
8415 3 unsigned char bp = 0;
8416  
8417 3 ▶ ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */ 3 0 3
ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */ 3 0 3
8418 3 ▶ ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */ 3 0 3
ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */ 0 3 3
8419 3 ▶ ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */ 3 0 3
ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */ 0 3 3
8420 3 ▶ ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */ 0 3 3
ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */ 0 0 0
8421 /* MTIME */
8422 3 ▶ ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, out, outsize); 3 0 3
ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
8423 3 ▶ ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA(0, out, outsize); 0 0 0
8424 3 ▶ ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA(0, out, outsize); 0 0 0
8425 3 ▶ ZOPFLI_APPEND_DATA(0, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA(0, out, outsize); 0 0 0
8426  
8427 3 ▶ ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */ 3 0 3
ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */ 0 3 3
8428 3 ▶ ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */ 0 3 3
ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */ 0 0 0
8429  
8430 3 ZopfliDeflate(options, 2 /* Dynamic block */, 1,
8431 in, insize, &bp, out, outsize);
8432  
8433 /* CRC */
8434 3 ▶ ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize); 0 0 0
8435 3 ▶ ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize); 0 0 0
8436 3 ▶ ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize); 0 0 0
8437 3 ▶ ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize); 0 0 0
8438  
8439 /* ISIZE */
8440 3 ▶ ZOPFLI_APPEND_DATA(insize % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(insize % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA(insize % 256, out, outsize); 0 0 0
8441 3 ▶ ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize); 0 0 0
8442 3 ▶ ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize); 0 0 0
8443 3 ▶ ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize); 0 3 3
ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize); 0 0 0
8444  
8445 3 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 3 3
8446 0 fprintf(stderr,
8447 "Original Size: %d, Gzip: %d, Compression: %f%% Removed\n",
8448 0 (int)insize, (int)*outsize,
8449 0 100.0 * (double)(insize - *outsize) / (double)insize);
8450 }
8451 3 }
8452 /*
8453 Copyright 2011 Google Inc. All Rights Reserved.
8454  
8455 Licensed under the Apache License, Version 2.0 (the "License");
8456 you may not use this file except in compliance with the License.
8457 You may obtain a copy of the License at
8458  
8459 http://www.apache.org/licenses/LICENSE-2.0
8460  
8461 Unless required by applicable law or agreed to in writing, software
8462 distributed under the License is distributed on an "AS IS" BASIS,
8463 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8464 See the License for the specific language governing permissions and
8465 limitations under the License.
8466  
8467 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8468 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8469 */
8470  
8471 /* hash.h */
8472 /*
8473 Copyright 2011 Google Inc. All Rights Reserved.
8474  
8475 Licensed under the Apache License, Version 2.0 (the "License");
8476 you may not use this file except in compliance with the License.
8477 You may obtain a copy of the License at
8478  
8479 http://www.apache.org/licenses/LICENSE-2.0
8480  
8481 Unless required by applicable law or agreed to in writing, software
8482 distributed under the License is distributed on an "AS IS" BASIS,
8483 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8484 See the License for the specific language governing permissions and
8485 limitations under the License.
8486  
8487 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8488 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8489 */
8490  
8491 /*
8492 The hash for ZopfliFindLongestMatch of lz77.c.
8493 */
8494  
8495 #ifndef ZOPFLI_HASH_H_
8496 #define ZOPFLI_HASH_H_
8497  
8498 /* util.h */
8499 /*
8500 Copyright 2011 Google Inc. All Rights Reserved.
8501  
8502 Licensed under the Apache License, Version 2.0 (the "License");
8503 you may not use this file except in compliance with the License.
8504 You may obtain a copy of the License at
8505  
8506 http://www.apache.org/licenses/LICENSE-2.0
8507  
8508 Unless required by applicable law or agreed to in writing, software
8509 distributed under the License is distributed on an "AS IS" BASIS,
8510 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8511 See the License for the specific language governing permissions and
8512 limitations under the License.
8513  
8514 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8515 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8516 */
8517  
8518 /*
8519 Several utilities, including: #defines to try different compression results,
8520 basic deflate specification values and generic program options.
8521 */
8522  
8523 #ifndef ZOPFLI_UTIL_H_
8524 #define ZOPFLI_UTIL_H_
8525  
8526 #include <string.h>
8527 #include <stdlib.h>
8528  
8529 /* Minimum and maximum length that can be encoded in deflate. */
8530 #define ZOPFLI_MAX_MATCH 258
8531 #define ZOPFLI_MIN_MATCH 3
8532  
8533 /* Number of distinct literal/length and distance symbols in DEFLATE */
8534 #define ZOPFLI_NUM_LL 288
8535 #define ZOPFLI_NUM_D 32
8536  
8537 /*
8538 The window size for deflate. Must be a power of two. This should be 32768, the
8539 maximum possible by the deflate spec. Anything less hurts compression more than
8540 speed.
8541 */
8542 #define ZOPFLI_WINDOW_SIZE 32768
8543  
8544 /*
8545 The window mask used to wrap indices into the window. This is why the
8546 window size must be a power of two.
8547 */
8548 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
8549  
8550 /*
8551 A block structure of huge, non-smart, blocks to divide the input into, to allow
8552 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
8553 The whole compression algorithm, including the smarter block splitting, will
8554 be executed independently on each huge block.
8555 Dividing into huge blocks hurts compression, but not much relative to the size.
8556 Set it to 0 to disable master blocks.
8557 */
8558 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
8559  
8560 /*
8561 Used to initialize costs for example
8562 */
8563 #define ZOPFLI_LARGE_FLOAT 1e30
8564  
8565 /*
8566 For longest match cache. max 256. Uses huge amounts of memory but makes it
8567 faster. Uses this many times three bytes per single byte of the input data.
8568 This is so because longest match finding has to find the exact distance
8569 that belongs to each length for the best lz77 strategy.
8570 Good values: e.g. 5, 8.
8571 */
8572 #define ZOPFLI_CACHE_LENGTH 8
8573  
8574 /*
8575 limit the max hash chain hits for this hash value. This has an effect only
8576 on files where the hash value is the same very often. On these files, this
8577 gives worse compression (the value should ideally be 32768, which is the
8578 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
8579 faster on some specific files.
8580 Good value: e.g. 8192.
8581 */
8582 #define ZOPFLI_MAX_CHAIN_HITS 8192
8583  
8584 /*
8585 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
8586 consumes a lot of memory but speeds it up. No effect on compression size.
8587 */
8588 #define ZOPFLI_LONGEST_MATCH_CACHE
8589  
8590 /*
8591 Enable to remember amount of successive identical bytes in the hash chain for
8592 finding longest match
8593 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
8594 This has no effect on the compression result, and enabling it increases speed.
8595 */
8596 #define ZOPFLI_HASH_SAME
8597  
8598 /*
8599 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
8600 best length so far is long enough. This is way faster for files with lots of
8601 identical bytes, on which the compressor is otherwise too slow. Regular files
8602 are unaffected or maybe a tiny bit slower.
8603 This has no effect on the compression result, only on speed.
8604 */
8605 #define ZOPFLI_HASH_SAME_HASH
8606  
8607 /*
8608 Enable this, to avoid slowness for files which are a repetition of the same
8609 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
8610 the compression result.
8611 */
8612 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
8613  
8614 /*
8615 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
8616 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
8617 varies from file to file.
8618 */
8619 #define ZOPFLI_LAZY_MATCHING
8620  
8621 /*
8622 Appends value to dynamically allocated memory, doubling its allocation size
8623 whenever needed.
8624  
8625 value: the value to append, type T
8626 data: pointer to the dynamic array to append to, type T**
8627 size: pointer to the size of the array to append to, type size_t*. This is the
8628 size that you consider the array to be, not the internal allocation size.
8629 Precondition: allocated size of data is at least a power of two greater than or
8630 equal than *size.
8631 */
8632 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
8633 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
8634 if (!((*size) & ((*size) - 1))) {\
8635 /*double alloc size if it's a power of two*/\
8636 void** data_void = reinterpret_cast<void**>(data);\
8637 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
8638 : realloc((*data), (*size) * 2 * sizeof(**data));\
8639 }\
8640 (*data)[(*size)] = (value);\
8641 (*size)++;\
8642 }
8643 #else /* C gives problems with strict-aliasing rules for (void**) cast */
8644 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
8645 if (!((*size) & ((*size) - 1))) {\
8646 /*double alloc size if it's a power of two*/\
8647 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
8648 : realloc((*data), (*size) * 2 * sizeof(**data));\
8649 }\
8650 (*data)[(*size)] = (value);\
8651 (*size)++;\
8652 }
8653 #endif
8654  
8655  
8656 #endif /* ZOPFLI_UTIL_H_ */
8657  
8658  
8659 typedef struct ZopfliHash {
8660 int* head; /* Hash value to index of its most recent occurrence. */
8661 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
8662 int* hashval; /* Index to hash value at this index. */
8663 int val; /* Current hash value. */
8664  
8665 #ifdef ZOPFLI_HASH_SAME_HASH
8666 /* Fields with similar purpose as the above hash, but for the second hash with
8667 a value that is calculated differently. */
8668 int* head2; /* Hash value to index of its most recent occurrence. */
8669 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
8670 int* hashval2; /* Index to hash value at this index. */
8671 int val2; /* Current hash value. */
8672 #endif
8673  
8674 #ifdef ZOPFLI_HASH_SAME
8675 unsigned short* same; /* Amount of repetitions of same byte after this .*/
8676 #endif
8677 } ZopfliHash;
8678  
8679 /* Allocates ZopfliHash memory. */
8680 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
8681  
8682 /* Resets all fields of ZopfliHash. */
8683 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
8684  
8685 /* Frees ZopfliHash memory. */
8686 void ZopfliCleanHash(ZopfliHash* h);
8687  
8688 /*
8689 Updates the hash values based on the current position in the array. All calls
8690 to this must be made for consecutive bytes.
8691 */
8692 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
8693 ZopfliHash* h);
8694  
8695 /*
8696 Prepopulates hash:
8697 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
8698 correctly.
8699 */
8700 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
8701 ZopfliHash* h);
8702  
8703 #endif /* ZOPFLI_HASH_H_ */
8704  
8705  
8706 #include <assert.h>
8707 #include <stdio.h>
8708 #include <stdlib.h>
8709  
8710 #define HASH_SHIFT 5
8711 #define HASH_MASK 32767
8712  
8713 11 void ZopfliAllocHash(size_t window_size, ZopfliHash* h) {
8714 11 h->head = (int*)malloc(sizeof(*h->head) * 65536);
8715 11 h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
8716 11 h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
8717  
8718 #ifdef ZOPFLI_HASH_SAME
8719 11 h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
8720 #endif
8721  
8722 #ifdef ZOPFLI_HASH_SAME_HASH
8723 11 h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
8724 11 h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
8725 11 h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
8726 #endif
8727 11 }
8728  
8729 135 void ZopfliResetHash(size_t window_size, ZopfliHash* h) {
8730 size_t i;
8731  
8732 135 h->val = 0;
8733 8847495 ▶ for (i = 0; i < 65536; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 65536; i++) { 8847360 135 8847495
8734 8847360 h->head[i] = -1; /* -1 indicates no head so far. */
8735 }
8736 4423815 ▶ for (i = 0; i < window_size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < window_size; i++) { 4423680 135 4423815
8737 4423680 h->prev[i] = i; /* If prev[j] == j, then prev[j] is uninitialized. */
8738 4423680 h->hashval[i] = -1;
8739 }
8740  
8741 #ifdef ZOPFLI_HASH_SAME
8742 4423815 ▶ for (i = 0; i < window_size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < window_size; i++) { 4423680 135 4423815
8743 4423680 h->same[i] = 0;
8744 }
8745 #endif
8746  
8747 #ifdef ZOPFLI_HASH_SAME_HASH
8748 135 h->val2 = 0;
8749 8847495 ▶ for (i = 0; i < 65536; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 65536; i++) { 8847360 135 8847495
8750 8847360 h->head2[i] = -1;
8751 }
8752 4423815 ▶ for (i = 0; i < window_size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < window_size; i++) { 4423680 135 4423815
8753 4423680 h->prev2[i] = i;
8754 4423680 h->hashval2[i] = -1;
8755 }
8756 #endif
8757 135 }
8758  
8759 11 void ZopfliCleanHash(ZopfliHash* h) {
8760 11 free(h->head);
8761 11 free(h->prev);
8762 11 free(h->hashval);
8763  
8764 #ifdef ZOPFLI_HASH_SAME_HASH
8765 11 free(h->head2);
8766 11 free(h->prev2);
8767 11 free(h->hashval2);
8768 #endif
8769  
8770 #ifdef ZOPFLI_HASH_SAME
8771 11 free(h->same);
8772 #endif
8773 11 }
8774  
8775 /*
8776 Update the sliding hash value with the given byte. All calls to this function
8777 must be made on consecutive input characters. Since the hash value exists out
8778 of multiple input bytes, a few warmups with this function are needed initially.
8779 */
8780 312687 static void UpdateHashValue(ZopfliHash* h, unsigned char c) {
8781 312687 h->val = (((h->val) << HASH_SHIFT) ^ (c)) & HASH_MASK;
8782 312687 }
8783  
8784 312417 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
8785 ZopfliHash* h) {
8786 312417 unsigned short hpos = pos & ZOPFLI_WINDOW_MASK;
8787 #ifdef ZOPFLI_HASH_SAME
8788 312417 size_t amount = 0;
8789 #endif
8790  
8791 624564 ▶ UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ?
Branch: 1 branch
BranchTrueFalseTotal
UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ? 312147 270 312417
8792 312147 array[pos + ZOPFLI_MIN_MATCH - 1] : 0);
8793 312417 h->hashval[hpos] = h->val;
8794 312417 ▶ if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) {
Branch: 2 branches
BranchTrueFalseTotal
if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) { 310998 1419 312417
if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) { 310998 0 310998
8795 310998 h->prev[hpos] = h->head[h->val];
8796 }
8797 1419 else h->prev[hpos] = hpos;
8798 312417 h->head[h->val] = hpos;
8799  
8800 #ifdef ZOPFLI_HASH_SAME
8801 /* Update "same". */
8802 312417 ▶ if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) { 0 312417 312417
8803 0 amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1;
8804 }
8805 312417 while (pos + amount + 1 < end &&
8806 312417 ▶ array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) {
Branch: 3 branches
BranchTrueFalseTotal
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) { 312282 135 312417
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) { 0 312282 312282
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) { 0 0 0
8807 0 amount++;
8808 }
8809 312417 h->same[hpos] = amount;
8810 #endif
8811  
8812 #ifdef ZOPFLI_HASH_SAME_HASH
8813 312417 h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val;
8814 312417 h->hashval2[hpos] = h->val2;
8815 312417 ▶ if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) {
Branch: 2 branches
BranchTrueFalseTotal
if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) { 310998 1419 312417
if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) { 310998 0 310998
8816 310998 h->prev2[hpos] = h->head2[h->val2];
8817 }
8818 1419 else h->prev2[hpos] = hpos;
8819 312417 h->head2[h->val2] = hpos;
8820 #endif
8821 312417 }
8822  
8823 135 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
8824 ZopfliHash* h) {
8825 135 UpdateHashValue(h, array[pos + 0]);
8826 135 ▶ if (pos + 1 < end) UpdateHashValue(h, array[pos + 1]);
Branch: 1 branch
BranchTrueFalseTotal
if (pos + 1 < end) UpdateHashValue(h, array[pos + 1]); 135 0 135
8827 135 }
8828 /*
8829 Copyright 2011 Google Inc. All Rights Reserved.
8830  
8831 Licensed under the Apache License, Version 2.0 (the "License");
8832 you may not use this file except in compliance with the License.
8833 You may obtain a copy of the License at
8834  
8835 http://www.apache.org/licenses/LICENSE-2.0
8836  
8837 Unless required by applicable law or agreed to in writing, software
8838 distributed under the License is distributed on an "AS IS" BASIS,
8839 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8840 See the License for the specific language governing permissions and
8841 limitations under the License.
8842  
8843 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8844 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8845 */
8846  
8847 /*
8848 Bounded package merge algorithm, based on the paper
8849 "A Fast and Space-Economical Algorithm for Length-Limited Coding
8850 Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
8851 */
8852  
8853 /* katajainen.h */
8854 /*
8855 Copyright 2011 Google Inc. All Rights Reserved.
8856  
8857 Licensed under the Apache License, Version 2.0 (the "License");
8858 you may not use this file except in compliance with the License.
8859 You may obtain a copy of the License at
8860  
8861 http://www.apache.org/licenses/LICENSE-2.0
8862  
8863 Unless required by applicable law or agreed to in writing, software
8864 distributed under the License is distributed on an "AS IS" BASIS,
8865 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
8866 See the License for the specific language governing permissions and
8867 limitations under the License.
8868  
8869 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
8870 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
8871 */
8872  
8873 #ifndef ZOPFLI_KATAJAINEN_H_
8874 #define ZOPFLI_KATAJAINEN_H_
8875  
8876 #include <string.h>
8877  
8878 /*
8879 Outputs minimum-redundancy length-limited code bitlengths for symbols with the
8880 given counts. The bitlengths are limited by maxbits.
8881  
8882 The output is tailored for DEFLATE: symbols that never occur, get a bit length
8883 of 0, and if only a single symbol occurs at least once, its bitlength will be 1,
8884 and not 0 as would theoretically be needed for a single symbol.
8885  
8886 frequencies: The amount of occurrences of each symbol.
8887 n: The amount of symbols.
8888 maxbits: Maximum bit length, inclusive.
8889 bitlengths: Output, the bitlengths for the symbol prefix codes.
8890 return: 0 for OK, non-0 for error.
8891 */
8892 int ZopfliLengthLimitedCodeLengths(
8893 const size_t* frequencies, int n, int maxbits, unsigned* bitlengths);
8894  
8895 #endif /* ZOPFLI_KATAJAINEN_H_ */
8896  
8897 #include <assert.h>
8898 #include <stdlib.h>
8899 #include <limits.h>
8900  
8901 typedef struct Node Node;
8902  
8903 /*
8904 Nodes forming chains. Also used to represent leaves.
8905 */
8906 struct Node {
8907 size_t weight; /* Total weight (symbol count) of this chain. */
8908 Node* tail; /* Previous node(s) of this chain, or 0 if none. */
8909 int count; /* Leaf symbol index, or number of leaves before this chain. */
8910 };
8911  
8912 /*
8913 Memory pool for nodes.
8914 */
8915 typedef struct NodePool {
8916 Node* next; /* Pointer to a free node in the pool. */
8917 } NodePool;
8918  
8919 /*
8920 Initializes a chain node with the given values and marks it as in use.
8921 */
8922 95672 static void InitNode(size_t weight, int count, Node* tail, Node* node) {
8923 95672 node->weight = weight;
8924 95672 node->count = count;
8925 95672 node->tail = tail;
8926 95672 }
8927  
8928 /*
8929 Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
8930 new chain is, depending on the weights, a leaf or a combination of two chains
8931 from the previous list.
8932 lists: The lists of chains.
8933 maxbits: Number of lists.
8934 leaves: The leaves, one per symbol.
8935 numsymbols: Number of leaves.
8936 pool: the node memory pool.
8937 index: The index of the list in which a new chain or leaf is required.
8938 */
8939 92965 static void BoundaryPM(Node* (*lists)[2], Node* leaves, int numsymbols,
8940 NodePool* pool, int index) {
8941 Node* newchain;
8942 Node* oldchain;
8943 92965 int lastcount = lists[index][1]->count; /* Count of last chain of list. */
8944  
8945 92965 ▶ if (index == 0 && lastcount >= numsymbols) return;
Branch: 2 branches
BranchTrueFalseTotal
if (index == 0 && lastcount >= numsymbols) return; 14148 78817 92965
if (index == 0 && lastcount >= numsymbols) return; 5539 8609 14148
8946  
8947 87426 newchain = pool->next++;
8948 87426 oldchain = lists[index][1];
8949  
8950 /* These are set up before the recursive calls below, so that there is a list
8951 pointing to the new node, to let the garbage collection know it's in use. */
8952 87426 lists[index][0] = oldchain;
8953 87426 lists[index][1] = newchain;
8954  
8955 87426 ▶ if (index == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (index == 0) { 8609 78817 87426
8956 /* New leaf node in list 0. */
8957 8609 InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain);
8958 } else {
8959 78817 size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
8960 78817 ▶ if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
Branch: 2 branches
BranchTrueFalseTotal
if (lastcount < numsymbols && sum > leaves[lastcount].weight) { 69496 9321 78817
if (lastcount < numsymbols && sum > leaves[lastcount].weight) { 43037 26459 69496
8961 /* New leaf inserted in list, so count is incremented. */
8962 43037 InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail,
8963 newchain);
8964 } else {
8965 35780 InitNode(sum, lastcount, lists[index - 1][1], newchain);
8966 /* Two lookahead chains of previous list used up, create new ones. */
8967 35780 BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
8968 35780 BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
8969 }
8970 }
8971 }
8972  
8973 4123 static void BoundaryPMFinal(Node* (*lists)[2],
8974 Node* leaves, int numsymbols, NodePool* pool, int index) {
8975 4123 int lastcount = lists[index][1]->count; /* Count of last chain of list. */
8976  
8977 4123 size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
8978  
8979 4123 ▶ if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
Branch: 2 branches
BranchTrueFalseTotal
if (lastcount < numsymbols && sum > leaves[lastcount].weight) { 2274 1849 4123
if (lastcount < numsymbols && sum > leaves[lastcount].weight) { 2274 0 2274
8980 2274 Node* newchain = pool->next;
8981 2274 Node* oldchain = lists[index][1]->tail;
8982  
8983 2274 lists[index][1] = newchain;
8984 2274 newchain->count = lastcount + 1;
8985 2274 newchain->tail = oldchain;
8986 } else {
8987 1849 lists[index][1]->tail = lists[index - 1][1];
8988 }
8989 4123 }
8990  
8991 /*
8992 Initializes each list with as lookahead chains the two leaves with lowest
8993 weights.
8994 */
8995 4123 static void InitLists(
8996 NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
8997 int i;
8998 4123 Node* node0 = pool->next++;
8999 4123 Node* node1 = pool->next++;
9000 4123 InitNode(leaves[0].weight, 1, 0, node0);
9001 4123 InitNode(leaves[1].weight, 2, 0, node1);
9002 20728 ▶ for (i = 0; i < maxbits; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < maxbits; i++) { 16605 4123 20728
9003 16605 lists[i][0] = node0;
9004 16605 lists[i][1] = node1;
9005 }
9006 4123 }
9007  
9008 /*
9009 Converts result of boundary package-merge to the bitlengths. The result in the
9010 last chain of the last list contains the amount of active leaves in each list.
9011 chain: Chain to extract the bit length from (last chain from last list).
9012 */
9013 4123 static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
9014 4123 int counts[16] = {0};
9015 4123 unsigned end = 16;
9016 4123 unsigned ptr = 15;
9017 4123 unsigned value = 1;
9018 Node* node;
9019 int val;
9020  
9021 17551 ▶ for (node = chain; node; node = node->tail) {
Branch: 1 branch
BranchTrueFalseTotal
for (node = chain; node; node = node->tail) { 13428 4123 17551
9022 13428 counts[--end] = node->count;
9023 }
9024  
9025 4123 val = counts[15];
9026 17551 ▶ while (ptr >= end) {
Branch: 1 branch
BranchTrueFalseTotal
while (ptr >= end) { 13428 4123 17551
9027 34438 ▶ for (; val > counts[ptr - 1]; val--) {
Branch: 1 branch
BranchTrueFalseTotal
for (; val > counts[ptr - 1]; val--) { 21010 13428 34438
9028 21010 bitlengths[leaves[val - 1].count] = value;
9029 }
9030 13428 ptr--;
9031 13428 value++;
9032 }
9033 4123 }
9034  
9035 /*
9036 Comparator for sorting the leaves. Has the function signature for qsort.
9037 */
9038 29750 static int LeafComparator(const void* a, const void* b) {
9039 29750 return ((const Node*)a)->weight - ((const Node*)b)->weight;
9040 }
9041  
9042 5229 int ZopfliLengthLimitedCodeLengths(
9043 const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) {
9044 NodePool pool;
9045 int i;
9046 5229 int numsymbols = 0; /* Amount of symbols with frequency > 0. */
9047 int numBoundaryPMRuns;
9048 Node* nodes;
9049  
9050 /* Array of lists of chains. Each list requires only two lookahead chains at
9051 a time, so each list is a array of two Node*'s. */
9052 Node* (*lists)[2];
9053  
9054 /* One leaf per symbol. Only numsymbols leaves will be used. */
9055 5229 Node* leaves = (Node*)malloc(n * sizeof(*leaves));
9056  
9057 /* Initialize all bitlengths at 0. */
9058 251784 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 246555 5229 251784
9059 246555 bitlengths[i] = 0;
9060 }
9061  
9062 /* Count used symbols and place them in the leaves. */
9063 251784 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 246555 5229 251784
9064 246555 ▶ if (frequencies[i]) {
Branch: 1 branch
BranchTrueFalseTotal
if (frequencies[i]) { 22612 223943 246555
9065 22612 leaves[numsymbols].weight = frequencies[i];
9066 22612 leaves[numsymbols].count = i; /* Index of symbol this leaf represents. */
9067 22612 numsymbols++;
9068 }
9069 }
9070  
9071 /* Check special cases and error conditions. */
9072 5229 ▶ if ((1 << maxbits) < numsymbols) {
Branch: 1 branch
BranchTrueFalseTotal
if ((1 << maxbits) < numsymbols) { 0 5229 5229
9073 0 free(leaves);
9074 0 return 1; /* Error, too few maxbits to represent symbols. */
9075 }
9076 5229 ▶ if (numsymbols == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (numsymbols == 0) { 88 5141 5229
9077 88 free(leaves);
9078 88 return 0; /* No symbols at all. OK. */
9079 }
9080 5141 ▶ if (numsymbols == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (numsymbols == 1) { 434 4707 5141
9081 434 bitlengths[leaves[0].count] = 1;
9082 434 free(leaves);
9083 434 return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */
9084 }
9085 4707 ▶ if (numsymbols == 2) {
Branch: 1 branch
BranchTrueFalseTotal
if (numsymbols == 2) { 584 4123 4707
9086 584 bitlengths[leaves[0].count]++;
9087 584 bitlengths[leaves[1].count]++;
9088 584 free(leaves);
9089 584 return 0;
9090 }
9091  
9092 /* Sort the leaves from lightest to heaviest. Add count into the same
9093 variable for stable sorting. */
9094 25133 ▶ for (i = 0; i < numsymbols; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < numsymbols; i++) { 21010 4123 25133
9095 21010 ▶ if (leaves[i].weight >=
Branch: 1 branch
BranchTrueFalseTotal
if (leaves[i].weight >= 0 21010 21010
9096 ((size_t)1 << (sizeof(leaves[0].weight) * CHAR_BIT - 9))) {
9097 0 free(leaves);
9098 0 return 1; /* Error, we need 9 bits for the count. */
9099 }
9100 21010 leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count;
9101 }
9102 4123 qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
9103 25133 ▶ for (i = 0; i < numsymbols; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < numsymbols; i++) { 21010 4123 25133
9104 21010 leaves[i].weight >>= 9;
9105 }
9106  
9107 4123 ▶ if (numsymbols - 1 < maxbits) {
Branch: 1 branch
BranchTrueFalseTotal
if (numsymbols - 1 < maxbits) { 4012 111 4123
9108 4012 maxbits = numsymbols - 1;
9109 }
9110  
9111 /* Initialize node memory pool. */
9112 4123 nodes = (Node*)malloc(maxbits * 2 * numsymbols * sizeof(Node));
9113 4123 pool.next = nodes;
9114  
9115 4123 lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
9116 4123 InitLists(&pool, leaves, maxbits, lists);
9117  
9118 /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
9119 are already created in the initialization. Each BoundaryPM run creates one. */
9120 4123 numBoundaryPMRuns = 2 * numsymbols - 4;
9121 25528 ▶ for (i = 0; i < numBoundaryPMRuns - 1; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < numBoundaryPMRuns - 1; i++) { 21405 4123 25528
9122 21405 BoundaryPM(lists, leaves, numsymbols, &pool, maxbits - 1);
9123 }
9124 4123 BoundaryPMFinal(lists, leaves, numsymbols, &pool, maxbits - 1);
9125  
9126 4123 ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
9127  
9128 4123 free(lists);
9129 4123 free(leaves);
9130 4123 free(nodes);
9131 4123 return 0; /* OK. */
9132 }
9133 /*
9134 Copyright 2011 Google Inc. All Rights Reserved.
9135  
9136 Licensed under the Apache License, Version 2.0 (the "License");
9137 you may not use this file except in compliance with the License.
9138 You may obtain a copy of the License at
9139  
9140 http://www.apache.org/licenses/LICENSE-2.0
9141  
9142 Unless required by applicable law or agreed to in writing, software
9143 distributed under the License is distributed on an "AS IS" BASIS,
9144 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9145 See the License for the specific language governing permissions and
9146 limitations under the License.
9147  
9148 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9149 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9150 */
9151  
9152 /* lz77.h */
9153 /*
9154 Copyright 2011 Google Inc. All Rights Reserved.
9155  
9156 Licensed under the Apache License, Version 2.0 (the "License");
9157 you may not use this file except in compliance with the License.
9158 You may obtain a copy of the License at
9159  
9160 http://www.apache.org/licenses/LICENSE-2.0
9161  
9162 Unless required by applicable law or agreed to in writing, software
9163 distributed under the License is distributed on an "AS IS" BASIS,
9164 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9165 See the License for the specific language governing permissions and
9166 limitations under the License.
9167  
9168 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9169 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9170 */
9171  
9172 /*
9173 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
9174 compression.
9175 */
9176  
9177 #ifndef ZOPFLI_LZ77_H_
9178 #define ZOPFLI_LZ77_H_
9179  
9180 #include <stdlib.h>
9181  
9182 /* cache.h */
9183 /*
9184 Copyright 2011 Google Inc. All Rights Reserved.
9185  
9186 Licensed under the Apache License, Version 2.0 (the "License");
9187 you may not use this file except in compliance with the License.
9188 You may obtain a copy of the License at
9189  
9190 http://www.apache.org/licenses/LICENSE-2.0
9191  
9192 Unless required by applicable law or agreed to in writing, software
9193 distributed under the License is distributed on an "AS IS" BASIS,
9194 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9195 See the License for the specific language governing permissions and
9196 limitations under the License.
9197  
9198 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9199 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9200 */
9201  
9202 /*
9203 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
9204 */
9205  
9206 #ifndef ZOPFLI_CACHE_H_
9207 #define ZOPFLI_CACHE_H_
9208  
9209 /* util.h */
9210 /*
9211 Copyright 2011 Google Inc. All Rights Reserved.
9212  
9213 Licensed under the Apache License, Version 2.0 (the "License");
9214 you may not use this file except in compliance with the License.
9215 You may obtain a copy of the License at
9216  
9217 http://www.apache.org/licenses/LICENSE-2.0
9218  
9219 Unless required by applicable law or agreed to in writing, software
9220 distributed under the License is distributed on an "AS IS" BASIS,
9221 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9222 See the License for the specific language governing permissions and
9223 limitations under the License.
9224  
9225 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9226 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9227 */
9228  
9229 /*
9230 Several utilities, including: #defines to try different compression results,
9231 basic deflate specification values and generic program options.
9232 */
9233  
9234 #ifndef ZOPFLI_UTIL_H_
9235 #define ZOPFLI_UTIL_H_
9236  
9237 #include <string.h>
9238 #include <stdlib.h>
9239  
9240 /* Minimum and maximum length that can be encoded in deflate. */
9241 #define ZOPFLI_MAX_MATCH 258
9242 #define ZOPFLI_MIN_MATCH 3
9243  
9244 /* Number of distinct literal/length and distance symbols in DEFLATE */
9245 #define ZOPFLI_NUM_LL 288
9246 #define ZOPFLI_NUM_D 32
9247  
9248 /*
9249 The window size for deflate. Must be a power of two. This should be 32768, the
9250 maximum possible by the deflate spec. Anything less hurts compression more than
9251 speed.
9252 */
9253 #define ZOPFLI_WINDOW_SIZE 32768
9254  
9255 /*
9256 The window mask used to wrap indices into the window. This is why the
9257 window size must be a power of two.
9258 */
9259 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
9260  
9261 /*
9262 A block structure of huge, non-smart, blocks to divide the input into, to allow
9263 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
9264 The whole compression algorithm, including the smarter block splitting, will
9265 be executed independently on each huge block.
9266 Dividing into huge blocks hurts compression, but not much relative to the size.
9267 Set it to 0 to disable master blocks.
9268 */
9269 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
9270  
9271 /*
9272 Used to initialize costs for example
9273 */
9274 #define ZOPFLI_LARGE_FLOAT 1e30
9275  
9276 /*
9277 For longest match cache. max 256. Uses huge amounts of memory but makes it
9278 faster. Uses this many times three bytes per single byte of the input data.
9279 This is so because longest match finding has to find the exact distance
9280 that belongs to each length for the best lz77 strategy.
9281 Good values: e.g. 5, 8.
9282 */
9283 #define ZOPFLI_CACHE_LENGTH 8
9284  
9285 /*
9286 limit the max hash chain hits for this hash value. This has an effect only
9287 on files where the hash value is the same very often. On these files, this
9288 gives worse compression (the value should ideally be 32768, which is the
9289 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
9290 faster on some specific files.
9291 Good value: e.g. 8192.
9292 */
9293 #define ZOPFLI_MAX_CHAIN_HITS 8192
9294  
9295 /*
9296 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
9297 consumes a lot of memory but speeds it up. No effect on compression size.
9298 */
9299 #define ZOPFLI_LONGEST_MATCH_CACHE
9300  
9301 /*
9302 Enable to remember amount of successive identical bytes in the hash chain for
9303 finding longest match
9304 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
9305 This has no effect on the compression result, and enabling it increases speed.
9306 */
9307 #define ZOPFLI_HASH_SAME
9308  
9309 /*
9310 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
9311 best length so far is long enough. This is way faster for files with lots of
9312 identical bytes, on which the compressor is otherwise too slow. Regular files
9313 are unaffected or maybe a tiny bit slower.
9314 This has no effect on the compression result, only on speed.
9315 */
9316 #define ZOPFLI_HASH_SAME_HASH
9317  
9318 /*
9319 Enable this, to avoid slowness for files which are a repetition of the same
9320 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
9321 the compression result.
9322 */
9323 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
9324  
9325 /*
9326 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
9327 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
9328 varies from file to file.
9329 */
9330 #define ZOPFLI_LAZY_MATCHING
9331  
9332 /*
9333 Appends value to dynamically allocated memory, doubling its allocation size
9334 whenever needed.
9335  
9336 value: the value to append, type T
9337 data: pointer to the dynamic array to append to, type T**
9338 size: pointer to the size of the array to append to, type size_t*. This is the
9339 size that you consider the array to be, not the internal allocation size.
9340 Precondition: allocated size of data is at least a power of two greater than or
9341 equal than *size.
9342 */
9343 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
9344 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
9345 if (!((*size) & ((*size) - 1))) {\
9346 /*double alloc size if it's a power of two*/\
9347 void** data_void = reinterpret_cast<void**>(data);\
9348 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
9349 : realloc((*data), (*size) * 2 * sizeof(**data));\
9350 }\
9351 (*data)[(*size)] = (value);\
9352 (*size)++;\
9353 }
9354 #else /* C gives problems with strict-aliasing rules for (void**) cast */
9355 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
9356 if (!((*size) & ((*size) - 1))) {\
9357 /*double alloc size if it's a power of two*/\
9358 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
9359 : realloc((*data), (*size) * 2 * sizeof(**data));\
9360 }\
9361 (*data)[(*size)] = (value);\
9362 (*size)++;\
9363 }
9364 #endif
9365  
9366  
9367 #endif /* ZOPFLI_UTIL_H_ */
9368  
9369  
9370 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
9371  
9372 /*
9373 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
9374 values.
9375 This is needed because the squeeze runs will ask these values multiple times for
9376 the same position.
9377 Uses large amounts of memory, since it has to remember the distance belonging
9378 to every possible shorter-than-the-best length (the so called "sublen" array).
9379 */
9380 typedef struct ZopfliLongestMatchCache {
9381 unsigned short* length;
9382 unsigned short* dist;
9383 unsigned char* sublen;
9384 } ZopfliLongestMatchCache;
9385  
9386 /* Initializes the ZopfliLongestMatchCache. */
9387 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
9388  
9389 /* Frees up the memory of the ZopfliLongestMatchCache. */
9390 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
9391  
9392 /* Stores sublen array in the cache. */
9393 void ZopfliSublenToCache(const unsigned short* sublen,
9394 size_t pos, size_t length,
9395 ZopfliLongestMatchCache* lmc);
9396  
9397 /* Extracts sublen array from the cache. */
9398 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
9399 size_t pos, size_t length,
9400 unsigned short* sublen);
9401 /* Returns the length up to which could be stored in the cache. */
9402 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
9403 size_t pos, size_t length);
9404  
9405 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
9406  
9407 #endif /* ZOPFLI_CACHE_H_ */
9408  
9409 /* hash.h */
9410 /*
9411 Copyright 2011 Google Inc. All Rights Reserved.
9412  
9413 Licensed under the Apache License, Version 2.0 (the "License");
9414 you may not use this file except in compliance with the License.
9415 You may obtain a copy of the License at
9416  
9417 http://www.apache.org/licenses/LICENSE-2.0
9418  
9419 Unless required by applicable law or agreed to in writing, software
9420 distributed under the License is distributed on an "AS IS" BASIS,
9421 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9422 See the License for the specific language governing permissions and
9423 limitations under the License.
9424  
9425 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9426 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9427 */
9428  
9429 /*
9430 The hash for ZopfliFindLongestMatch of lz77.c.
9431 */
9432  
9433 #ifndef ZOPFLI_HASH_H_
9434 #define ZOPFLI_HASH_H_
9435  
9436 /* util.h */
9437 /*
9438 Copyright 2011 Google Inc. All Rights Reserved.
9439  
9440 Licensed under the Apache License, Version 2.0 (the "License");
9441 you may not use this file except in compliance with the License.
9442 You may obtain a copy of the License at
9443  
9444 http://www.apache.org/licenses/LICENSE-2.0
9445  
9446 Unless required by applicable law or agreed to in writing, software
9447 distributed under the License is distributed on an "AS IS" BASIS,
9448 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9449 See the License for the specific language governing permissions and
9450 limitations under the License.
9451  
9452 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9453 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9454 */
9455  
9456 /*
9457 Several utilities, including: #defines to try different compression results,
9458 basic deflate specification values and generic program options.
9459 */
9460  
9461 #ifndef ZOPFLI_UTIL_H_
9462 #define ZOPFLI_UTIL_H_
9463  
9464 #include <string.h>
9465 #include <stdlib.h>
9466  
9467 /* Minimum and maximum length that can be encoded in deflate. */
9468 #define ZOPFLI_MAX_MATCH 258
9469 #define ZOPFLI_MIN_MATCH 3
9470  
9471 /* Number of distinct literal/length and distance symbols in DEFLATE */
9472 #define ZOPFLI_NUM_LL 288
9473 #define ZOPFLI_NUM_D 32
9474  
9475 /*
9476 The window size for deflate. Must be a power of two. This should be 32768, the
9477 maximum possible by the deflate spec. Anything less hurts compression more than
9478 speed.
9479 */
9480 #define ZOPFLI_WINDOW_SIZE 32768
9481  
9482 /*
9483 The window mask used to wrap indices into the window. This is why the
9484 window size must be a power of two.
9485 */
9486 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
9487  
9488 /*
9489 A block structure of huge, non-smart, blocks to divide the input into, to allow
9490 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
9491 The whole compression algorithm, including the smarter block splitting, will
9492 be executed independently on each huge block.
9493 Dividing into huge blocks hurts compression, but not much relative to the size.
9494 Set it to 0 to disable master blocks.
9495 */
9496 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
9497  
9498 /*
9499 Used to initialize costs for example
9500 */
9501 #define ZOPFLI_LARGE_FLOAT 1e30
9502  
9503 /*
9504 For longest match cache. max 256. Uses huge amounts of memory but makes it
9505 faster. Uses this many times three bytes per single byte of the input data.
9506 This is so because longest match finding has to find the exact distance
9507 that belongs to each length for the best lz77 strategy.
9508 Good values: e.g. 5, 8.
9509 */
9510 #define ZOPFLI_CACHE_LENGTH 8
9511  
9512 /*
9513 limit the max hash chain hits for this hash value. This has an effect only
9514 on files where the hash value is the same very often. On these files, this
9515 gives worse compression (the value should ideally be 32768, which is the
9516 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
9517 faster on some specific files.
9518 Good value: e.g. 8192.
9519 */
9520 #define ZOPFLI_MAX_CHAIN_HITS 8192
9521  
9522 /*
9523 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
9524 consumes a lot of memory but speeds it up. No effect on compression size.
9525 */
9526 #define ZOPFLI_LONGEST_MATCH_CACHE
9527  
9528 /*
9529 Enable to remember amount of successive identical bytes in the hash chain for
9530 finding longest match
9531 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
9532 This has no effect on the compression result, and enabling it increases speed.
9533 */
9534 #define ZOPFLI_HASH_SAME
9535  
9536 /*
9537 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
9538 best length so far is long enough. This is way faster for files with lots of
9539 identical bytes, on which the compressor is otherwise too slow. Regular files
9540 are unaffected or maybe a tiny bit slower.
9541 This has no effect on the compression result, only on speed.
9542 */
9543 #define ZOPFLI_HASH_SAME_HASH
9544  
9545 /*
9546 Enable this, to avoid slowness for files which are a repetition of the same
9547 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
9548 the compression result.
9549 */
9550 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
9551  
9552 /*
9553 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
9554 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
9555 varies from file to file.
9556 */
9557 #define ZOPFLI_LAZY_MATCHING
9558  
9559 /*
9560 Appends value to dynamically allocated memory, doubling its allocation size
9561 whenever needed.
9562  
9563 value: the value to append, type T
9564 data: pointer to the dynamic array to append to, type T**
9565 size: pointer to the size of the array to append to, type size_t*. This is the
9566 size that you consider the array to be, not the internal allocation size.
9567 Precondition: allocated size of data is at least a power of two greater than or
9568 equal than *size.
9569 */
9570 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
9571 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
9572 if (!((*size) & ((*size) - 1))) {\
9573 /*double alloc size if it's a power of two*/\
9574 void** data_void = reinterpret_cast<void**>(data);\
9575 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
9576 : realloc((*data), (*size) * 2 * sizeof(**data));\
9577 }\
9578 (*data)[(*size)] = (value);\
9579 (*size)++;\
9580 }
9581 #else /* C gives problems with strict-aliasing rules for (void**) cast */
9582 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
9583 if (!((*size) & ((*size) - 1))) {\
9584 /*double alloc size if it's a power of two*/\
9585 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
9586 : realloc((*data), (*size) * 2 * sizeof(**data));\
9587 }\
9588 (*data)[(*size)] = (value);\
9589 (*size)++;\
9590 }
9591 #endif
9592  
9593  
9594 #endif /* ZOPFLI_UTIL_H_ */
9595  
9596  
9597 typedef struct ZopfliHash {
9598 int* head; /* Hash value to index of its most recent occurrence. */
9599 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
9600 int* hashval; /* Index to hash value at this index. */
9601 int val; /* Current hash value. */
9602  
9603 #ifdef ZOPFLI_HASH_SAME_HASH
9604 /* Fields with similar purpose as the above hash, but for the second hash with
9605 a value that is calculated differently. */
9606 int* head2; /* Hash value to index of its most recent occurrence. */
9607 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
9608 int* hashval2; /* Index to hash value at this index. */
9609 int val2; /* Current hash value. */
9610 #endif
9611  
9612 #ifdef ZOPFLI_HASH_SAME
9613 unsigned short* same; /* Amount of repetitions of same byte after this .*/
9614 #endif
9615 } ZopfliHash;
9616  
9617 /* Allocates ZopfliHash memory. */
9618 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
9619  
9620 /* Resets all fields of ZopfliHash. */
9621 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
9622  
9623 /* Frees ZopfliHash memory. */
9624 void ZopfliCleanHash(ZopfliHash* h);
9625  
9626 /*
9627 Updates the hash values based on the current position in the array. All calls
9628 to this must be made for consecutive bytes.
9629 */
9630 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
9631 ZopfliHash* h);
9632  
9633 /*
9634 Prepopulates hash:
9635 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
9636 correctly.
9637 */
9638 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
9639 ZopfliHash* h);
9640  
9641 #endif /* ZOPFLI_HASH_H_ */
9642  
9643 /* zopfli.h */
9644 /*
9645 Copyright 2011 Google Inc. All Rights Reserved.
9646  
9647 Licensed under the Apache License, Version 2.0 (the "License");
9648 you may not use this file except in compliance with the License.
9649 You may obtain a copy of the License at
9650  
9651 http://www.apache.org/licenses/LICENSE-2.0
9652  
9653 Unless required by applicable law or agreed to in writing, software
9654 distributed under the License is distributed on an "AS IS" BASIS,
9655 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9656 See the License for the specific language governing permissions and
9657 limitations under the License.
9658  
9659 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9660 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9661 */
9662  
9663 #ifndef ZOPFLI_ZOPFLI_H_
9664 #define ZOPFLI_ZOPFLI_H_
9665  
9666 #include <stddef.h>
9667 #include <stdlib.h> /* for size_t */
9668  
9669 #ifdef __cplusplus
9670 extern "C" {
9671 #endif
9672  
9673 /*
9674 Options used throughout the program.
9675 */
9676 typedef struct ZopfliOptions {
9677 /* Whether to print output */
9678 int verbose;
9679  
9680 /* Whether to print more detailed output */
9681 int verbose_more;
9682  
9683 /*
9684 Maximum amount of times to rerun forward and backward pass to optimize LZ77
9685 compression cost. Good values: 10, 15 for small files, 5 for files over
9686 several MB in size or it will be too slow.
9687 */
9688 int numiterations;
9689  
9690 /*
9691 If true, splits the data in multiple deflate blocks with optimal choice
9692 for the block boundaries. Block splitting gives better compression. Default:
9693 true (1).
9694 */
9695 int blocksplitting;
9696  
9697 /*
9698 No longer used, left for compatibility.
9699 */
9700 int blocksplittinglast;
9701  
9702 /*
9703 Maximum amount of blocks to split into (0 for unlimited, but this can give
9704 extreme results that hurt compression on some files). Default value: 15.
9705 */
9706 int blocksplittingmax;
9707 } ZopfliOptions;
9708  
9709 /* Initializes options with default values. */
9710 void ZopfliInitOptions(ZopfliOptions* options);
9711  
9712 /* Output format */
9713 typedef enum {
9714 ZOPFLI_FORMAT_GZIP,
9715 ZOPFLI_FORMAT_ZLIB,
9716 ZOPFLI_FORMAT_DEFLATE
9717 } ZopfliFormat;
9718  
9719 /*
9720 Compresses according to the given output format and appends the result to the
9721 output.
9722  
9723 options: global program options
9724 output_type: the output format to use
9725 out: pointer to the dynamic output array to which the result is appended. Must
9726 be freed after use
9727 outsize: pointer to the dynamic output array size
9728 */
9729 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
9730 const unsigned char* in, size_t insize,
9731 unsigned char** out, size_t* outsize);
9732  
9733 #ifdef __cplusplus
9734 } // extern "C"
9735 #endif
9736  
9737 #endif /* ZOPFLI_ZOPFLI_H_ */
9738  
9739  
9740 /*
9741 Stores lit/length and dist pairs for LZ77.
9742 Parameter litlens: Contains the literal symbols or length values.
9743 Parameter dists: Contains the distances. A value is 0 to indicate that there is
9744 no dist and the corresponding litlens value is a literal instead of a length.
9745 Parameter size: The size of both the litlens and dists arrays.
9746 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
9747 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
9748  
9749 */
9750 typedef struct ZopfliLZ77Store {
9751 unsigned short* litlens; /* Lit or len. */
9752 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
9753 if > 0: length in corresponding litlens, this is the distance. */
9754 size_t size;
9755  
9756 const unsigned char* data; /* original data */
9757 size_t* pos; /* position in data where this LZ77 command begins */
9758  
9759 unsigned short* ll_symbol;
9760 unsigned short* d_symbol;
9761  
9762 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
9763 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
9764 precise histogram at every N symbols, and the rest can be calculated by
9765 looping through the actual symbols of this chunk. */
9766 size_t* ll_counts;
9767 size_t* d_counts;
9768 } ZopfliLZ77Store;
9769  
9770 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
9771 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
9772 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
9773 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
9774 size_t pos, ZopfliLZ77Store* store);
9775 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
9776 ZopfliLZ77Store* target);
9777 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
9778 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
9779 size_t lstart, size_t lend);
9780 /* Gets the histogram of lit/len and dist symbols in the given range, using the
9781 cumulative histograms, so faster than adding one by one for large range. Does
9782 not add the one end symbol of value 256. */
9783 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
9784 size_t lstart, size_t lend,
9785 size_t* ll_counts, size_t* d_counts);
9786  
9787 /*
9788 Some state information for compressing a block.
9789 This is currently a bit under-used (with mainly only the longest match cache),
9790 but is kept for easy future expansion.
9791 */
9792 typedef struct ZopfliBlockState {
9793 const ZopfliOptions* options;
9794  
9795 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
9796 /* Cache for length/distance pairs found so far. */
9797 ZopfliLongestMatchCache* lmc;
9798 #endif
9799  
9800 /* The start (inclusive) and end (not inclusive) of the current block. */
9801 size_t blockstart;
9802 size_t blockend;
9803 } ZopfliBlockState;
9804  
9805 void ZopfliInitBlockState(const ZopfliOptions* options,
9806 size_t blockstart, size_t blockend, int add_lmc,
9807 ZopfliBlockState* s);
9808 void ZopfliCleanBlockState(ZopfliBlockState* s);
9809  
9810 /*
9811 Finds the longest match (length and corresponding distance) for LZ77
9812 compression.
9813 Even when not using "sublen", it can be more efficient to provide an array,
9814 because only then the caching is used.
9815 array: the data
9816 pos: position in the data to find the match for
9817 size: size of the data
9818 limit: limit length to maximum this value (default should be 258). This allows
9819 finding a shorter dist for that length (= less extra bits). Must be
9820 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
9821 sublen: output array of 259 elements, or null. Has, for each length, the
9822 smallest distance required to reach this length. Only 256 of its 259 values
9823 are used, the first 3 are ignored (the shortest length is 3. It is purely
9824 for convenience that the array is made 3 longer).
9825 */
9826 void ZopfliFindLongestMatch(
9827 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
9828 size_t pos, size_t size, size_t limit,
9829 unsigned short* sublen, unsigned short* distance, unsigned short* length);
9830  
9831 /*
9832 Verifies if length and dist are indeed valid, only used for assertion.
9833 */
9834 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
9835 unsigned short dist, unsigned short length);
9836  
9837 /*
9838 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
9839 with the slow but better "squeeze" implementation.
9840 The result is placed in the ZopfliLZ77Store.
9841 If instart is larger than 0, it uses values before instart as starting
9842 dictionary.
9843 */
9844 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
9845 size_t instart, size_t inend,
9846 ZopfliLZ77Store* store, ZopfliHash* h);
9847  
9848 #endif /* ZOPFLI_LZ77_H_ */
9849  
9850 /* symbols.h */
9851 /*
9852 Copyright 2016 Google Inc. All Rights Reserved.
9853  
9854 Licensed under the Apache License, Version 2.0 (the "License");
9855 you may not use this file except in compliance with the License.
9856 You may obtain a copy of the License at
9857  
9858 http://www.apache.org/licenses/LICENSE-2.0
9859  
9860 Unless required by applicable law or agreed to in writing, software
9861 distributed under the License is distributed on an "AS IS" BASIS,
9862 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9863 See the License for the specific language governing permissions and
9864 limitations under the License.
9865  
9866 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
9867 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
9868 */
9869  
9870 /*
9871 Utilities for using the lz77 symbols of the deflate spec.
9872 */
9873  
9874 #ifndef ZOPFLI_SYMBOLS_H_
9875 #define ZOPFLI_SYMBOLS_H_
9876  
9877 /* __has_builtin available in clang */
9878 #ifdef __has_builtin
9879 # if __has_builtin(__builtin_clz)
9880 # define ZOPFLI_HAS_BUILTIN_CLZ
9881 # endif
9882 /* __builtin_clz available beginning with GCC 3.4 */
9883 #elif __GNUC__ * 100 + __GNUC_MINOR__ >= 304
9884 # define ZOPFLI_HAS_BUILTIN_CLZ
9885 #endif
9886  
9887 /* Gets the amount of extra bits for the given dist, cfr. the DEFLATE spec. */
9888 static int ZopfliGetDistExtraBits(int dist) {
9889 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
9890 if (dist < 5) return 0;
9891 return (31 ^ __builtin_clz(dist - 1)) - 1; /* log2(dist - 1) - 1 */
9892 #else
9893 if (dist < 5) return 0;
9894 else if (dist < 9) return 1;
9895 else if (dist < 17) return 2;
9896 else if (dist < 33) return 3;
9897 else if (dist < 65) return 4;
9898 else if (dist < 129) return 5;
9899 else if (dist < 257) return 6;
9900 else if (dist < 513) return 7;
9901 else if (dist < 1025) return 8;
9902 else if (dist < 2049) return 9;
9903 else if (dist < 4097) return 10;
9904 else if (dist < 8193) return 11;
9905 else if (dist < 16385) return 12;
9906 else return 13;
9907 #endif
9908 }
9909  
9910 /* Gets value of the extra bits for the given dist, cfr. the DEFLATE spec. */
9911 static int ZopfliGetDistExtraBitsValue(int dist) {
9912 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
9913 if (dist < 5) {
9914 return 0;
9915 } else {
9916 int l = 31 ^ __builtin_clz(dist - 1); /* log2(dist - 1) */
9917 return (dist - (1 + (1 << l))) & ((1 << (l - 1)) - 1);
9918 }
9919 #else
9920 if (dist < 5) return 0;
9921 else if (dist < 9) return (dist - 5) & 1;
9922 else if (dist < 17) return (dist - 9) & 3;
9923 else if (dist < 33) return (dist - 17) & 7;
9924 else if (dist < 65) return (dist - 33) & 15;
9925 else if (dist < 129) return (dist - 65) & 31;
9926 else if (dist < 257) return (dist - 129) & 63;
9927 else if (dist < 513) return (dist - 257) & 127;
9928 else if (dist < 1025) return (dist - 513) & 255;
9929 else if (dist < 2049) return (dist - 1025) & 511;
9930 else if (dist < 4097) return (dist - 2049) & 1023;
9931 else if (dist < 8193) return (dist - 4097) & 2047;
9932 else if (dist < 16385) return (dist - 8193) & 4095;
9933 else return (dist - 16385) & 8191;
9934 #endif
9935 }
9936  
9937 /* Gets the symbol for the given dist, cfr. the DEFLATE spec. */
9938 static int ZopfliGetDistSymbol(int dist) {
9939 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
9940 if (dist < 5) {
9941 return dist - 1;
9942 } else {
9943 int l = (31 ^ __builtin_clz(dist - 1)); /* log2(dist - 1) */
9944 int r = ((dist - 1) >> (l - 1)) & 1;
9945 return l * 2 + r;
9946 }
9947 #else
9948 if (dist < 193) {
9949 if (dist < 13) { /* dist 0..13. */
9950 if (dist < 5) return dist - 1;
9951 else if (dist < 7) return 4;
9952 else if (dist < 9) return 5;
9953 else return 6;
9954 } else { /* dist 13..193. */
9955 if (dist < 17) return 7;
9956 else if (dist < 25) return 8;
9957 else if (dist < 33) return 9;
9958 else if (dist < 49) return 10;
9959 else if (dist < 65) return 11;
9960 else if (dist < 97) return 12;
9961 else if (dist < 129) return 13;
9962 else return 14;
9963 }
9964 } else {
9965 if (dist < 2049) { /* dist 193..2049. */
9966 if (dist < 257) return 15;
9967 else if (dist < 385) return 16;
9968 else if (dist < 513) return 17;
9969 else if (dist < 769) return 18;
9970 else if (dist < 1025) return 19;
9971 else if (dist < 1537) return 20;
9972 else return 21;
9973 } else { /* dist 2049..32768. */
9974 if (dist < 3073) return 22;
9975 else if (dist < 4097) return 23;
9976 else if (dist < 6145) return 24;
9977 else if (dist < 8193) return 25;
9978 else if (dist < 12289) return 26;
9979 else if (dist < 16385) return 27;
9980 else if (dist < 24577) return 28;
9981 else return 29;
9982 }
9983 }
9984 #endif
9985 }
9986  
9987 /* Gets the amount of extra bits for the given length, cfr. the DEFLATE spec. */
9988 static int ZopfliGetLengthExtraBits(int l) {
9989 static const int table[259] = {
9990 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
9991 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
9992 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
9993 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
9994 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
9995 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
9996 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
9997 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
9998 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
9999 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10000 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10001 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10002 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10003 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10004 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
10005 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0
10006 };
10007 return table[l];
10008 }
10009  
10010 /* Gets value of the extra bits for the given length, cfr. the DEFLATE spec. */
10011 static int ZopfliGetLengthExtraBitsValue(int l) {
10012 static const int table[259] = {
10013 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0,
10014 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5,
10015 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6,
10016 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
10017 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2,
10018 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10019 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
10020 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
10021 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6,
10022 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
10023 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
10024 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0
10025 };
10026 return table[l];
10027 }
10028  
10029 /*
10030 Gets the symbol for the given length, cfr. the DEFLATE spec.
10031 Returns the symbol in the range [257-285] (inclusive)
10032 */
10033 static int ZopfliGetLengthSymbol(int l) {
10034 static const int table[259] = {
10035 0, 0, 0, 257, 258, 259, 260, 261, 262, 263, 264,
10036 265, 265, 266, 266, 267, 267, 268, 268,
10037 269, 269, 269, 269, 270, 270, 270, 270,
10038 271, 271, 271, 271, 272, 272, 272, 272,
10039 273, 273, 273, 273, 273, 273, 273, 273,
10040 274, 274, 274, 274, 274, 274, 274, 274,
10041 275, 275, 275, 275, 275, 275, 275, 275,
10042 276, 276, 276, 276, 276, 276, 276, 276,
10043 277, 277, 277, 277, 277, 277, 277, 277,
10044 277, 277, 277, 277, 277, 277, 277, 277,
10045 278, 278, 278, 278, 278, 278, 278, 278,
10046 278, 278, 278, 278, 278, 278, 278, 278,
10047 279, 279, 279, 279, 279, 279, 279, 279,
10048 279, 279, 279, 279, 279, 279, 279, 279,
10049 280, 280, 280, 280, 280, 280, 280, 280,
10050 280, 280, 280, 280, 280, 280, 280, 280,
10051 281, 281, 281, 281, 281, 281, 281, 281,
10052 281, 281, 281, 281, 281, 281, 281, 281,
10053 281, 281, 281, 281, 281, 281, 281, 281,
10054 281, 281, 281, 281, 281, 281, 281, 281,
10055 282, 282, 282, 282, 282, 282, 282, 282,
10056 282, 282, 282, 282, 282, 282, 282, 282,
10057 282, 282, 282, 282, 282, 282, 282, 282,
10058 282, 282, 282, 282, 282, 282, 282, 282,
10059 283, 283, 283, 283, 283, 283, 283, 283,
10060 283, 283, 283, 283, 283, 283, 283, 283,
10061 283, 283, 283, 283, 283, 283, 283, 283,
10062 283, 283, 283, 283, 283, 283, 283, 283,
10063 284, 284, 284, 284, 284, 284, 284, 284,
10064 284, 284, 284, 284, 284, 284, 284, 284,
10065 284, 284, 284, 284, 284, 284, 284, 284,
10066 284, 284, 284, 284, 284, 284, 284, 285
10067 };
10068 return table[l];
10069 }
10070  
10071 /* Gets the amount of extra bits for the given length symbol. */
10072 static int ZopfliGetLengthSymbolExtraBits(int s) {
10073 static const int table[29] = {
10074 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
10075 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0
10076 };
10077 return table[s - 257];
10078 }
10079  
10080 /* Gets the amount of extra bits for the given distance symbol. */
10081 static int ZopfliGetDistSymbolExtraBits(int s) {
10082 static const int table[30] = {
10083 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
10084 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
10085 };
10086 return table[s];
10087 }
10088  
10089 #endif /* ZOPFLI_SYMBOLS_H_ */
10090  
10091 /* util.h */
10092 /*
10093 Copyright 2011 Google Inc. All Rights Reserved.
10094  
10095 Licensed under the Apache License, Version 2.0 (the "License");
10096 you may not use this file except in compliance with the License.
10097 You may obtain a copy of the License at
10098  
10099 http://www.apache.org/licenses/LICENSE-2.0
10100  
10101 Unless required by applicable law or agreed to in writing, software
10102 distributed under the License is distributed on an "AS IS" BASIS,
10103 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10104 See the License for the specific language governing permissions and
10105 limitations under the License.
10106  
10107 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10108 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10109 */
10110  
10111 /*
10112 Several utilities, including: #defines to try different compression results,
10113 basic deflate specification values and generic program options.
10114 */
10115  
10116 #ifndef ZOPFLI_UTIL_H_
10117 #define ZOPFLI_UTIL_H_
10118  
10119 #include <string.h>
10120 #include <stdlib.h>
10121  
10122 /* Minimum and maximum length that can be encoded in deflate. */
10123 #define ZOPFLI_MAX_MATCH 258
10124 #define ZOPFLI_MIN_MATCH 3
10125  
10126 /* Number of distinct literal/length and distance symbols in DEFLATE */
10127 #define ZOPFLI_NUM_LL 288
10128 #define ZOPFLI_NUM_D 32
10129  
10130 /*
10131 The window size for deflate. Must be a power of two. This should be 32768, the
10132 maximum possible by the deflate spec. Anything less hurts compression more than
10133 speed.
10134 */
10135 #define ZOPFLI_WINDOW_SIZE 32768
10136  
10137 /*
10138 The window mask used to wrap indices into the window. This is why the
10139 window size must be a power of two.
10140 */
10141 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
10142  
10143 /*
10144 A block structure of huge, non-smart, blocks to divide the input into, to allow
10145 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
10146 The whole compression algorithm, including the smarter block splitting, will
10147 be executed independently on each huge block.
10148 Dividing into huge blocks hurts compression, but not much relative to the size.
10149 Set it to 0 to disable master blocks.
10150 */
10151 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
10152  
10153 /*
10154 Used to initialize costs for example
10155 */
10156 #define ZOPFLI_LARGE_FLOAT 1e30
10157  
10158 /*
10159 For longest match cache. max 256. Uses huge amounts of memory but makes it
10160 faster. Uses this many times three bytes per single byte of the input data.
10161 This is so because longest match finding has to find the exact distance
10162 that belongs to each length for the best lz77 strategy.
10163 Good values: e.g. 5, 8.
10164 */
10165 #define ZOPFLI_CACHE_LENGTH 8
10166  
10167 /*
10168 limit the max hash chain hits for this hash value. This has an effect only
10169 on files where the hash value is the same very often. On these files, this
10170 gives worse compression (the value should ideally be 32768, which is the
10171 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
10172 faster on some specific files.
10173 Good value: e.g. 8192.
10174 */
10175 #define ZOPFLI_MAX_CHAIN_HITS 8192
10176  
10177 /*
10178 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
10179 consumes a lot of memory but speeds it up. No effect on compression size.
10180 */
10181 #define ZOPFLI_LONGEST_MATCH_CACHE
10182  
10183 /*
10184 Enable to remember amount of successive identical bytes in the hash chain for
10185 finding longest match
10186 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
10187 This has no effect on the compression result, and enabling it increases speed.
10188 */
10189 #define ZOPFLI_HASH_SAME
10190  
10191 /*
10192 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
10193 best length so far is long enough. This is way faster for files with lots of
10194 identical bytes, on which the compressor is otherwise too slow. Regular files
10195 are unaffected or maybe a tiny bit slower.
10196 This has no effect on the compression result, only on speed.
10197 */
10198 #define ZOPFLI_HASH_SAME_HASH
10199  
10200 /*
10201 Enable this, to avoid slowness for files which are a repetition of the same
10202 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
10203 the compression result.
10204 */
10205 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
10206  
10207 /*
10208 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
10209 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
10210 varies from file to file.
10211 */
10212 #define ZOPFLI_LAZY_MATCHING
10213  
10214 /*
10215 Appends value to dynamically allocated memory, doubling its allocation size
10216 whenever needed.
10217  
10218 value: the value to append, type T
10219 data: pointer to the dynamic array to append to, type T**
10220 size: pointer to the size of the array to append to, type size_t*. This is the
10221 size that you consider the array to be, not the internal allocation size.
10222 Precondition: allocated size of data is at least a power of two greater than or
10223 equal than *size.
10224 */
10225 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
10226 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
10227 if (!((*size) & ((*size) - 1))) {\
10228 /*double alloc size if it's a power of two*/\
10229 void** data_void = reinterpret_cast<void**>(data);\
10230 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
10231 : realloc((*data), (*size) * 2 * sizeof(**data));\
10232 }\
10233 (*data)[(*size)] = (value);\
10234 (*size)++;\
10235 }
10236 #else /* C gives problems with strict-aliasing rules for (void**) cast */
10237 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
10238 if (!((*size) & ((*size) - 1))) {\
10239 /*double alloc size if it's a power of two*/\
10240 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
10241 : realloc((*data), (*size) * 2 * sizeof(**data));\
10242 }\
10243 (*data)[(*size)] = (value);\
10244 (*size)++;\
10245 }
10246 #endif
10247  
10248  
10249 #endif /* ZOPFLI_UTIL_H_ */
10250  
10251  
10252 #include <assert.h>
10253 #include <stdio.h>
10254 #include <stdlib.h>
10255  
10256 82 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store) {
10257 82 store->size = 0;
10258 82 store->litlens = 0;
10259 82 store->dists = 0;
10260 82 store->pos = 0;
10261 82 store->data = data;
10262 82 store->ll_symbol = 0;
10263 82 store->d_symbol = 0;
10264 82 store->ll_counts = 0;
10265 82 store->d_counts = 0;
10266 82 }
10267  
10268 82 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store) {
10269 82 free(store->litlens);
10270 82 free(store->dists);
10271 82 free(store->pos);
10272 82 free(store->ll_symbol);
10273 82 free(store->d_symbol);
10274 82 free(store->ll_counts);
10275 82 free(store->d_counts);
10276 82 }
10277  
10278 8 static size_t CeilDiv(size_t a, size_t b) {
10279 8 return (a + b - 1) / b;
10280 }
10281  
10282 4 void ZopfliCopyLZ77Store(
10283 const ZopfliLZ77Store* source, ZopfliLZ77Store* dest) {
10284 size_t i;
10285 4 size_t llsize = ZOPFLI_NUM_LL * CeilDiv(source->size, ZOPFLI_NUM_LL);
10286 4 size_t dsize = ZOPFLI_NUM_D * CeilDiv(source->size, ZOPFLI_NUM_D);
10287 4 ZopfliCleanLZ77Store(dest);
10288 4 ZopfliInitLZ77Store(source->data, dest);
10289 4 dest->litlens =
10290 4 (unsigned short*)malloc(sizeof(*dest->litlens) * source->size);
10291 4 dest->dists = (unsigned short*)malloc(sizeof(*dest->dists) * source->size);
10292 4 dest->pos = (size_t*)malloc(sizeof(*dest->pos) * source->size);
10293 4 dest->ll_symbol =
10294 4 (unsigned short*)malloc(sizeof(*dest->ll_symbol) * source->size);
10295 4 dest->d_symbol =
10296 4 (unsigned short*)malloc(sizeof(*dest->d_symbol) * source->size);
10297 4 dest->ll_counts = (size_t*)malloc(sizeof(*dest->ll_counts) * llsize);
10298 4 dest->d_counts = (size_t*)malloc(sizeof(*dest->d_counts) * dsize);
10299  
10300 /* Allocation failed. */
10301 4 ▶ if (!dest->litlens || !dest->dists) exit(-1);
Branch: 2 branches
BranchTrueFalseTotal
if (!dest->litlens || !dest->dists) exit(-1); 4 0 4
if (!dest->litlens || !dest->dists) exit(-1); 0 4 4
10302 4 ▶ if (!dest->pos) exit(-1);
Branch: 1 branch
BranchTrueFalseTotal
if (!dest->pos) exit(-1); 0 4 4
10303 4 ▶ if (!dest->ll_symbol || !dest->d_symbol) exit(-1);
Branch: 2 branches
BranchTrueFalseTotal
if (!dest->ll_symbol || !dest->d_symbol) exit(-1); 4 0 4
if (!dest->ll_symbol || !dest->d_symbol) exit(-1); 0 4 4
10304 4 ▶ if (!dest->ll_counts || !dest->d_counts) exit(-1);
Branch: 2 branches
BranchTrueFalseTotal
if (!dest->ll_counts || !dest->d_counts) exit(-1); 4 0 4
if (!dest->ll_counts || !dest->d_counts) exit(-1); 0 4 4
10305  
10306 4 dest->size = source->size;
10307 68 ▶ for (i = 0; i < source->size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < source->size; i++) { 64 4 68
10308 64 dest->litlens[i] = source->litlens[i];
10309 64 dest->dists[i] = source->dists[i];
10310 64 dest->pos[i] = source->pos[i];
10311 64 dest->ll_symbol[i] = source->ll_symbol[i];
10312 64 dest->d_symbol[i] = source->d_symbol[i];
10313 }
10314 1156 ▶ for (i = 0; i < llsize; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < llsize; i++) { 1152 4 1156
10315 1152 dest->ll_counts[i] = source->ll_counts[i];
10316 }
10317 164 ▶ for (i = 0; i < dsize; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < dsize; i++) { 160 4 164
10318 160 dest->d_counts[i] = source->d_counts[i];
10319 }
10320 4 }
10321  
10322 /*
10323 Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store.
10324 context must be a ZopfliLZ77Store*.
10325 */
10326 1216 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
10327 size_t pos, ZopfliLZ77Store* store) {
10328 size_t i;
10329 /* Needed for using ZOPFLI_APPEND_DATA multiple times. */
10330 1216 size_t origsize = store->size;
10331 1216 size_t llstart = ZOPFLI_NUM_LL * (origsize / ZOPFLI_NUM_LL);
10332 1216 size_t dstart = ZOPFLI_NUM_D * (origsize / ZOPFLI_NUM_D);
10333  
10334 /* Everytime the index wraps around, a new cumulative histogram is made: we're
10335 keeping one histogram value per LZ77 symbol rather than a full histogram for
10336 each to save memory. */
10337 1216 ▶ if (origsize % ZOPFLI_NUM_LL == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (origsize % ZOPFLI_NUM_LL == 0) { 74 1142 1216
10338 74 size_t llsize = origsize;
10339 21386 ▶ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_LL; i++) { 21312 74 21386
10340 21312 ▶ ZOPFLI_APPEND_DATA(
Branch: 3 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA( 740 20572 21312
ZOPFLI_APPEND_DATA( 74 666 740
ZOPFLI_APPEND_DATA( 0 21312 21312
10341 origsize == 0 ? 0 : store->ll_counts[origsize - ZOPFLI_NUM_LL + i],
10342 &store->ll_counts, &llsize);
10343 }
10344 }
10345 1216 ▶ if (origsize % ZOPFLI_NUM_D == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (origsize % ZOPFLI_NUM_D == 0) { 93 1123 1216
10346 93 size_t dsize = origsize;
10347 3069 ▶ for (i = 0; i < ZOPFLI_NUM_D; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_D; i++) { 2976 93 3069
10348 2976 ▶ ZOPFLI_APPEND_DATA(
Branch: 3 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA( 463 2513 2976
ZOPFLI_APPEND_DATA( 74 389 463
ZOPFLI_APPEND_DATA( 608 2368 2976
10349 origsize == 0 ? 0 : store->d_counts[origsize - ZOPFLI_NUM_D + i],
10350 &store->d_counts, &dsize);
10351 }
10352 }
10353  
10354 1216 ▶ ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size); 408 808 1216
ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size); 74 334 408
10355 1216 store->size = origsize;
10356 1216 ▶ ZOPFLI_APPEND_DATA(dist, &store->dists, &store->size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(dist, &store->dists, &store->size); 408 808 1216
ZOPFLI_APPEND_DATA(dist, &store->dists, &store->size); 74 334 408
10357 1216 store->size = origsize;
10358 1216 ▶ ZOPFLI_APPEND_DATA(pos, &store->pos, &store->size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(pos, &store->pos, &store->size); 408 808 1216
ZOPFLI_APPEND_DATA(pos, &store->pos, &store->size); 74 334 408
10359 assert(length < 259);
10360  
10361 1216 ▶ if (dist == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist == 0) { 513 703 1216
10362 513 store->size = origsize;
10363 513 ▶ ZOPFLI_APPEND_DATA(length, &store->ll_symbol, &store->size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(length, &store->ll_symbol, &store->size); 285 228 513
ZOPFLI_APPEND_DATA(length, &store->ll_symbol, &store->size); 57 228 285
10364 513 store->size = origsize;
10365 513 ▶ ZOPFLI_APPEND_DATA(0, &store->d_symbol, &store->size);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(0, &store->d_symbol, &store->size); 285 228 513
ZOPFLI_APPEND_DATA(0, &store->d_symbol, &store->size); 57 228 285
10366 513 store->ll_counts[llstart + length]++;
10367 } else {
10368 703 store->size = origsize;
10369 703 ▶ ZOPFLI_APPEND_DATA(ZopfliGetLengthSymbol(length),
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(ZopfliGetLengthSymbol(length), 123 580 703
ZOPFLI_APPEND_DATA(ZopfliGetLengthSymbol(length), 17 106 123
10370 &store->ll_symbol, &store->size);
10371 703 store->size = origsize;
10372 703 ▶ ZOPFLI_APPEND_DATA(ZopfliGetDistSymbol(dist),
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(ZopfliGetDistSymbol(dist), 123 580 703
ZOPFLI_APPEND_DATA(ZopfliGetDistSymbol(dist), 17 106 123
10373 &store->d_symbol, &store->size);
10374 703 store->ll_counts[llstart + ZopfliGetLengthSymbol(length)]++;
10375 703 store->d_counts[dstart + ZopfliGetDistSymbol(dist)]++;
10376 }
10377 1216 }
10378  
10379 4 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
10380 ZopfliLZ77Store* target) {
10381 size_t i;
10382 68 ▶ for (i = 0; i < store->size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < store->size; i++) { 64 4 68
10383 64 ZopfliStoreLitLenDist(store->litlens[i], store->dists[i],
10384 64 store->pos[i], target);
10385 }
10386 4 }
10387  
10388 204 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
10389 size_t lstart, size_t lend) {
10390 204 size_t l = lend - 1;
10391 204 ▶ if (lstart == lend) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (lstart == lend) return 0; 0 204 204
10392 408 return lz77->pos[l] + ((lz77->dists[l] == 0) ?
10393 204 ▶ 1 : lz77->litlens[l]) - lz77->pos[lstart];
Branch: 1 branch
BranchTrueFalseTotal
1 : lz77->litlens[l]) - lz77->pos[lstart]; 174 30 204
10394 }
10395  
10396 0 static void ZopfliLZ77GetHistogramAt(const ZopfliLZ77Store* lz77, size_t lpos,
10397 size_t* ll_counts, size_t* d_counts) {
10398 /* The real histogram is created by using the histogram for this chunk, but
10399 all superfluous values of this chunk subtracted. */
10400 0 size_t llpos = ZOPFLI_NUM_LL * (lpos / ZOPFLI_NUM_LL);
10401 0 size_t dpos = ZOPFLI_NUM_D * (lpos / ZOPFLI_NUM_D);
10402 size_t i;
10403 0 ▶ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_LL; i++) { 0 0 0
10404 0 ll_counts[i] = lz77->ll_counts[llpos + i];
10405 }
10406 0 ▶ for (i = lpos + 1; i < llpos + ZOPFLI_NUM_LL && i < lz77->size; i++) {
Branch: 2 branches
BranchTrueFalseTotal
for (i = lpos + 1; i < llpos + ZOPFLI_NUM_LL && i < lz77->size; i++) { 0 0 0
for (i = lpos + 1; i < llpos + ZOPFLI_NUM_LL && i < lz77->size; i++) { 0 0 0
10407 0 ll_counts[lz77->ll_symbol[i]]--;
10408 }
10409 0 ▶ for (i = 0; i < ZOPFLI_NUM_D; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_D; i++) { 0 0 0
10410 0 d_counts[i] = lz77->d_counts[dpos + i];
10411 }
10412 0 ▶ for (i = lpos + 1; i < dpos + ZOPFLI_NUM_D && i < lz77->size; i++) {
Branch: 2 branches
BranchTrueFalseTotal
for (i = lpos + 1; i < dpos + ZOPFLI_NUM_D && i < lz77->size; i++) { 0 0 0
for (i = lpos + 1; i < dpos + ZOPFLI_NUM_D && i < lz77->size; i++) { 0 0 0
10413 0 ▶ if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]--;
Branch: 1 branch
BranchTrueFalseTotal
if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]--; 0 0 0
10414 }
10415 0 }
10416  
10417 261 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
10418 size_t lstart, size_t lend,
10419 size_t* ll_counts, size_t* d_counts) {
10420 size_t i;
10421 261 ▶ if (lstart + ZOPFLI_NUM_LL * 3 > lend) {
Branch: 1 branch
BranchTrueFalseTotal
if (lstart + ZOPFLI_NUM_LL * 3 > lend) { 261 0 261
10422 261 memset(ll_counts, 0, sizeof(*ll_counts) * ZOPFLI_NUM_LL);
10423 261 memset(d_counts, 0, sizeof(*d_counts) * ZOPFLI_NUM_D);
10424 4676 ▶ for (i = lstart; i < lend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = lstart; i < lend; i++) { 4415 261 4676
10425 4415 ll_counts[lz77->ll_symbol[i]]++;
10426 4415 ▶ if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]++;
Branch: 1 branch
BranchTrueFalseTotal
if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]++; 3380 1035 4415
10427 }
10428 } else {
10429 /* Subtract the cumulative histograms at the end and the start to get the
10430 histogram for this range. */
10431 0 ZopfliLZ77GetHistogramAt(lz77, lend - 1, ll_counts, d_counts);
10432 0 ▶ if (lstart > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (lstart > 0) { 0 0 0
10433 size_t ll_counts2[ZOPFLI_NUM_LL];
10434 size_t d_counts2[ZOPFLI_NUM_D];
10435 0 ZopfliLZ77GetHistogramAt(lz77, lstart - 1, ll_counts2, d_counts2);
10436  
10437 0 ▶ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_LL; i++) { 0 0 0
10438 0 ll_counts[i] -= ll_counts2[i];
10439 }
10440 0 ▶ for (i = 0; i < ZOPFLI_NUM_D; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_D; i++) { 0 0 0
10441 0 d_counts[i] -= d_counts2[i];
10442 }
10443 }
10444 }
10445 261 }
10446  
10447 11 void ZopfliInitBlockState(const ZopfliOptions* options,
10448 size_t blockstart, size_t blockend, int add_lmc,
10449 ZopfliBlockState* s) {
10450 11 s->options = options;
10451 11 s->blockstart = blockstart;
10452 11 s->blockend = blockend;
10453 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
10454 11 ▶ if (add_lmc) {
Branch: 1 branch
BranchTrueFalseTotal
if (add_lmc) { 8 3 11
10455 8 s->lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
10456 8 ZopfliInitCache(blockend - blockstart, s->lmc);
10457 } else {
10458 3 s->lmc = 0;
10459 }
10460 #endif
10461 11 }
10462  
10463 11 void ZopfliCleanBlockState(ZopfliBlockState* s) {
10464 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
10465 11 ▶ if (s->lmc) {
Branch: 1 branch
BranchTrueFalseTotal
if (s->lmc) { 8 3 11
10466 8 ZopfliCleanCache(s->lmc);
10467 8 free(s->lmc);
10468 }
10469 #endif
10470 11 }
10471  
10472 /*
10473 Gets a score of the length given the distance. Typically, the score of the
10474 length is the length itself, but if the distance is very long, decrease the
10475 score of the length a bit to make up for the fact that long distances use large
10476 amounts of extra bits.
10477  
10478 This is not an accurate score, it is a heuristic only for the greedy LZ77
10479 implementation. More accurate cost models are employed later. Making this
10480 heuristic more accurate may hurt rather than improve compression.
10481  
10482 The two direct uses of this heuristic are:
10483 -avoid using a length of 3 in combination with a long distance. This only has
10484 an effect if length == 3.
10485 -make a slightly better choice between the two options of the lazy matching.
10486  
10487 Indirectly, this affects:
10488 -the block split points if the default of block splitting first is used, in a
10489 rather unpredictable way
10490 -the first zopfli run, so it affects the chance of the first run being closer
10491 to the optimal output
10492 */
10493 268 static int GetLengthScore(int length, int distance) {
10494 /*
10495 At 1024, the distance uses 9+ extra bits and this seems to be the sweet spot
10496 on tested files.
10497 */
10498 268 ▶ return distance > 1024 ? length - 1 : length;
Branch: 1 branch
BranchTrueFalseTotal
return distance > 1024 ? length - 1 : length; 0 268 268
10499 }
10500  
10501 666 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
10502 unsigned short dist, unsigned short length) {
10503  
10504 /* TODO(lode): make this only run in a debug compile, it's for assert only. */
10505 size_t i;
10506  
10507 assert(pos + length <= datasize);
10508 165420 ▶ for (i = 0; i < length; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < length; i++) { 164754 666 165420
10509 164754 ▶ if (data[pos - dist + i] != data[pos + i]) {
Branch: 1 branch
BranchTrueFalseTotal
if (data[pos - dist + i] != data[pos + i]) { 0 164754 164754
10510 assert(data[pos - dist + i] == data[pos + i]);
10511 0 break;
10512 }
10513 }
10514 666 }
10515  
10516 /*
10517 Finds how long the match of scan and match is. Can be used to find how many
10518 bytes starting from scan, and from match, are equal. Returns the last byte
10519 after scan, which is still equal to the correspondinb byte after match.
10520 scan is the position to compare
10521 match is the earlier position to compare.
10522 end is the last possible byte, beyond which to stop looking.
10523 safe_end is a few (8) bytes before end, for comparing multiple bytes at once.
10524 */
10525 24170 static const unsigned char* GetMatch(const unsigned char* scan,
10526 const unsigned char* match,
10527 const unsigned char* end,
10528 const unsigned char* safe_end) {
10529  
10530 if (sizeof(size_t) == 8) {
10531 /* 8 checks at once per array bounds check (size_t is 64-bit). */
10532 661010 ▶ while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) {
Branch: 2 branches
BranchTrueFalseTotal
while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) { 636840 24170 661010
while (scan < safe_end && *((size_t*)scan) == *((size_t*)match)) { 636840 0 636840
10533 636840 scan += 8;
10534 636840 match += 8;
10535 }
10536 } else if (sizeof(unsigned int) == 4) {
10537 /* 4 checks at once per array bounds check (unsigned int is 32-bit). */
10538 while (scan < safe_end
10539 && *((unsigned int*)scan) == *((unsigned int*)match)) {
10540 scan += 4;
10541 match += 4;
10542 }
10543 } else {
10544 /* do 8 checks at once per array bounds check. */
10545 while (scan < safe_end && *scan == *match && *++scan == *++match
10546 && *++scan == *++match && *++scan == *++match
10547 && *++scan == *++match && *++scan == *++match
10548 && *++scan == *++match && *++scan == *++match) {
10549 scan++; match++;
10550 }
10551 }
10552  
10553 /* The remaining few bytes. */
10554 89156 ▶ while (scan != end && *scan == *match) {
Branch: 2 branches
BranchTrueFalseTotal
while (scan != end && *scan == *match) { 64986 24170 89156
while (scan != end && *scan == *match) { 64986 0 64986
10555 64986 scan++; match++;
10556 }
10557  
10558 24170 return scan;
10559 }
10560  
10561 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
10562 /*
10563 Gets distance, length and sublen values from the cache if possible.
10564 Returns 1 if it got the values from the cache, 0 if not.
10565 Updates the limit value to a smaller one if possible with more limited
10566 information from the cache.
10567 */
10568 147606 static int TryGetFromLongestMatchCache(ZopfliBlockState* s,
10569 size_t pos, size_t* limit,
10570 unsigned short* sublen, unsigned short* distance, unsigned short* length) {
10571 /* The LMC cache starts at the beginning of the block rather than the
10572 beginning of the whole array. */
10573 147606 size_t lmcpos = pos - s->blockstart;
10574  
10575 /* Length > 0 and dist 0 is invalid combination, which indicates on purpose
10576 that this cache value is not filled in yet. */
10577 295145 ▶ unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 ||
Branch: 2 branches
BranchTrueFalseTotal
unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || 147539 67 147606
unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || 147539 0 147539
10578 147539 ▶ s->lmc->dist[lmcpos] != 0);
Branch: 1 branch
BranchTrueFalseTotal
s->lmc->dist[lmcpos] != 0); 122868 24671 147539
10579 270474 ▶ unsigned char limit_ok_for_cache = cache_available &&
Branch: 1 branch
BranchTrueFalseTotal
unsigned char limit_ok_for_cache = cache_available && 122868 24738 147606
10580 122868 ▶ (*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit ||
Branch: 3 branches
BranchTrueFalseTotal
(*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit || 14 122854 122868
(*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit || 14 0 14
(*limit == ZOPFLI_MAX_MATCH || s->lmc->length[lmcpos] <= *limit || 0 14 14
10581 0 (sublen && ZopfliMaxCachedSublen(s->lmc,
10582 0 ▶ lmcpos, s->lmc->length[lmcpos]) >= *limit));
Branch: 1 branch
BranchTrueFalseTotal
lmcpos, s->lmc->length[lmcpos]) >= *limit)); 0 0 0
10583  
10584 147606 ▶ if (s->lmc && limit_ok_for_cache && cache_available) {
Branch: 3 branches
BranchTrueFalseTotal
if (s->lmc && limit_ok_for_cache && cache_available) { 147539 67 147606
if (s->lmc && limit_ok_for_cache && cache_available) { 122854 24685 147539
if (s->lmc && limit_ok_for_cache && cache_available) { 122854 0 122854
10585 245164 if (!sublen || s->lmc->length[lmcpos]
10586 122310 <= ZopfliMaxCachedSublen(s->lmc, lmcpos, s->lmc->length[lmcpos])) {
10587 122854 *length = s->lmc->length[lmcpos];
10588 122854 ▶ if (*length > *limit) *length = *limit;
Branch: 1 branch
BranchTrueFalseTotal
if (*length > *limit) *length = *limit; 0 122854 122854
10589 122854 ▶ if (sublen) {
Branch: 1 branch
BranchTrueFalseTotal
if (sublen) { 122310 544 122854
10590 122310 ZopfliCacheToSublen(s->lmc, lmcpos, *length, sublen);
10591 122310 *distance = sublen[*length];
10592 122310 ▶ if (*limit == ZOPFLI_MAX_MATCH && *length >= ZOPFLI_MIN_MATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if (*limit == ZOPFLI_MAX_MATCH && *length >= ZOPFLI_MIN_MATCH) { 122310 0 122310
10593 assert(sublen[*length] == s->lmc->dist[lmcpos]);
10594 }
10595 } else {
10596 544 *distance = s->lmc->dist[lmcpos];
10597 }
10598 122854 return 1;
10599 }
10600 /* Can't use much of the cache, since the "sublens" need to be calculated,
10601 but at least we already know when to stop. */
10602 0 *limit = s->lmc->length[lmcpos];
10603 }
10604  
10605 24752 return 0;
10606 }
10607  
10608 /*
10609 Stores the found sublen, distance and length in the longest match cache, if
10610 possible.
10611 */
10612 24622 static void StoreInLongestMatchCache(ZopfliBlockState* s,
10613 size_t pos, size_t limit,
10614 const unsigned short* sublen,
10615 unsigned short distance, unsigned short length) {
10616 /* The LMC cache starts at the beginning of the block rather than the
10617 beginning of the whole array. */
10618 24622 size_t lmcpos = pos - s->blockstart;
10619  
10620 /* Length > 0 and dist 0 is invalid combination, which indicates on purpose
10621 that this cache value is not filled in yet. */
10622 49177 ▶ unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 ||
Branch: 2 branches
BranchTrueFalseTotal
unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || 24555 67 24622
unsigned char cache_available = s->lmc && (s->lmc->length[lmcpos] == 0 || 24555 0 24555
10623 24555 ▶ s->lmc->dist[lmcpos] != 0);
Branch: 1 branch
BranchTrueFalseTotal
s->lmc->dist[lmcpos] != 0); 14 24541 24555
10624  
10625 24622 ▶ if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) {
Branch: 4 branches
BranchTrueFalseTotal
if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { 24555 67 24622
if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { 17468 7087 24555
if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { 17468 0 17468
if (s->lmc && limit == ZOPFLI_MAX_MATCH && sublen && !cache_available) { 17468 0 17468
10626 assert(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0);
10627 17468 ▶ s->lmc->dist[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : distance;
Branch: 1 branch
BranchTrueFalseTotal
s->lmc->dist[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : distance; 17468 0 17468
10628 17468 ▶ s->lmc->length[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : length;
Branch: 1 branch
BranchTrueFalseTotal
s->lmc->length[lmcpos] = length < ZOPFLI_MIN_MATCH ? 0 : length; 17468 0 17468
10629 assert(!(s->lmc->length[lmcpos] == 1 && s->lmc->dist[lmcpos] == 0));
10630 17468 ZopfliSublenToCache(sublen, lmcpos, length, s->lmc);
10631 }
10632 24622 }
10633 #endif
10634  
10635 147606 void ZopfliFindLongestMatch(ZopfliBlockState* s, const ZopfliHash* h,
10636 const unsigned char* array,
10637 size_t pos, size_t size, size_t limit,
10638 unsigned short* sublen, unsigned short* distance, unsigned short* length) {
10639 147606 unsigned short hpos = pos & ZOPFLI_WINDOW_MASK, p, pp;
10640 147606 unsigned short bestdist = 0;
10641 147606 unsigned short bestlength = 1;
10642 const unsigned char* scan;
10643 const unsigned char* match;
10644 const unsigned char* arrayend;
10645 const unsigned char* arrayend_safe;
10646 #if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE
10647 147606 int chain_counter = ZOPFLI_MAX_CHAIN_HITS; /* For quitting early. */
10648 #endif
10649  
10650 147606 unsigned dist = 0; /* Not unsigned short on purpose. */
10651  
10652 147606 int* hhead = h->head;
10653 147606 unsigned short* hprev = h->prev;
10654 147606 int* hhashval = h->hashval;
10655 147606 int hval = h->val;
10656  
10657 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
10658 147606 ▶ if (TryGetFromLongestMatchCache(s, pos, &limit, sublen, distance, length)) {
Branch: 1 branch
BranchTrueFalseTotal
if (TryGetFromLongestMatchCache(s, pos, &limit, sublen, distance, length)) { 122854 24752 147606
10659 assert(pos + *length <= size);
10660 122854 return;
10661 }
10662 #endif
10663  
10664 assert(limit <= ZOPFLI_MAX_MATCH);
10665 assert(limit >= ZOPFLI_MIN_MATCH);
10666 assert(pos < size);
10667  
10668 24752 ▶ if (size - pos < ZOPFLI_MIN_MATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if (size - pos < ZOPFLI_MIN_MATCH) { 130 24622 24752
10669 /* The rest of the code assumes there are at least ZOPFLI_MIN_MATCH bytes to
10670 try. */
10671 130 *length = 0;
10672 130 *distance = 0;
10673 130 return;
10674 }
10675  
10676 24622 ▶ if (pos + limit > size) {
Branch: 1 branch
BranchTrueFalseTotal
if (pos + limit > size) { 7063 17559 24622
10677 7063 limit = size - pos;
10678 }
10679 24622 arrayend = &array[pos] + limit;
10680 24622 arrayend_safe = arrayend - 8;
10681  
10682 assert(hval < 65536);
10683  
10684 24622 pp = hhead[hval]; /* During the whole loop, p == hprev[pp]. */
10685 24622 p = hprev[pp];
10686  
10687 assert(pp == hpos);
10688  
10689 24622 ▶ dist = p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp);
Branch: 1 branch
BranchTrueFalseTotal
dist = p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); 24170 452 24622
10690  
10691 /* Go through all distances. */
10692 24622 ▶ while (dist < ZOPFLI_WINDOW_SIZE) {
Branch: 1 branch
BranchTrueFalseTotal
while (dist < ZOPFLI_WINDOW_SIZE) { 24170 452 24622
10693 24170 unsigned short currentlength = 0;
10694  
10695 assert(p < ZOPFLI_WINDOW_SIZE);
10696 assert(p == hprev[pp]);
10697 assert(hhashval[p] == hval);
10698  
10699 24170 ▶ if (dist > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist > 0) { 24170 0 24170
10700 assert(pos < size);
10701 assert(dist <= pos);
10702 24170 scan = &array[pos];
10703 24170 match = &array[pos - dist];
10704  
10705 /* Testing the byte at position bestlength first, goes slightly faster. */
10706 24170 ▶ if (pos + bestlength >= size
Branch: 1 branch
BranchTrueFalseTotal
if (pos + bestlength >= size 24170 0 24170
10707 24170 ▶ || *(scan + bestlength) == *(match + bestlength)) {
Branch: 1 branch
BranchTrueFalseTotal
|| *(scan + bestlength) == *(match + bestlength)) { 24170 0 24170
10708  
10709 #ifdef ZOPFLI_HASH_SAME
10710 24170 unsigned short same0 = h->same[pos & ZOPFLI_WINDOW_MASK];
10711 24170 ▶ if (same0 > 2 && *scan == *match) {
Branch: 2 branches
BranchTrueFalseTotal
if (same0 > 2 && *scan == *match) { 0 24170 24170
if (same0 > 2 && *scan == *match) { 0 0 0
10712 0 unsigned short same1 = h->same[(pos - dist) & ZOPFLI_WINDOW_MASK];
10713 0 unsigned short same = same0 < same1 ? same0 : same1;
10714 0 ▶ if (same > limit) same = limit;
Branch: 1 branch
BranchTrueFalseTotal
if (same > limit) same = limit; 0 0 0
10715 0 scan += same;
10716 0 match += same;
10717 }
10718 #endif
10719 24170 scan = GetMatch(scan, match, arrayend, arrayend_safe);
10720 24170 currentlength = scan - &array[pos]; /* The found length. */
10721 }
10722  
10723 24170 ▶ if (currentlength > bestlength) {
Branch: 1 branch
BranchTrueFalseTotal
if (currentlength > bestlength) { 24170 0 24170
10724 24170 ▶ if (sublen) {
Branch: 1 branch
BranchTrueFalseTotal
if (sublen) { 24122 48 24170
10725 unsigned short j;
10726 5153610 ▶ for (j = bestlength + 1; j <= currentlength; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = bestlength + 1; j <= currentlength; j++) { 5129488 24122 5153610
10727 5129488 sublen[j] = dist;
10728 }
10729 }
10730 24170 bestdist = dist;
10731 24170 bestlength = currentlength;
10732 24170 ▶ if (currentlength >= limit) break;
Branch: 1 branch
BranchTrueFalseTotal
if (currentlength >= limit) break; 24170 0 24170
10733 }
10734 }
10735  
10736  
10737 #ifdef ZOPFLI_HASH_SAME_HASH
10738 /* Switch to the other hash once this will be more efficient. */
10739 0 ▶ if (hhead != h->head2 && bestlength >= h->same[hpos] &&
Branch: 2 branches
BranchTrueFalseTotal
if (hhead != h->head2 && bestlength >= h->same[hpos] && 0 0 0
if (hhead != h->head2 && bestlength >= h->same[hpos] && 0 0 0
10740 0 ▶ h->val2 == h->hashval2[p]) {
Branch: 1 branch
BranchTrueFalseTotal
h->val2 == h->hashval2[p]) { 0 0 0
10741 /* Now use the hash that encodes the length and first byte. */
10742 0 hhead = h->head2;
10743 0 hprev = h->prev2;
10744 0 hhashval = h->hashval2;
10745 0 hval = h->val2;
10746 }
10747 #endif
10748  
10749 0 pp = p;
10750 0 p = hprev[p];
10751 0 ▶ if (p == pp) break; /* Uninited prev value. */
Branch: 1 branch
BranchTrueFalseTotal
if (p == pp) break; /* Uninited prev value. */ 0 0 0
10752  
10753 0 ▶ dist += p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp);
Branch: 1 branch
BranchTrueFalseTotal
dist += p < pp ? pp - p : ((ZOPFLI_WINDOW_SIZE - p) + pp); 0 0 0
10754  
10755 #if ZOPFLI_MAX_CHAIN_HITS < ZOPFLI_WINDOW_SIZE
10756 0 chain_counter--;
10757 0 ▶ if (chain_counter <= 0) break;
Branch: 1 branch
BranchTrueFalseTotal
if (chain_counter <= 0) break; 0 0 0
10758 #endif
10759 }
10760  
10761 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
10762 24622 StoreInLongestMatchCache(s, pos, limit, sublen, bestdist, bestlength);
10763 #endif
10764  
10765 assert(bestlength <= limit);
10766  
10767 24622 *distance = bestdist;
10768 24622 *length = bestlength;
10769 assert(pos + *length <= size);
10770 }
10771  
10772 7 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
10773 size_t instart, size_t inend,
10774 ZopfliLZ77Store* store, ZopfliHash* h) {
10775 7 size_t i = 0, j;
10776 unsigned short leng;
10777 unsigned short dist;
10778 int lengthscore;
10779 7 size_t windowstart = instart > ZOPFLI_WINDOW_SIZE
10780 7 ? instart - ZOPFLI_WINDOW_SIZE : 0;
10781 unsigned short dummysublen[259];
10782  
10783 #ifdef ZOPFLI_LAZY_MATCHING
10784 /* Lazy matching. */
10785 7 unsigned prev_length = 0;
10786 7 unsigned prev_match = 0;
10787 int prevlengthscore;
10788 7 int match_available = 0;
10789 #endif
10790  
10791 7 ▶ if (instart == inend) return;
Branch: 1 branch
BranchTrueFalseTotal
if (instart == inend) return; 0 7 7
10792  
10793 7 ZopfliResetHash(ZOPFLI_WINDOW_SIZE, h);
10794 7 ZopfliWarmupHash(in, windowstart, inend, h);
10795 16 ▶ for (i = windowstart; i < instart; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = windowstart; i < instart; i++) { 9 7 16
10796 9 ZopfliUpdateHash(in, i, inend, h);
10797 }
10798  
10799 141 ▶ for (i = instart; i < inend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = instart; i < inend; i++) { 134 7 141
10800 134 ZopfliUpdateHash(in, i, inend, h);
10801  
10802 134 ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, dummysublen,
10803 &dist, &leng);
10804 134 lengthscore = GetLengthScore(leng, dist);
10805  
10806 #ifdef ZOPFLI_LAZY_MATCHING
10807 /* Lazy matching. */
10808 134 prevlengthscore = GetLengthScore(prev_length, prev_match);
10809 134 ▶ if (match_available) {
Branch: 1 branch
BranchTrueFalseTotal
if (match_available) { 6 128 134
10810 6 match_available = 0;
10811 6 ▶ if (lengthscore > prevlengthscore + 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (lengthscore > prevlengthscore + 1) { 0 6 6
10812 0 ZopfliStoreLitLenDist(in[i - 1], 0, i - 1, store);
10813 0 ▶ if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) {
Branch: 2 branches
BranchTrueFalseTotal
if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { 0 0 0
if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { 0 0 0
10814 0 match_available = 1;
10815 0 prev_length = leng;
10816 0 prev_match = dist;
10817 0 continue;
10818 }
10819 } else {
10820 /* Add previous to output. */
10821 6 leng = prev_length;
10822 6 dist = prev_match;
10823 6 lengthscore = prevlengthscore;
10824 /* Add to output. */
10825 6 ZopfliVerifyLenDist(in, inend, i - 1, dist, leng);
10826 6 ZopfliStoreLitLenDist(leng, dist, i - 1, store);
10827 756 ▶ for (j = 2; j < leng; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 2; j < leng; j++) { 750 6 756
10828 assert(i < inend);
10829 750 i++;
10830 750 ZopfliUpdateHash(in, i, inend, h);
10831 }
10832 6 continue;
10833 }
10834 }
10835 128 ▶ else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) {
Branch: 2 branches
BranchTrueFalseTotal
else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { 74 54 128
else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { 6 68 74
10836 6 match_available = 1;
10837 6 prev_length = leng;
10838 6 prev_match = dist;
10839 6 continue;
10840 }
10841 /* End of lazy matching. */
10842 #endif
10843  
10844 /* Add to output. */
10845 122 ▶ if (lengthscore >= ZOPFLI_MIN_MATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if (lengthscore >= ZOPFLI_MIN_MATCH) { 68 54 122
10846 68 ZopfliVerifyLenDist(in, inend, i, dist, leng);
10847 68 ZopfliStoreLitLenDist(leng, dist, i, store);
10848 } else {
10849 54 leng = 1;
10850 54 ZopfliStoreLitLenDist(in[i], 0, i, store);
10851 }
10852 17598 ▶ for (j = 1; j < leng; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 1; j < leng; j++) { 17476 122 17598
10853 assert(i < inend);
10854 17476 i++;
10855 17476 ZopfliUpdateHash(in, i, inend, h);
10856 }
10857 }
10858 }
10859 /*
10860 Copyright 2011 Google Inc. All Rights Reserved.
10861  
10862 Licensed under the Apache License, Version 2.0 (the "License");
10863 you may not use this file except in compliance with the License.
10864 You may obtain a copy of the License at
10865  
10866 http://www.apache.org/licenses/LICENSE-2.0
10867  
10868 Unless required by applicable law or agreed to in writing, software
10869 distributed under the License is distributed on an "AS IS" BASIS,
10870 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10871 See the License for the specific language governing permissions and
10872 limitations under the License.
10873  
10874 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10875 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10876 */
10877  
10878 /* squeeze.h */
10879 /*
10880 Copyright 2011 Google Inc. All Rights Reserved.
10881  
10882 Licensed under the Apache License, Version 2.0 (the "License");
10883 you may not use this file except in compliance with the License.
10884 You may obtain a copy of the License at
10885  
10886 http://www.apache.org/licenses/LICENSE-2.0
10887  
10888 Unless required by applicable law or agreed to in writing, software
10889 distributed under the License is distributed on an "AS IS" BASIS,
10890 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10891 See the License for the specific language governing permissions and
10892 limitations under the License.
10893  
10894 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10895 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10896 */
10897  
10898 /*
10899 The squeeze functions do enhanced LZ77 compression by optimal parsing with a
10900 cost model, rather than greedily choosing the longest length or using a single
10901 step of lazy matching like regular implementations.
10902  
10903 Since the cost model is based on the Huffman tree that can only be calculated
10904 after the LZ77 data is generated, there is a chicken and egg problem, and
10905 multiple runs are done with updated cost models to converge to a better
10906 solution.
10907 */
10908  
10909 #ifndef ZOPFLI_SQUEEZE_H_
10910 #define ZOPFLI_SQUEEZE_H_
10911  
10912 /* lz77.h */
10913 /*
10914 Copyright 2011 Google Inc. All Rights Reserved.
10915  
10916 Licensed under the Apache License, Version 2.0 (the "License");
10917 you may not use this file except in compliance with the License.
10918 You may obtain a copy of the License at
10919  
10920 http://www.apache.org/licenses/LICENSE-2.0
10921  
10922 Unless required by applicable law or agreed to in writing, software
10923 distributed under the License is distributed on an "AS IS" BASIS,
10924 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10925 See the License for the specific language governing permissions and
10926 limitations under the License.
10927  
10928 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10929 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10930 */
10931  
10932 /*
10933 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
10934 compression.
10935 */
10936  
10937 #ifndef ZOPFLI_LZ77_H_
10938 #define ZOPFLI_LZ77_H_
10939  
10940 #include <stdlib.h>
10941  
10942 /* cache.h */
10943 /*
10944 Copyright 2011 Google Inc. All Rights Reserved.
10945  
10946 Licensed under the Apache License, Version 2.0 (the "License");
10947 you may not use this file except in compliance with the License.
10948 You may obtain a copy of the License at
10949  
10950 http://www.apache.org/licenses/LICENSE-2.0
10951  
10952 Unless required by applicable law or agreed to in writing, software
10953 distributed under the License is distributed on an "AS IS" BASIS,
10954 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10955 See the License for the specific language governing permissions and
10956 limitations under the License.
10957  
10958 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10959 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10960 */
10961  
10962 /*
10963 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
10964 */
10965  
10966 #ifndef ZOPFLI_CACHE_H_
10967 #define ZOPFLI_CACHE_H_
10968  
10969 /* util.h */
10970 /*
10971 Copyright 2011 Google Inc. All Rights Reserved.
10972  
10973 Licensed under the Apache License, Version 2.0 (the "License");
10974 you may not use this file except in compliance with the License.
10975 You may obtain a copy of the License at
10976  
10977 http://www.apache.org/licenses/LICENSE-2.0
10978  
10979 Unless required by applicable law or agreed to in writing, software
10980 distributed under the License is distributed on an "AS IS" BASIS,
10981 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10982 See the License for the specific language governing permissions and
10983 limitations under the License.
10984  
10985 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
10986 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
10987 */
10988  
10989 /*
10990 Several utilities, including: #defines to try different compression results,
10991 basic deflate specification values and generic program options.
10992 */
10993  
10994 #ifndef ZOPFLI_UTIL_H_
10995 #define ZOPFLI_UTIL_H_
10996  
10997 #include <string.h>
10998 #include <stdlib.h>
10999  
11000 /* Minimum and maximum length that can be encoded in deflate. */
11001 #define ZOPFLI_MAX_MATCH 258
11002 #define ZOPFLI_MIN_MATCH 3
11003  
11004 /* Number of distinct literal/length and distance symbols in DEFLATE */
11005 #define ZOPFLI_NUM_LL 288
11006 #define ZOPFLI_NUM_D 32
11007  
11008 /*
11009 The window size for deflate. Must be a power of two. This should be 32768, the
11010 maximum possible by the deflate spec. Anything less hurts compression more than
11011 speed.
11012 */
11013 #define ZOPFLI_WINDOW_SIZE 32768
11014  
11015 /*
11016 The window mask used to wrap indices into the window. This is why the
11017 window size must be a power of two.
11018 */
11019 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
11020  
11021 /*
11022 A block structure of huge, non-smart, blocks to divide the input into, to allow
11023 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
11024 The whole compression algorithm, including the smarter block splitting, will
11025 be executed independently on each huge block.
11026 Dividing into huge blocks hurts compression, but not much relative to the size.
11027 Set it to 0 to disable master blocks.
11028 */
11029 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
11030  
11031 /*
11032 Used to initialize costs for example
11033 */
11034 #define ZOPFLI_LARGE_FLOAT 1e30
11035  
11036 /*
11037 For longest match cache. max 256. Uses huge amounts of memory but makes it
11038 faster. Uses this many times three bytes per single byte of the input data.
11039 This is so because longest match finding has to find the exact distance
11040 that belongs to each length for the best lz77 strategy.
11041 Good values: e.g. 5, 8.
11042 */
11043 #define ZOPFLI_CACHE_LENGTH 8
11044  
11045 /*
11046 limit the max hash chain hits for this hash value. This has an effect only
11047 on files where the hash value is the same very often. On these files, this
11048 gives worse compression (the value should ideally be 32768, which is the
11049 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
11050 faster on some specific files.
11051 Good value: e.g. 8192.
11052 */
11053 #define ZOPFLI_MAX_CHAIN_HITS 8192
11054  
11055 /*
11056 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
11057 consumes a lot of memory but speeds it up. No effect on compression size.
11058 */
11059 #define ZOPFLI_LONGEST_MATCH_CACHE
11060  
11061 /*
11062 Enable to remember amount of successive identical bytes in the hash chain for
11063 finding longest match
11064 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
11065 This has no effect on the compression result, and enabling it increases speed.
11066 */
11067 #define ZOPFLI_HASH_SAME
11068  
11069 /*
11070 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
11071 best length so far is long enough. This is way faster for files with lots of
11072 identical bytes, on which the compressor is otherwise too slow. Regular files
11073 are unaffected or maybe a tiny bit slower.
11074 This has no effect on the compression result, only on speed.
11075 */
11076 #define ZOPFLI_HASH_SAME_HASH
11077  
11078 /*
11079 Enable this, to avoid slowness for files which are a repetition of the same
11080 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
11081 the compression result.
11082 */
11083 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
11084  
11085 /*
11086 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
11087 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
11088 varies from file to file.
11089 */
11090 #define ZOPFLI_LAZY_MATCHING
11091  
11092 /*
11093 Appends value to dynamically allocated memory, doubling its allocation size
11094 whenever needed.
11095  
11096 value: the value to append, type T
11097 data: pointer to the dynamic array to append to, type T**
11098 size: pointer to the size of the array to append to, type size_t*. This is the
11099 size that you consider the array to be, not the internal allocation size.
11100 Precondition: allocated size of data is at least a power of two greater than or
11101 equal than *size.
11102 */
11103 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
11104 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11105 if (!((*size) & ((*size) - 1))) {\
11106 /*double alloc size if it's a power of two*/\
11107 void** data_void = reinterpret_cast<void**>(data);\
11108 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
11109 : realloc((*data), (*size) * 2 * sizeof(**data));\
11110 }\
11111 (*data)[(*size)] = (value);\
11112 (*size)++;\
11113 }
11114 #else /* C gives problems with strict-aliasing rules for (void**) cast */
11115 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11116 if (!((*size) & ((*size) - 1))) {\
11117 /*double alloc size if it's a power of two*/\
11118 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
11119 : realloc((*data), (*size) * 2 * sizeof(**data));\
11120 }\
11121 (*data)[(*size)] = (value);\
11122 (*size)++;\
11123 }
11124 #endif
11125  
11126  
11127 #endif /* ZOPFLI_UTIL_H_ */
11128  
11129  
11130 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
11131  
11132 /*
11133 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
11134 values.
11135 This is needed because the squeeze runs will ask these values multiple times for
11136 the same position.
11137 Uses large amounts of memory, since it has to remember the distance belonging
11138 to every possible shorter-than-the-best length (the so called "sublen" array).
11139 */
11140 typedef struct ZopfliLongestMatchCache {
11141 unsigned short* length;
11142 unsigned short* dist;
11143 unsigned char* sublen;
11144 } ZopfliLongestMatchCache;
11145  
11146 /* Initializes the ZopfliLongestMatchCache. */
11147 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
11148  
11149 /* Frees up the memory of the ZopfliLongestMatchCache. */
11150 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
11151  
11152 /* Stores sublen array in the cache. */
11153 void ZopfliSublenToCache(const unsigned short* sublen,
11154 size_t pos, size_t length,
11155 ZopfliLongestMatchCache* lmc);
11156  
11157 /* Extracts sublen array from the cache. */
11158 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
11159 size_t pos, size_t length,
11160 unsigned short* sublen);
11161 /* Returns the length up to which could be stored in the cache. */
11162 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
11163 size_t pos, size_t length);
11164  
11165 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
11166  
11167 #endif /* ZOPFLI_CACHE_H_ */
11168  
11169 /* hash.h */
11170 /*
11171 Copyright 2011 Google Inc. All Rights Reserved.
11172  
11173 Licensed under the Apache License, Version 2.0 (the "License");
11174 you may not use this file except in compliance with the License.
11175 You may obtain a copy of the License at
11176  
11177 http://www.apache.org/licenses/LICENSE-2.0
11178  
11179 Unless required by applicable law or agreed to in writing, software
11180 distributed under the License is distributed on an "AS IS" BASIS,
11181 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11182 See the License for the specific language governing permissions and
11183 limitations under the License.
11184  
11185 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11186 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11187 */
11188  
11189 /*
11190 The hash for ZopfliFindLongestMatch of lz77.c.
11191 */
11192  
11193 #ifndef ZOPFLI_HASH_H_
11194 #define ZOPFLI_HASH_H_
11195  
11196 /* util.h */
11197 /*
11198 Copyright 2011 Google Inc. All Rights Reserved.
11199  
11200 Licensed under the Apache License, Version 2.0 (the "License");
11201 you may not use this file except in compliance with the License.
11202 You may obtain a copy of the License at
11203  
11204 http://www.apache.org/licenses/LICENSE-2.0
11205  
11206 Unless required by applicable law or agreed to in writing, software
11207 distributed under the License is distributed on an "AS IS" BASIS,
11208 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11209 See the License for the specific language governing permissions and
11210 limitations under the License.
11211  
11212 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11213 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11214 */
11215  
11216 /*
11217 Several utilities, including: #defines to try different compression results,
11218 basic deflate specification values and generic program options.
11219 */
11220  
11221 #ifndef ZOPFLI_UTIL_H_
11222 #define ZOPFLI_UTIL_H_
11223  
11224 #include <string.h>
11225 #include <stdlib.h>
11226  
11227 /* Minimum and maximum length that can be encoded in deflate. */
11228 #define ZOPFLI_MAX_MATCH 258
11229 #define ZOPFLI_MIN_MATCH 3
11230  
11231 /* Number of distinct literal/length and distance symbols in DEFLATE */
11232 #define ZOPFLI_NUM_LL 288
11233 #define ZOPFLI_NUM_D 32
11234  
11235 /*
11236 The window size for deflate. Must be a power of two. This should be 32768, the
11237 maximum possible by the deflate spec. Anything less hurts compression more than
11238 speed.
11239 */
11240 #define ZOPFLI_WINDOW_SIZE 32768
11241  
11242 /*
11243 The window mask used to wrap indices into the window. This is why the
11244 window size must be a power of two.
11245 */
11246 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
11247  
11248 /*
11249 A block structure of huge, non-smart, blocks to divide the input into, to allow
11250 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
11251 The whole compression algorithm, including the smarter block splitting, will
11252 be executed independently on each huge block.
11253 Dividing into huge blocks hurts compression, but not much relative to the size.
11254 Set it to 0 to disable master blocks.
11255 */
11256 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
11257  
11258 /*
11259 Used to initialize costs for example
11260 */
11261 #define ZOPFLI_LARGE_FLOAT 1e30
11262  
11263 /*
11264 For longest match cache. max 256. Uses huge amounts of memory but makes it
11265 faster. Uses this many times three bytes per single byte of the input data.
11266 This is so because longest match finding has to find the exact distance
11267 that belongs to each length for the best lz77 strategy.
11268 Good values: e.g. 5, 8.
11269 */
11270 #define ZOPFLI_CACHE_LENGTH 8
11271  
11272 /*
11273 limit the max hash chain hits for this hash value. This has an effect only
11274 on files where the hash value is the same very often. On these files, this
11275 gives worse compression (the value should ideally be 32768, which is the
11276 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
11277 faster on some specific files.
11278 Good value: e.g. 8192.
11279 */
11280 #define ZOPFLI_MAX_CHAIN_HITS 8192
11281  
11282 /*
11283 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
11284 consumes a lot of memory but speeds it up. No effect on compression size.
11285 */
11286 #define ZOPFLI_LONGEST_MATCH_CACHE
11287  
11288 /*
11289 Enable to remember amount of successive identical bytes in the hash chain for
11290 finding longest match
11291 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
11292 This has no effect on the compression result, and enabling it increases speed.
11293 */
11294 #define ZOPFLI_HASH_SAME
11295  
11296 /*
11297 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
11298 best length so far is long enough. This is way faster for files with lots of
11299 identical bytes, on which the compressor is otherwise too slow. Regular files
11300 are unaffected or maybe a tiny bit slower.
11301 This has no effect on the compression result, only on speed.
11302 */
11303 #define ZOPFLI_HASH_SAME_HASH
11304  
11305 /*
11306 Enable this, to avoid slowness for files which are a repetition of the same
11307 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
11308 the compression result.
11309 */
11310 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
11311  
11312 /*
11313 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
11314 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
11315 varies from file to file.
11316 */
11317 #define ZOPFLI_LAZY_MATCHING
11318  
11319 /*
11320 Appends value to dynamically allocated memory, doubling its allocation size
11321 whenever needed.
11322  
11323 value: the value to append, type T
11324 data: pointer to the dynamic array to append to, type T**
11325 size: pointer to the size of the array to append to, type size_t*. This is the
11326 size that you consider the array to be, not the internal allocation size.
11327 Precondition: allocated size of data is at least a power of two greater than or
11328 equal than *size.
11329 */
11330 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
11331 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11332 if (!((*size) & ((*size) - 1))) {\
11333 /*double alloc size if it's a power of two*/\
11334 void** data_void = reinterpret_cast<void**>(data);\
11335 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
11336 : realloc((*data), (*size) * 2 * sizeof(**data));\
11337 }\
11338 (*data)[(*size)] = (value);\
11339 (*size)++;\
11340 }
11341 #else /* C gives problems with strict-aliasing rules for (void**) cast */
11342 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11343 if (!((*size) & ((*size) - 1))) {\
11344 /*double alloc size if it's a power of two*/\
11345 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
11346 : realloc((*data), (*size) * 2 * sizeof(**data));\
11347 }\
11348 (*data)[(*size)] = (value);\
11349 (*size)++;\
11350 }
11351 #endif
11352  
11353  
11354 #endif /* ZOPFLI_UTIL_H_ */
11355  
11356  
11357 typedef struct ZopfliHash {
11358 int* head; /* Hash value to index of its most recent occurrence. */
11359 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
11360 int* hashval; /* Index to hash value at this index. */
11361 int val; /* Current hash value. */
11362  
11363 #ifdef ZOPFLI_HASH_SAME_HASH
11364 /* Fields with similar purpose as the above hash, but for the second hash with
11365 a value that is calculated differently. */
11366 int* head2; /* Hash value to index of its most recent occurrence. */
11367 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
11368 int* hashval2; /* Index to hash value at this index. */
11369 int val2; /* Current hash value. */
11370 #endif
11371  
11372 #ifdef ZOPFLI_HASH_SAME
11373 unsigned short* same; /* Amount of repetitions of same byte after this .*/
11374 #endif
11375 } ZopfliHash;
11376  
11377 /* Allocates ZopfliHash memory. */
11378 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
11379  
11380 /* Resets all fields of ZopfliHash. */
11381 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
11382  
11383 /* Frees ZopfliHash memory. */
11384 void ZopfliCleanHash(ZopfliHash* h);
11385  
11386 /*
11387 Updates the hash values based on the current position in the array. All calls
11388 to this must be made for consecutive bytes.
11389 */
11390 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
11391 ZopfliHash* h);
11392  
11393 /*
11394 Prepopulates hash:
11395 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
11396 correctly.
11397 */
11398 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
11399 ZopfliHash* h);
11400  
11401 #endif /* ZOPFLI_HASH_H_ */
11402  
11403 /* zopfli.h */
11404 /*
11405 Copyright 2011 Google Inc. All Rights Reserved.
11406  
11407 Licensed under the Apache License, Version 2.0 (the "License");
11408 you may not use this file except in compliance with the License.
11409 You may obtain a copy of the License at
11410  
11411 http://www.apache.org/licenses/LICENSE-2.0
11412  
11413 Unless required by applicable law or agreed to in writing, software
11414 distributed under the License is distributed on an "AS IS" BASIS,
11415 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11416 See the License for the specific language governing permissions and
11417 limitations under the License.
11418  
11419 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11420 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11421 */
11422  
11423 #ifndef ZOPFLI_ZOPFLI_H_
11424 #define ZOPFLI_ZOPFLI_H_
11425  
11426 #include <stddef.h>
11427 #include <stdlib.h> /* for size_t */
11428  
11429 #ifdef __cplusplus
11430 extern "C" {
11431 #endif
11432  
11433 /*
11434 Options used throughout the program.
11435 */
11436 typedef struct ZopfliOptions {
11437 /* Whether to print output */
11438 int verbose;
11439  
11440 /* Whether to print more detailed output */
11441 int verbose_more;
11442  
11443 /*
11444 Maximum amount of times to rerun forward and backward pass to optimize LZ77
11445 compression cost. Good values: 10, 15 for small files, 5 for files over
11446 several MB in size or it will be too slow.
11447 */
11448 int numiterations;
11449  
11450 /*
11451 If true, splits the data in multiple deflate blocks with optimal choice
11452 for the block boundaries. Block splitting gives better compression. Default:
11453 true (1).
11454 */
11455 int blocksplitting;
11456  
11457 /*
11458 No longer used, left for compatibility.
11459 */
11460 int blocksplittinglast;
11461  
11462 /*
11463 Maximum amount of blocks to split into (0 for unlimited, but this can give
11464 extreme results that hurt compression on some files). Default value: 15.
11465 */
11466 int blocksplittingmax;
11467 } ZopfliOptions;
11468  
11469 /* Initializes options with default values. */
11470 void ZopfliInitOptions(ZopfliOptions* options);
11471  
11472 /* Output format */
11473 typedef enum {
11474 ZOPFLI_FORMAT_GZIP,
11475 ZOPFLI_FORMAT_ZLIB,
11476 ZOPFLI_FORMAT_DEFLATE
11477 } ZopfliFormat;
11478  
11479 /*
11480 Compresses according to the given output format and appends the result to the
11481 output.
11482  
11483 options: global program options
11484 output_type: the output format to use
11485 out: pointer to the dynamic output array to which the result is appended. Must
11486 be freed after use
11487 outsize: pointer to the dynamic output array size
11488 */
11489 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
11490 const unsigned char* in, size_t insize,
11491 unsigned char** out, size_t* outsize);
11492  
11493 #ifdef __cplusplus
11494 } // extern "C"
11495 #endif
11496  
11497 #endif /* ZOPFLI_ZOPFLI_H_ */
11498  
11499  
11500 /*
11501 Stores lit/length and dist pairs for LZ77.
11502 Parameter litlens: Contains the literal symbols or length values.
11503 Parameter dists: Contains the distances. A value is 0 to indicate that there is
11504 no dist and the corresponding litlens value is a literal instead of a length.
11505 Parameter size: The size of both the litlens and dists arrays.
11506 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
11507 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
11508  
11509 */
11510 typedef struct ZopfliLZ77Store {
11511 unsigned short* litlens; /* Lit or len. */
11512 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
11513 if > 0: length in corresponding litlens, this is the distance. */
11514 size_t size;
11515  
11516 const unsigned char* data; /* original data */
11517 size_t* pos; /* position in data where this LZ77 command begins */
11518  
11519 unsigned short* ll_symbol;
11520 unsigned short* d_symbol;
11521  
11522 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
11523 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
11524 precise histogram at every N symbols, and the rest can be calculated by
11525 looping through the actual symbols of this chunk. */
11526 size_t* ll_counts;
11527 size_t* d_counts;
11528 } ZopfliLZ77Store;
11529  
11530 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
11531 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
11532 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
11533 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
11534 size_t pos, ZopfliLZ77Store* store);
11535 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
11536 ZopfliLZ77Store* target);
11537 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
11538 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
11539 size_t lstart, size_t lend);
11540 /* Gets the histogram of lit/len and dist symbols in the given range, using the
11541 cumulative histograms, so faster than adding one by one for large range. Does
11542 not add the one end symbol of value 256. */
11543 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
11544 size_t lstart, size_t lend,
11545 size_t* ll_counts, size_t* d_counts);
11546  
11547 /*
11548 Some state information for compressing a block.
11549 This is currently a bit under-used (with mainly only the longest match cache),
11550 but is kept for easy future expansion.
11551 */
11552 typedef struct ZopfliBlockState {
11553 const ZopfliOptions* options;
11554  
11555 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
11556 /* Cache for length/distance pairs found so far. */
11557 ZopfliLongestMatchCache* lmc;
11558 #endif
11559  
11560 /* The start (inclusive) and end (not inclusive) of the current block. */
11561 size_t blockstart;
11562 size_t blockend;
11563 } ZopfliBlockState;
11564  
11565 void ZopfliInitBlockState(const ZopfliOptions* options,
11566 size_t blockstart, size_t blockend, int add_lmc,
11567 ZopfliBlockState* s);
11568 void ZopfliCleanBlockState(ZopfliBlockState* s);
11569  
11570 /*
11571 Finds the longest match (length and corresponding distance) for LZ77
11572 compression.
11573 Even when not using "sublen", it can be more efficient to provide an array,
11574 because only then the caching is used.
11575 array: the data
11576 pos: position in the data to find the match for
11577 size: size of the data
11578 limit: limit length to maximum this value (default should be 258). This allows
11579 finding a shorter dist for that length (= less extra bits). Must be
11580 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
11581 sublen: output array of 259 elements, or null. Has, for each length, the
11582 smallest distance required to reach this length. Only 256 of its 259 values
11583 are used, the first 3 are ignored (the shortest length is 3. It is purely
11584 for convenience that the array is made 3 longer).
11585 */
11586 void ZopfliFindLongestMatch(
11587 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
11588 size_t pos, size_t size, size_t limit,
11589 unsigned short* sublen, unsigned short* distance, unsigned short* length);
11590  
11591 /*
11592 Verifies if length and dist are indeed valid, only used for assertion.
11593 */
11594 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
11595 unsigned short dist, unsigned short length);
11596  
11597 /*
11598 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
11599 with the slow but better "squeeze" implementation.
11600 The result is placed in the ZopfliLZ77Store.
11601 If instart is larger than 0, it uses values before instart as starting
11602 dictionary.
11603 */
11604 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
11605 size_t instart, size_t inend,
11606 ZopfliLZ77Store* store, ZopfliHash* h);
11607  
11608 #endif /* ZOPFLI_LZ77_H_ */
11609  
11610  
11611 /*
11612 Calculates lit/len and dist pairs for given data.
11613 If instart is larger than 0, it uses values before instart as starting
11614 dictionary.
11615 */
11616 void ZopfliLZ77Optimal(ZopfliBlockState *s,
11617 const unsigned char* in, size_t instart, size_t inend,
11618 int numiterations,
11619 ZopfliLZ77Store* store);
11620  
11621 /*
11622 Does the same as ZopfliLZ77Optimal, but optimized for the fixed tree of the
11623 deflate standard.
11624 The fixed tree never gives the best compression. But this gives the best
11625 possible LZ77 encoding possible with the fixed tree.
11626 This does not create or output any fixed tree, only LZ77 data optimized for
11627 using with a fixed tree.
11628 If instart is larger than 0, it uses values before instart as starting
11629 dictionary.
11630 */
11631 void ZopfliLZ77OptimalFixed(ZopfliBlockState *s,
11632 const unsigned char* in,
11633 size_t instart, size_t inend,
11634 ZopfliLZ77Store* store);
11635  
11636 #endif /* ZOPFLI_SQUEEZE_H_ */
11637  
11638  
11639 #include <assert.h>
11640 #include <math.h>
11641 #include <stdio.h>
11642  
11643 /* blocksplitter.h */
11644 /*
11645 Copyright 2011 Google Inc. All Rights Reserved.
11646  
11647 Licensed under the Apache License, Version 2.0 (the "License");
11648 you may not use this file except in compliance with the License.
11649 You may obtain a copy of the License at
11650  
11651 http://www.apache.org/licenses/LICENSE-2.0
11652  
11653 Unless required by applicable law or agreed to in writing, software
11654 distributed under the License is distributed on an "AS IS" BASIS,
11655 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11656 See the License for the specific language governing permissions and
11657 limitations under the License.
11658  
11659 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11660 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11661 */
11662  
11663 /*
11664 Functions to choose good boundaries for block splitting. Deflate allows encoding
11665 the data in multiple blocks, with a separate Huffman tree for each block. The
11666 Huffman tree itself requires some bytes to encode, so by choosing certain
11667 blocks, you can either hurt, or enhance compression. These functions choose good
11668 ones that enhance it.
11669 */
11670  
11671 #ifndef ZOPFLI_BLOCKSPLITTER_H_
11672 #define ZOPFLI_BLOCKSPLITTER_H_
11673  
11674 #include <stdlib.h>
11675  
11676 /* lz77.h */
11677 /*
11678 Copyright 2011 Google Inc. All Rights Reserved.
11679  
11680 Licensed under the Apache License, Version 2.0 (the "License");
11681 you may not use this file except in compliance with the License.
11682 You may obtain a copy of the License at
11683  
11684 http://www.apache.org/licenses/LICENSE-2.0
11685  
11686 Unless required by applicable law or agreed to in writing, software
11687 distributed under the License is distributed on an "AS IS" BASIS,
11688 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11689 See the License for the specific language governing permissions and
11690 limitations under the License.
11691  
11692 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11693 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11694 */
11695  
11696 /*
11697 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
11698 compression.
11699 */
11700  
11701 #ifndef ZOPFLI_LZ77_H_
11702 #define ZOPFLI_LZ77_H_
11703  
11704 #include <stdlib.h>
11705  
11706 /* cache.h */
11707 /*
11708 Copyright 2011 Google Inc. All Rights Reserved.
11709  
11710 Licensed under the Apache License, Version 2.0 (the "License");
11711 you may not use this file except in compliance with the License.
11712 You may obtain a copy of the License at
11713  
11714 http://www.apache.org/licenses/LICENSE-2.0
11715  
11716 Unless required by applicable law or agreed to in writing, software
11717 distributed under the License is distributed on an "AS IS" BASIS,
11718 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11719 See the License for the specific language governing permissions and
11720 limitations under the License.
11721  
11722 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11723 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11724 */
11725  
11726 /*
11727 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
11728 */
11729  
11730 #ifndef ZOPFLI_CACHE_H_
11731 #define ZOPFLI_CACHE_H_
11732  
11733 /* util.h */
11734 /*
11735 Copyright 2011 Google Inc. All Rights Reserved.
11736  
11737 Licensed under the Apache License, Version 2.0 (the "License");
11738 you may not use this file except in compliance with the License.
11739 You may obtain a copy of the License at
11740  
11741 http://www.apache.org/licenses/LICENSE-2.0
11742  
11743 Unless required by applicable law or agreed to in writing, software
11744 distributed under the License is distributed on an "AS IS" BASIS,
11745 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11746 See the License for the specific language governing permissions and
11747 limitations under the License.
11748  
11749 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11750 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11751 */
11752  
11753 /*
11754 Several utilities, including: #defines to try different compression results,
11755 basic deflate specification values and generic program options.
11756 */
11757  
11758 #ifndef ZOPFLI_UTIL_H_
11759 #define ZOPFLI_UTIL_H_
11760  
11761 #include <string.h>
11762 #include <stdlib.h>
11763  
11764 /* Minimum and maximum length that can be encoded in deflate. */
11765 #define ZOPFLI_MAX_MATCH 258
11766 #define ZOPFLI_MIN_MATCH 3
11767  
11768 /* Number of distinct literal/length and distance symbols in DEFLATE */
11769 #define ZOPFLI_NUM_LL 288
11770 #define ZOPFLI_NUM_D 32
11771  
11772 /*
11773 The window size for deflate. Must be a power of two. This should be 32768, the
11774 maximum possible by the deflate spec. Anything less hurts compression more than
11775 speed.
11776 */
11777 #define ZOPFLI_WINDOW_SIZE 32768
11778  
11779 /*
11780 The window mask used to wrap indices into the window. This is why the
11781 window size must be a power of two.
11782 */
11783 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
11784  
11785 /*
11786 A block structure of huge, non-smart, blocks to divide the input into, to allow
11787 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
11788 The whole compression algorithm, including the smarter block splitting, will
11789 be executed independently on each huge block.
11790 Dividing into huge blocks hurts compression, but not much relative to the size.
11791 Set it to 0 to disable master blocks.
11792 */
11793 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
11794  
11795 /*
11796 Used to initialize costs for example
11797 */
11798 #define ZOPFLI_LARGE_FLOAT 1e30
11799  
11800 /*
11801 For longest match cache. max 256. Uses huge amounts of memory but makes it
11802 faster. Uses this many times three bytes per single byte of the input data.
11803 This is so because longest match finding has to find the exact distance
11804 that belongs to each length for the best lz77 strategy.
11805 Good values: e.g. 5, 8.
11806 */
11807 #define ZOPFLI_CACHE_LENGTH 8
11808  
11809 /*
11810 limit the max hash chain hits for this hash value. This has an effect only
11811 on files where the hash value is the same very often. On these files, this
11812 gives worse compression (the value should ideally be 32768, which is the
11813 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
11814 faster on some specific files.
11815 Good value: e.g. 8192.
11816 */
11817 #define ZOPFLI_MAX_CHAIN_HITS 8192
11818  
11819 /*
11820 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
11821 consumes a lot of memory but speeds it up. No effect on compression size.
11822 */
11823 #define ZOPFLI_LONGEST_MATCH_CACHE
11824  
11825 /*
11826 Enable to remember amount of successive identical bytes in the hash chain for
11827 finding longest match
11828 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
11829 This has no effect on the compression result, and enabling it increases speed.
11830 */
11831 #define ZOPFLI_HASH_SAME
11832  
11833 /*
11834 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
11835 best length so far is long enough. This is way faster for files with lots of
11836 identical bytes, on which the compressor is otherwise too slow. Regular files
11837 are unaffected or maybe a tiny bit slower.
11838 This has no effect on the compression result, only on speed.
11839 */
11840 #define ZOPFLI_HASH_SAME_HASH
11841  
11842 /*
11843 Enable this, to avoid slowness for files which are a repetition of the same
11844 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
11845 the compression result.
11846 */
11847 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
11848  
11849 /*
11850 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
11851 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
11852 varies from file to file.
11853 */
11854 #define ZOPFLI_LAZY_MATCHING
11855  
11856 /*
11857 Appends value to dynamically allocated memory, doubling its allocation size
11858 whenever needed.
11859  
11860 value: the value to append, type T
11861 data: pointer to the dynamic array to append to, type T**
11862 size: pointer to the size of the array to append to, type size_t*. This is the
11863 size that you consider the array to be, not the internal allocation size.
11864 Precondition: allocated size of data is at least a power of two greater than or
11865 equal than *size.
11866 */
11867 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
11868 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11869 if (!((*size) & ((*size) - 1))) {\
11870 /*double alloc size if it's a power of two*/\
11871 void** data_void = reinterpret_cast<void**>(data);\
11872 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
11873 : realloc((*data), (*size) * 2 * sizeof(**data));\
11874 }\
11875 (*data)[(*size)] = (value);\
11876 (*size)++;\
11877 }
11878 #else /* C gives problems with strict-aliasing rules for (void**) cast */
11879 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
11880 if (!((*size) & ((*size) - 1))) {\
11881 /*double alloc size if it's a power of two*/\
11882 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
11883 : realloc((*data), (*size) * 2 * sizeof(**data));\
11884 }\
11885 (*data)[(*size)] = (value);\
11886 (*size)++;\
11887 }
11888 #endif
11889  
11890  
11891 #endif /* ZOPFLI_UTIL_H_ */
11892  
11893  
11894 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
11895  
11896 /*
11897 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
11898 values.
11899 This is needed because the squeeze runs will ask these values multiple times for
11900 the same position.
11901 Uses large amounts of memory, since it has to remember the distance belonging
11902 to every possible shorter-than-the-best length (the so called "sublen" array).
11903 */
11904 typedef struct ZopfliLongestMatchCache {
11905 unsigned short* length;
11906 unsigned short* dist;
11907 unsigned char* sublen;
11908 } ZopfliLongestMatchCache;
11909  
11910 /* Initializes the ZopfliLongestMatchCache. */
11911 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
11912  
11913 /* Frees up the memory of the ZopfliLongestMatchCache. */
11914 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
11915  
11916 /* Stores sublen array in the cache. */
11917 void ZopfliSublenToCache(const unsigned short* sublen,
11918 size_t pos, size_t length,
11919 ZopfliLongestMatchCache* lmc);
11920  
11921 /* Extracts sublen array from the cache. */
11922 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
11923 size_t pos, size_t length,
11924 unsigned short* sublen);
11925 /* Returns the length up to which could be stored in the cache. */
11926 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
11927 size_t pos, size_t length);
11928  
11929 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
11930  
11931 #endif /* ZOPFLI_CACHE_H_ */
11932  
11933 /* hash.h */
11934 /*
11935 Copyright 2011 Google Inc. All Rights Reserved.
11936  
11937 Licensed under the Apache License, Version 2.0 (the "License");
11938 you may not use this file except in compliance with the License.
11939 You may obtain a copy of the License at
11940  
11941 http://www.apache.org/licenses/LICENSE-2.0
11942  
11943 Unless required by applicable law or agreed to in writing, software
11944 distributed under the License is distributed on an "AS IS" BASIS,
11945 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11946 See the License for the specific language governing permissions and
11947 limitations under the License.
11948  
11949 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11950 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11951 */
11952  
11953 /*
11954 The hash for ZopfliFindLongestMatch of lz77.c.
11955 */
11956  
11957 #ifndef ZOPFLI_HASH_H_
11958 #define ZOPFLI_HASH_H_
11959  
11960 /* util.h */
11961 /*
11962 Copyright 2011 Google Inc. All Rights Reserved.
11963  
11964 Licensed under the Apache License, Version 2.0 (the "License");
11965 you may not use this file except in compliance with the License.
11966 You may obtain a copy of the License at
11967  
11968 http://www.apache.org/licenses/LICENSE-2.0
11969  
11970 Unless required by applicable law or agreed to in writing, software
11971 distributed under the License is distributed on an "AS IS" BASIS,
11972 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11973 See the License for the specific language governing permissions and
11974 limitations under the License.
11975  
11976 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
11977 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
11978 */
11979  
11980 /*
11981 Several utilities, including: #defines to try different compression results,
11982 basic deflate specification values and generic program options.
11983 */
11984  
11985 #ifndef ZOPFLI_UTIL_H_
11986 #define ZOPFLI_UTIL_H_
11987  
11988 #include <string.h>
11989 #include <stdlib.h>
11990  
11991 /* Minimum and maximum length that can be encoded in deflate. */
11992 #define ZOPFLI_MAX_MATCH 258
11993 #define ZOPFLI_MIN_MATCH 3
11994  
11995 /* Number of distinct literal/length and distance symbols in DEFLATE */
11996 #define ZOPFLI_NUM_LL 288
11997 #define ZOPFLI_NUM_D 32
11998  
11999 /*
12000 The window size for deflate. Must be a power of two. This should be 32768, the
12001 maximum possible by the deflate spec. Anything less hurts compression more than
12002 speed.
12003 */
12004 #define ZOPFLI_WINDOW_SIZE 32768
12005  
12006 /*
12007 The window mask used to wrap indices into the window. This is why the
12008 window size must be a power of two.
12009 */
12010 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
12011  
12012 /*
12013 A block structure of huge, non-smart, blocks to divide the input into, to allow
12014 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
12015 The whole compression algorithm, including the smarter block splitting, will
12016 be executed independently on each huge block.
12017 Dividing into huge blocks hurts compression, but not much relative to the size.
12018 Set it to 0 to disable master blocks.
12019 */
12020 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
12021  
12022 /*
12023 Used to initialize costs for example
12024 */
12025 #define ZOPFLI_LARGE_FLOAT 1e30
12026  
12027 /*
12028 For longest match cache. max 256. Uses huge amounts of memory but makes it
12029 faster. Uses this many times three bytes per single byte of the input data.
12030 This is so because longest match finding has to find the exact distance
12031 that belongs to each length for the best lz77 strategy.
12032 Good values: e.g. 5, 8.
12033 */
12034 #define ZOPFLI_CACHE_LENGTH 8
12035  
12036 /*
12037 limit the max hash chain hits for this hash value. This has an effect only
12038 on files where the hash value is the same very often. On these files, this
12039 gives worse compression (the value should ideally be 32768, which is the
12040 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
12041 faster on some specific files.
12042 Good value: e.g. 8192.
12043 */
12044 #define ZOPFLI_MAX_CHAIN_HITS 8192
12045  
12046 /*
12047 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
12048 consumes a lot of memory but speeds it up. No effect on compression size.
12049 */
12050 #define ZOPFLI_LONGEST_MATCH_CACHE
12051  
12052 /*
12053 Enable to remember amount of successive identical bytes in the hash chain for
12054 finding longest match
12055 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
12056 This has no effect on the compression result, and enabling it increases speed.
12057 */
12058 #define ZOPFLI_HASH_SAME
12059  
12060 /*
12061 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
12062 best length so far is long enough. This is way faster for files with lots of
12063 identical bytes, on which the compressor is otherwise too slow. Regular files
12064 are unaffected or maybe a tiny bit slower.
12065 This has no effect on the compression result, only on speed.
12066 */
12067 #define ZOPFLI_HASH_SAME_HASH
12068  
12069 /*
12070 Enable this, to avoid slowness for files which are a repetition of the same
12071 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
12072 the compression result.
12073 */
12074 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
12075  
12076 /*
12077 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
12078 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
12079 varies from file to file.
12080 */
12081 #define ZOPFLI_LAZY_MATCHING
12082  
12083 /*
12084 Appends value to dynamically allocated memory, doubling its allocation size
12085 whenever needed.
12086  
12087 value: the value to append, type T
12088 data: pointer to the dynamic array to append to, type T**
12089 size: pointer to the size of the array to append to, type size_t*. This is the
12090 size that you consider the array to be, not the internal allocation size.
12091 Precondition: allocated size of data is at least a power of two greater than or
12092 equal than *size.
12093 */
12094 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
12095 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12096 if (!((*size) & ((*size) - 1))) {\
12097 /*double alloc size if it's a power of two*/\
12098 void** data_void = reinterpret_cast<void**>(data);\
12099 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
12100 : realloc((*data), (*size) * 2 * sizeof(**data));\
12101 }\
12102 (*data)[(*size)] = (value);\
12103 (*size)++;\
12104 }
12105 #else /* C gives problems with strict-aliasing rules for (void**) cast */
12106 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12107 if (!((*size) & ((*size) - 1))) {\
12108 /*double alloc size if it's a power of two*/\
12109 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
12110 : realloc((*data), (*size) * 2 * sizeof(**data));\
12111 }\
12112 (*data)[(*size)] = (value);\
12113 (*size)++;\
12114 }
12115 #endif
12116  
12117  
12118 #endif /* ZOPFLI_UTIL_H_ */
12119  
12120  
12121 typedef struct ZopfliHash {
12122 int* head; /* Hash value to index of its most recent occurrence. */
12123 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
12124 int* hashval; /* Index to hash value at this index. */
12125 int val; /* Current hash value. */
12126  
12127 #ifdef ZOPFLI_HASH_SAME_HASH
12128 /* Fields with similar purpose as the above hash, but for the second hash with
12129 a value that is calculated differently. */
12130 int* head2; /* Hash value to index of its most recent occurrence. */
12131 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
12132 int* hashval2; /* Index to hash value at this index. */
12133 int val2; /* Current hash value. */
12134 #endif
12135  
12136 #ifdef ZOPFLI_HASH_SAME
12137 unsigned short* same; /* Amount of repetitions of same byte after this .*/
12138 #endif
12139 } ZopfliHash;
12140  
12141 /* Allocates ZopfliHash memory. */
12142 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
12143  
12144 /* Resets all fields of ZopfliHash. */
12145 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
12146  
12147 /* Frees ZopfliHash memory. */
12148 void ZopfliCleanHash(ZopfliHash* h);
12149  
12150 /*
12151 Updates the hash values based on the current position in the array. All calls
12152 to this must be made for consecutive bytes.
12153 */
12154 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
12155 ZopfliHash* h);
12156  
12157 /*
12158 Prepopulates hash:
12159 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
12160 correctly.
12161 */
12162 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
12163 ZopfliHash* h);
12164  
12165 #endif /* ZOPFLI_HASH_H_ */
12166  
12167 /* zopfli.h */
12168 /*
12169 Copyright 2011 Google Inc. All Rights Reserved.
12170  
12171 Licensed under the Apache License, Version 2.0 (the "License");
12172 you may not use this file except in compliance with the License.
12173 You may obtain a copy of the License at
12174  
12175 http://www.apache.org/licenses/LICENSE-2.0
12176  
12177 Unless required by applicable law or agreed to in writing, software
12178 distributed under the License is distributed on an "AS IS" BASIS,
12179 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12180 See the License for the specific language governing permissions and
12181 limitations under the License.
12182  
12183 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12184 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12185 */
12186  
12187 #ifndef ZOPFLI_ZOPFLI_H_
12188 #define ZOPFLI_ZOPFLI_H_
12189  
12190 #include <stddef.h>
12191 #include <stdlib.h> /* for size_t */
12192  
12193 #ifdef __cplusplus
12194 extern "C" {
12195 #endif
12196  
12197 /*
12198 Options used throughout the program.
12199 */
12200 typedef struct ZopfliOptions {
12201 /* Whether to print output */
12202 int verbose;
12203  
12204 /* Whether to print more detailed output */
12205 int verbose_more;
12206  
12207 /*
12208 Maximum amount of times to rerun forward and backward pass to optimize LZ77
12209 compression cost. Good values: 10, 15 for small files, 5 for files over
12210 several MB in size or it will be too slow.
12211 */
12212 int numiterations;
12213  
12214 /*
12215 If true, splits the data in multiple deflate blocks with optimal choice
12216 for the block boundaries. Block splitting gives better compression. Default:
12217 true (1).
12218 */
12219 int blocksplitting;
12220  
12221 /*
12222 No longer used, left for compatibility.
12223 */
12224 int blocksplittinglast;
12225  
12226 /*
12227 Maximum amount of blocks to split into (0 for unlimited, but this can give
12228 extreme results that hurt compression on some files). Default value: 15.
12229 */
12230 int blocksplittingmax;
12231 } ZopfliOptions;
12232  
12233 /* Initializes options with default values. */
12234 void ZopfliInitOptions(ZopfliOptions* options);
12235  
12236 /* Output format */
12237 typedef enum {
12238 ZOPFLI_FORMAT_GZIP,
12239 ZOPFLI_FORMAT_ZLIB,
12240 ZOPFLI_FORMAT_DEFLATE
12241 } ZopfliFormat;
12242  
12243 /*
12244 Compresses according to the given output format and appends the result to the
12245 output.
12246  
12247 options: global program options
12248 output_type: the output format to use
12249 out: pointer to the dynamic output array to which the result is appended. Must
12250 be freed after use
12251 outsize: pointer to the dynamic output array size
12252 */
12253 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
12254 const unsigned char* in, size_t insize,
12255 unsigned char** out, size_t* outsize);
12256  
12257 #ifdef __cplusplus
12258 } // extern "C"
12259 #endif
12260  
12261 #endif /* ZOPFLI_ZOPFLI_H_ */
12262  
12263  
12264 /*
12265 Stores lit/length and dist pairs for LZ77.
12266 Parameter litlens: Contains the literal symbols or length values.
12267 Parameter dists: Contains the distances. A value is 0 to indicate that there is
12268 no dist and the corresponding litlens value is a literal instead of a length.
12269 Parameter size: The size of both the litlens and dists arrays.
12270 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
12271 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
12272  
12273 */
12274 typedef struct ZopfliLZ77Store {
12275 unsigned short* litlens; /* Lit or len. */
12276 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
12277 if > 0: length in corresponding litlens, this is the distance. */
12278 size_t size;
12279  
12280 const unsigned char* data; /* original data */
12281 size_t* pos; /* position in data where this LZ77 command begins */
12282  
12283 unsigned short* ll_symbol;
12284 unsigned short* d_symbol;
12285  
12286 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
12287 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
12288 precise histogram at every N symbols, and the rest can be calculated by
12289 looping through the actual symbols of this chunk. */
12290 size_t* ll_counts;
12291 size_t* d_counts;
12292 } ZopfliLZ77Store;
12293  
12294 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
12295 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
12296 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
12297 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
12298 size_t pos, ZopfliLZ77Store* store);
12299 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
12300 ZopfliLZ77Store* target);
12301 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
12302 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
12303 size_t lstart, size_t lend);
12304 /* Gets the histogram of lit/len and dist symbols in the given range, using the
12305 cumulative histograms, so faster than adding one by one for large range. Does
12306 not add the one end symbol of value 256. */
12307 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
12308 size_t lstart, size_t lend,
12309 size_t* ll_counts, size_t* d_counts);
12310  
12311 /*
12312 Some state information for compressing a block.
12313 This is currently a bit under-used (with mainly only the longest match cache),
12314 but is kept for easy future expansion.
12315 */
12316 typedef struct ZopfliBlockState {
12317 const ZopfliOptions* options;
12318  
12319 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
12320 /* Cache for length/distance pairs found so far. */
12321 ZopfliLongestMatchCache* lmc;
12322 #endif
12323  
12324 /* The start (inclusive) and end (not inclusive) of the current block. */
12325 size_t blockstart;
12326 size_t blockend;
12327 } ZopfliBlockState;
12328  
12329 void ZopfliInitBlockState(const ZopfliOptions* options,
12330 size_t blockstart, size_t blockend, int add_lmc,
12331 ZopfliBlockState* s);
12332 void ZopfliCleanBlockState(ZopfliBlockState* s);
12333  
12334 /*
12335 Finds the longest match (length and corresponding distance) for LZ77
12336 compression.
12337 Even when not using "sublen", it can be more efficient to provide an array,
12338 because only then the caching is used.
12339 array: the data
12340 pos: position in the data to find the match for
12341 size: size of the data
12342 limit: limit length to maximum this value (default should be 258). This allows
12343 finding a shorter dist for that length (= less extra bits). Must be
12344 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
12345 sublen: output array of 259 elements, or null. Has, for each length, the
12346 smallest distance required to reach this length. Only 256 of its 259 values
12347 are used, the first 3 are ignored (the shortest length is 3. It is purely
12348 for convenience that the array is made 3 longer).
12349 */
12350 void ZopfliFindLongestMatch(
12351 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
12352 size_t pos, size_t size, size_t limit,
12353 unsigned short* sublen, unsigned short* distance, unsigned short* length);
12354  
12355 /*
12356 Verifies if length and dist are indeed valid, only used for assertion.
12357 */
12358 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
12359 unsigned short dist, unsigned short length);
12360  
12361 /*
12362 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
12363 with the slow but better "squeeze" implementation.
12364 The result is placed in the ZopfliLZ77Store.
12365 If instart is larger than 0, it uses values before instart as starting
12366 dictionary.
12367 */
12368 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
12369 size_t instart, size_t inend,
12370 ZopfliLZ77Store* store, ZopfliHash* h);
12371  
12372 #endif /* ZOPFLI_LZ77_H_ */
12373  
12374 /* zopfli.h */
12375 /*
12376 Copyright 2011 Google Inc. All Rights Reserved.
12377  
12378 Licensed under the Apache License, Version 2.0 (the "License");
12379 you may not use this file except in compliance with the License.
12380 You may obtain a copy of the License at
12381  
12382 http://www.apache.org/licenses/LICENSE-2.0
12383  
12384 Unless required by applicable law or agreed to in writing, software
12385 distributed under the License is distributed on an "AS IS" BASIS,
12386 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12387 See the License for the specific language governing permissions and
12388 limitations under the License.
12389  
12390 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12391 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12392 */
12393  
12394 #ifndef ZOPFLI_ZOPFLI_H_
12395 #define ZOPFLI_ZOPFLI_H_
12396  
12397 #include <stddef.h>
12398 #include <stdlib.h> /* for size_t */
12399  
12400 #ifdef __cplusplus
12401 extern "C" {
12402 #endif
12403  
12404 /*
12405 Options used throughout the program.
12406 */
12407 typedef struct ZopfliOptions {
12408 /* Whether to print output */
12409 int verbose;
12410  
12411 /* Whether to print more detailed output */
12412 int verbose_more;
12413  
12414 /*
12415 Maximum amount of times to rerun forward and backward pass to optimize LZ77
12416 compression cost. Good values: 10, 15 for small files, 5 for files over
12417 several MB in size or it will be too slow.
12418 */
12419 int numiterations;
12420  
12421 /*
12422 If true, splits the data in multiple deflate blocks with optimal choice
12423 for the block boundaries. Block splitting gives better compression. Default:
12424 true (1).
12425 */
12426 int blocksplitting;
12427  
12428 /*
12429 No longer used, left for compatibility.
12430 */
12431 int blocksplittinglast;
12432  
12433 /*
12434 Maximum amount of blocks to split into (0 for unlimited, but this can give
12435 extreme results that hurt compression on some files). Default value: 15.
12436 */
12437 int blocksplittingmax;
12438 } ZopfliOptions;
12439  
12440 /* Initializes options with default values. */
12441 void ZopfliInitOptions(ZopfliOptions* options);
12442  
12443 /* Output format */
12444 typedef enum {
12445 ZOPFLI_FORMAT_GZIP,
12446 ZOPFLI_FORMAT_ZLIB,
12447 ZOPFLI_FORMAT_DEFLATE
12448 } ZopfliFormat;
12449  
12450 /*
12451 Compresses according to the given output format and appends the result to the
12452 output.
12453  
12454 options: global program options
12455 output_type: the output format to use
12456 out: pointer to the dynamic output array to which the result is appended. Must
12457 be freed after use
12458 outsize: pointer to the dynamic output array size
12459 */
12460 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
12461 const unsigned char* in, size_t insize,
12462 unsigned char** out, size_t* outsize);
12463  
12464 #ifdef __cplusplus
12465 } // extern "C"
12466 #endif
12467  
12468 #endif /* ZOPFLI_ZOPFLI_H_ */
12469  
12470  
12471  
12472 /*
12473 Does blocksplitting on LZ77 data.
12474 The output splitpoints are indices in the LZ77 data.
12475 maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit.
12476 */
12477 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
12478 const ZopfliLZ77Store* lz77, size_t maxblocks,
12479 size_t** splitpoints, size_t* npoints);
12480  
12481 /*
12482 Does blocksplitting on uncompressed data.
12483 The output splitpoints are indices in the uncompressed bytes.
12484  
12485 options: general program options.
12486 in: uncompressed input data
12487 instart: where to start splitting
12488 inend: where to end splitting (not inclusive)
12489 maxblocks: maximum amount of blocks to split into, or 0 for no limit
12490 splitpoints: dynamic array to put the resulting split point coordinates into.
12491 The coordinates are indices in the input array.
12492 npoints: pointer to amount of splitpoints, for the dynamic array. The amount of
12493 blocks is the amount of splitpoitns + 1.
12494 */
12495 void ZopfliBlockSplit(const ZopfliOptions* options,
12496 const unsigned char* in, size_t instart, size_t inend,
12497 size_t maxblocks, size_t** splitpoints, size_t* npoints);
12498  
12499 /*
12500 Divides the input into equal blocks, does not even take LZ77 lengths into
12501 account.
12502 */
12503 void ZopfliBlockSplitSimple(const unsigned char* in,
12504 size_t instart, size_t inend,
12505 size_t blocksize,
12506 size_t** splitpoints, size_t* npoints);
12507  
12508 #endif /* ZOPFLI_BLOCKSPLITTER_H_ */
12509  
12510 /* deflate.h */
12511 /*
12512 Copyright 2011 Google Inc. All Rights Reserved.
12513  
12514 Licensed under the Apache License, Version 2.0 (the "License");
12515 you may not use this file except in compliance with the License.
12516 You may obtain a copy of the License at
12517  
12518 http://www.apache.org/licenses/LICENSE-2.0
12519  
12520 Unless required by applicable law or agreed to in writing, software
12521 distributed under the License is distributed on an "AS IS" BASIS,
12522 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12523 See the License for the specific language governing permissions and
12524 limitations under the License.
12525  
12526 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12527 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12528 */
12529  
12530 #ifndef ZOPFLI_DEFLATE_H_
12531 #define ZOPFLI_DEFLATE_H_
12532  
12533 /*
12534 Functions to compress according to the DEFLATE specification, using the
12535 "squeeze" LZ77 compression backend.
12536 */
12537  
12538 /* lz77.h */
12539 /*
12540 Copyright 2011 Google Inc. All Rights Reserved.
12541  
12542 Licensed under the Apache License, Version 2.0 (the "License");
12543 you may not use this file except in compliance with the License.
12544 You may obtain a copy of the License at
12545  
12546 http://www.apache.org/licenses/LICENSE-2.0
12547  
12548 Unless required by applicable law or agreed to in writing, software
12549 distributed under the License is distributed on an "AS IS" BASIS,
12550 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12551 See the License for the specific language governing permissions and
12552 limitations under the License.
12553  
12554 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12555 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12556 */
12557  
12558 /*
12559 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
12560 compression.
12561 */
12562  
12563 #ifndef ZOPFLI_LZ77_H_
12564 #define ZOPFLI_LZ77_H_
12565  
12566 #include <stdlib.h>
12567  
12568 /* cache.h */
12569 /*
12570 Copyright 2011 Google Inc. All Rights Reserved.
12571  
12572 Licensed under the Apache License, Version 2.0 (the "License");
12573 you may not use this file except in compliance with the License.
12574 You may obtain a copy of the License at
12575  
12576 http://www.apache.org/licenses/LICENSE-2.0
12577  
12578 Unless required by applicable law or agreed to in writing, software
12579 distributed under the License is distributed on an "AS IS" BASIS,
12580 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12581 See the License for the specific language governing permissions and
12582 limitations under the License.
12583  
12584 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12585 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12586 */
12587  
12588 /*
12589 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
12590 */
12591  
12592 #ifndef ZOPFLI_CACHE_H_
12593 #define ZOPFLI_CACHE_H_
12594  
12595 /* util.h */
12596 /*
12597 Copyright 2011 Google Inc. All Rights Reserved.
12598  
12599 Licensed under the Apache License, Version 2.0 (the "License");
12600 you may not use this file except in compliance with the License.
12601 You may obtain a copy of the License at
12602  
12603 http://www.apache.org/licenses/LICENSE-2.0
12604  
12605 Unless required by applicable law or agreed to in writing, software
12606 distributed under the License is distributed on an "AS IS" BASIS,
12607 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12608 See the License for the specific language governing permissions and
12609 limitations under the License.
12610  
12611 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12612 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12613 */
12614  
12615 /*
12616 Several utilities, including: #defines to try different compression results,
12617 basic deflate specification values and generic program options.
12618 */
12619  
12620 #ifndef ZOPFLI_UTIL_H_
12621 #define ZOPFLI_UTIL_H_
12622  
12623 #include <string.h>
12624 #include <stdlib.h>
12625  
12626 /* Minimum and maximum length that can be encoded in deflate. */
12627 #define ZOPFLI_MAX_MATCH 258
12628 #define ZOPFLI_MIN_MATCH 3
12629  
12630 /* Number of distinct literal/length and distance symbols in DEFLATE */
12631 #define ZOPFLI_NUM_LL 288
12632 #define ZOPFLI_NUM_D 32
12633  
12634 /*
12635 The window size for deflate. Must be a power of two. This should be 32768, the
12636 maximum possible by the deflate spec. Anything less hurts compression more than
12637 speed.
12638 */
12639 #define ZOPFLI_WINDOW_SIZE 32768
12640  
12641 /*
12642 The window mask used to wrap indices into the window. This is why the
12643 window size must be a power of two.
12644 */
12645 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
12646  
12647 /*
12648 A block structure of huge, non-smart, blocks to divide the input into, to allow
12649 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
12650 The whole compression algorithm, including the smarter block splitting, will
12651 be executed independently on each huge block.
12652 Dividing into huge blocks hurts compression, but not much relative to the size.
12653 Set it to 0 to disable master blocks.
12654 */
12655 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
12656  
12657 /*
12658 Used to initialize costs for example
12659 */
12660 #define ZOPFLI_LARGE_FLOAT 1e30
12661  
12662 /*
12663 For longest match cache. max 256. Uses huge amounts of memory but makes it
12664 faster. Uses this many times three bytes per single byte of the input data.
12665 This is so because longest match finding has to find the exact distance
12666 that belongs to each length for the best lz77 strategy.
12667 Good values: e.g. 5, 8.
12668 */
12669 #define ZOPFLI_CACHE_LENGTH 8
12670  
12671 /*
12672 limit the max hash chain hits for this hash value. This has an effect only
12673 on files where the hash value is the same very often. On these files, this
12674 gives worse compression (the value should ideally be 32768, which is the
12675 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
12676 faster on some specific files.
12677 Good value: e.g. 8192.
12678 */
12679 #define ZOPFLI_MAX_CHAIN_HITS 8192
12680  
12681 /*
12682 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
12683 consumes a lot of memory but speeds it up. No effect on compression size.
12684 */
12685 #define ZOPFLI_LONGEST_MATCH_CACHE
12686  
12687 /*
12688 Enable to remember amount of successive identical bytes in the hash chain for
12689 finding longest match
12690 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
12691 This has no effect on the compression result, and enabling it increases speed.
12692 */
12693 #define ZOPFLI_HASH_SAME
12694  
12695 /*
12696 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
12697 best length so far is long enough. This is way faster for files with lots of
12698 identical bytes, on which the compressor is otherwise too slow. Regular files
12699 are unaffected or maybe a tiny bit slower.
12700 This has no effect on the compression result, only on speed.
12701 */
12702 #define ZOPFLI_HASH_SAME_HASH
12703  
12704 /*
12705 Enable this, to avoid slowness for files which are a repetition of the same
12706 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
12707 the compression result.
12708 */
12709 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
12710  
12711 /*
12712 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
12713 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
12714 varies from file to file.
12715 */
12716 #define ZOPFLI_LAZY_MATCHING
12717  
12718 /*
12719 Appends value to dynamically allocated memory, doubling its allocation size
12720 whenever needed.
12721  
12722 value: the value to append, type T
12723 data: pointer to the dynamic array to append to, type T**
12724 size: pointer to the size of the array to append to, type size_t*. This is the
12725 size that you consider the array to be, not the internal allocation size.
12726 Precondition: allocated size of data is at least a power of two greater than or
12727 equal than *size.
12728 */
12729 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
12730 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12731 if (!((*size) & ((*size) - 1))) {\
12732 /*double alloc size if it's a power of two*/\
12733 void** data_void = reinterpret_cast<void**>(data);\
12734 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
12735 : realloc((*data), (*size) * 2 * sizeof(**data));\
12736 }\
12737 (*data)[(*size)] = (value);\
12738 (*size)++;\
12739 }
12740 #else /* C gives problems with strict-aliasing rules for (void**) cast */
12741 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12742 if (!((*size) & ((*size) - 1))) {\
12743 /*double alloc size if it's a power of two*/\
12744 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
12745 : realloc((*data), (*size) * 2 * sizeof(**data));\
12746 }\
12747 (*data)[(*size)] = (value);\
12748 (*size)++;\
12749 }
12750 #endif
12751  
12752  
12753 #endif /* ZOPFLI_UTIL_H_ */
12754  
12755  
12756 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
12757  
12758 /*
12759 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
12760 values.
12761 This is needed because the squeeze runs will ask these values multiple times for
12762 the same position.
12763 Uses large amounts of memory, since it has to remember the distance belonging
12764 to every possible shorter-than-the-best length (the so called "sublen" array).
12765 */
12766 typedef struct ZopfliLongestMatchCache {
12767 unsigned short* length;
12768 unsigned short* dist;
12769 unsigned char* sublen;
12770 } ZopfliLongestMatchCache;
12771  
12772 /* Initializes the ZopfliLongestMatchCache. */
12773 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
12774  
12775 /* Frees up the memory of the ZopfliLongestMatchCache. */
12776 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
12777  
12778 /* Stores sublen array in the cache. */
12779 void ZopfliSublenToCache(const unsigned short* sublen,
12780 size_t pos, size_t length,
12781 ZopfliLongestMatchCache* lmc);
12782  
12783 /* Extracts sublen array from the cache. */
12784 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
12785 size_t pos, size_t length,
12786 unsigned short* sublen);
12787 /* Returns the length up to which could be stored in the cache. */
12788 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
12789 size_t pos, size_t length);
12790  
12791 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
12792  
12793 #endif /* ZOPFLI_CACHE_H_ */
12794  
12795 /* hash.h */
12796 /*
12797 Copyright 2011 Google Inc. All Rights Reserved.
12798  
12799 Licensed under the Apache License, Version 2.0 (the "License");
12800 you may not use this file except in compliance with the License.
12801 You may obtain a copy of the License at
12802  
12803 http://www.apache.org/licenses/LICENSE-2.0
12804  
12805 Unless required by applicable law or agreed to in writing, software
12806 distributed under the License is distributed on an "AS IS" BASIS,
12807 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12808 See the License for the specific language governing permissions and
12809 limitations under the License.
12810  
12811 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12812 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12813 */
12814  
12815 /*
12816 The hash for ZopfliFindLongestMatch of lz77.c.
12817 */
12818  
12819 #ifndef ZOPFLI_HASH_H_
12820 #define ZOPFLI_HASH_H_
12821  
12822 /* util.h */
12823 /*
12824 Copyright 2011 Google Inc. All Rights Reserved.
12825  
12826 Licensed under the Apache License, Version 2.0 (the "License");
12827 you may not use this file except in compliance with the License.
12828 You may obtain a copy of the License at
12829  
12830 http://www.apache.org/licenses/LICENSE-2.0
12831  
12832 Unless required by applicable law or agreed to in writing, software
12833 distributed under the License is distributed on an "AS IS" BASIS,
12834 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12835 See the License for the specific language governing permissions and
12836 limitations under the License.
12837  
12838 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
12839 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
12840 */
12841  
12842 /*
12843 Several utilities, including: #defines to try different compression results,
12844 basic deflate specification values and generic program options.
12845 */
12846  
12847 #ifndef ZOPFLI_UTIL_H_
12848 #define ZOPFLI_UTIL_H_
12849  
12850 #include <string.h>
12851 #include <stdlib.h>
12852  
12853 /* Minimum and maximum length that can be encoded in deflate. */
12854 #define ZOPFLI_MAX_MATCH 258
12855 #define ZOPFLI_MIN_MATCH 3
12856  
12857 /* Number of distinct literal/length and distance symbols in DEFLATE */
12858 #define ZOPFLI_NUM_LL 288
12859 #define ZOPFLI_NUM_D 32
12860  
12861 /*
12862 The window size for deflate. Must be a power of two. This should be 32768, the
12863 maximum possible by the deflate spec. Anything less hurts compression more than
12864 speed.
12865 */
12866 #define ZOPFLI_WINDOW_SIZE 32768
12867  
12868 /*
12869 The window mask used to wrap indices into the window. This is why the
12870 window size must be a power of two.
12871 */
12872 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
12873  
12874 /*
12875 A block structure of huge, non-smart, blocks to divide the input into, to allow
12876 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
12877 The whole compression algorithm, including the smarter block splitting, will
12878 be executed independently on each huge block.
12879 Dividing into huge blocks hurts compression, but not much relative to the size.
12880 Set it to 0 to disable master blocks.
12881 */
12882 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
12883  
12884 /*
12885 Used to initialize costs for example
12886 */
12887 #define ZOPFLI_LARGE_FLOAT 1e30
12888  
12889 /*
12890 For longest match cache. max 256. Uses huge amounts of memory but makes it
12891 faster. Uses this many times three bytes per single byte of the input data.
12892 This is so because longest match finding has to find the exact distance
12893 that belongs to each length for the best lz77 strategy.
12894 Good values: e.g. 5, 8.
12895 */
12896 #define ZOPFLI_CACHE_LENGTH 8
12897  
12898 /*
12899 limit the max hash chain hits for this hash value. This has an effect only
12900 on files where the hash value is the same very often. On these files, this
12901 gives worse compression (the value should ideally be 32768, which is the
12902 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
12903 faster on some specific files.
12904 Good value: e.g. 8192.
12905 */
12906 #define ZOPFLI_MAX_CHAIN_HITS 8192
12907  
12908 /*
12909 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
12910 consumes a lot of memory but speeds it up. No effect on compression size.
12911 */
12912 #define ZOPFLI_LONGEST_MATCH_CACHE
12913  
12914 /*
12915 Enable to remember amount of successive identical bytes in the hash chain for
12916 finding longest match
12917 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
12918 This has no effect on the compression result, and enabling it increases speed.
12919 */
12920 #define ZOPFLI_HASH_SAME
12921  
12922 /*
12923 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
12924 best length so far is long enough. This is way faster for files with lots of
12925 identical bytes, on which the compressor is otherwise too slow. Regular files
12926 are unaffected or maybe a tiny bit slower.
12927 This has no effect on the compression result, only on speed.
12928 */
12929 #define ZOPFLI_HASH_SAME_HASH
12930  
12931 /*
12932 Enable this, to avoid slowness for files which are a repetition of the same
12933 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
12934 the compression result.
12935 */
12936 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
12937  
12938 /*
12939 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
12940 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
12941 varies from file to file.
12942 */
12943 #define ZOPFLI_LAZY_MATCHING
12944  
12945 /*
12946 Appends value to dynamically allocated memory, doubling its allocation size
12947 whenever needed.
12948  
12949 value: the value to append, type T
12950 data: pointer to the dynamic array to append to, type T**
12951 size: pointer to the size of the array to append to, type size_t*. This is the
12952 size that you consider the array to be, not the internal allocation size.
12953 Precondition: allocated size of data is at least a power of two greater than or
12954 equal than *size.
12955 */
12956 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
12957 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12958 if (!((*size) & ((*size) - 1))) {\
12959 /*double alloc size if it's a power of two*/\
12960 void** data_void = reinterpret_cast<void**>(data);\
12961 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
12962 : realloc((*data), (*size) * 2 * sizeof(**data));\
12963 }\
12964 (*data)[(*size)] = (value);\
12965 (*size)++;\
12966 }
12967 #else /* C gives problems with strict-aliasing rules for (void**) cast */
12968 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
12969 if (!((*size) & ((*size) - 1))) {\
12970 /*double alloc size if it's a power of two*/\
12971 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
12972 : realloc((*data), (*size) * 2 * sizeof(**data));\
12973 }\
12974 (*data)[(*size)] = (value);\
12975 (*size)++;\
12976 }
12977 #endif
12978  
12979  
12980 #endif /* ZOPFLI_UTIL_H_ */
12981  
12982  
12983 typedef struct ZopfliHash {
12984 int* head; /* Hash value to index of its most recent occurrence. */
12985 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
12986 int* hashval; /* Index to hash value at this index. */
12987 int val; /* Current hash value. */
12988  
12989 #ifdef ZOPFLI_HASH_SAME_HASH
12990 /* Fields with similar purpose as the above hash, but for the second hash with
12991 a value that is calculated differently. */
12992 int* head2; /* Hash value to index of its most recent occurrence. */
12993 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
12994 int* hashval2; /* Index to hash value at this index. */
12995 int val2; /* Current hash value. */
12996 #endif
12997  
12998 #ifdef ZOPFLI_HASH_SAME
12999 unsigned short* same; /* Amount of repetitions of same byte after this .*/
13000 #endif
13001 } ZopfliHash;
13002  
13003 /* Allocates ZopfliHash memory. */
13004 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
13005  
13006 /* Resets all fields of ZopfliHash. */
13007 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
13008  
13009 /* Frees ZopfliHash memory. */
13010 void ZopfliCleanHash(ZopfliHash* h);
13011  
13012 /*
13013 Updates the hash values based on the current position in the array. All calls
13014 to this must be made for consecutive bytes.
13015 */
13016 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
13017 ZopfliHash* h);
13018  
13019 /*
13020 Prepopulates hash:
13021 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
13022 correctly.
13023 */
13024 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
13025 ZopfliHash* h);
13026  
13027 #endif /* ZOPFLI_HASH_H_ */
13028  
13029 /* zopfli.h */
13030 /*
13031 Copyright 2011 Google Inc. All Rights Reserved.
13032  
13033 Licensed under the Apache License, Version 2.0 (the "License");
13034 you may not use this file except in compliance with the License.
13035 You may obtain a copy of the License at
13036  
13037 http://www.apache.org/licenses/LICENSE-2.0
13038  
13039 Unless required by applicable law or agreed to in writing, software
13040 distributed under the License is distributed on an "AS IS" BASIS,
13041 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13042 See the License for the specific language governing permissions and
13043 limitations under the License.
13044  
13045 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
13046 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
13047 */
13048  
13049 #ifndef ZOPFLI_ZOPFLI_H_
13050 #define ZOPFLI_ZOPFLI_H_
13051  
13052 #include <stddef.h>
13053 #include <stdlib.h> /* for size_t */
13054  
13055 #ifdef __cplusplus
13056 extern "C" {
13057 #endif
13058  
13059 /*
13060 Options used throughout the program.
13061 */
13062 typedef struct ZopfliOptions {
13063 /* Whether to print output */
13064 int verbose;
13065  
13066 /* Whether to print more detailed output */
13067 int verbose_more;
13068  
13069 /*
13070 Maximum amount of times to rerun forward and backward pass to optimize LZ77
13071 compression cost. Good values: 10, 15 for small files, 5 for files over
13072 several MB in size or it will be too slow.
13073 */
13074 int numiterations;
13075  
13076 /*
13077 If true, splits the data in multiple deflate blocks with optimal choice
13078 for the block boundaries. Block splitting gives better compression. Default:
13079 true (1).
13080 */
13081 int blocksplitting;
13082  
13083 /*
13084 No longer used, left for compatibility.
13085 */
13086 int blocksplittinglast;
13087  
13088 /*
13089 Maximum amount of blocks to split into (0 for unlimited, but this can give
13090 extreme results that hurt compression on some files). Default value: 15.
13091 */
13092 int blocksplittingmax;
13093 } ZopfliOptions;
13094  
13095 /* Initializes options with default values. */
13096 void ZopfliInitOptions(ZopfliOptions* options);
13097  
13098 /* Output format */
13099 typedef enum {
13100 ZOPFLI_FORMAT_GZIP,
13101 ZOPFLI_FORMAT_ZLIB,
13102 ZOPFLI_FORMAT_DEFLATE
13103 } ZopfliFormat;
13104  
13105 /*
13106 Compresses according to the given output format and appends the result to the
13107 output.
13108  
13109 options: global program options
13110 output_type: the output format to use
13111 out: pointer to the dynamic output array to which the result is appended. Must
13112 be freed after use
13113 outsize: pointer to the dynamic output array size
13114 */
13115 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
13116 const unsigned char* in, size_t insize,
13117 unsigned char** out, size_t* outsize);
13118  
13119 #ifdef __cplusplus
13120 } // extern "C"
13121 #endif
13122  
13123 #endif /* ZOPFLI_ZOPFLI_H_ */
13124  
13125  
13126 /*
13127 Stores lit/length and dist pairs for LZ77.
13128 Parameter litlens: Contains the literal symbols or length values.
13129 Parameter dists: Contains the distances. A value is 0 to indicate that there is
13130 no dist and the corresponding litlens value is a literal instead of a length.
13131 Parameter size: The size of both the litlens and dists arrays.
13132 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
13133 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
13134  
13135 */
13136 typedef struct ZopfliLZ77Store {
13137 unsigned short* litlens; /* Lit or len. */
13138 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
13139 if > 0: length in corresponding litlens, this is the distance. */
13140 size_t size;
13141  
13142 const unsigned char* data; /* original data */
13143 size_t* pos; /* position in data where this LZ77 command begins */
13144  
13145 unsigned short* ll_symbol;
13146 unsigned short* d_symbol;
13147  
13148 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
13149 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
13150 precise histogram at every N symbols, and the rest can be calculated by
13151 looping through the actual symbols of this chunk. */
13152 size_t* ll_counts;
13153 size_t* d_counts;
13154 } ZopfliLZ77Store;
13155  
13156 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
13157 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
13158 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
13159 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
13160 size_t pos, ZopfliLZ77Store* store);
13161 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
13162 ZopfliLZ77Store* target);
13163 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
13164 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
13165 size_t lstart, size_t lend);
13166 /* Gets the histogram of lit/len and dist symbols in the given range, using the
13167 cumulative histograms, so faster than adding one by one for large range. Does
13168 not add the one end symbol of value 256. */
13169 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
13170 size_t lstart, size_t lend,
13171 size_t* ll_counts, size_t* d_counts);
13172  
13173 /*
13174 Some state information for compressing a block.
13175 This is currently a bit under-used (with mainly only the longest match cache),
13176 but is kept for easy future expansion.
13177 */
13178 typedef struct ZopfliBlockState {
13179 const ZopfliOptions* options;
13180  
13181 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
13182 /* Cache for length/distance pairs found so far. */
13183 ZopfliLongestMatchCache* lmc;
13184 #endif
13185  
13186 /* The start (inclusive) and end (not inclusive) of the current block. */
13187 size_t blockstart;
13188 size_t blockend;
13189 } ZopfliBlockState;
13190  
13191 void ZopfliInitBlockState(const ZopfliOptions* options,
13192 size_t blockstart, size_t blockend, int add_lmc,
13193 ZopfliBlockState* s);
13194 void ZopfliCleanBlockState(ZopfliBlockState* s);
13195  
13196 /*
13197 Finds the longest match (length and corresponding distance) for LZ77
13198 compression.
13199 Even when not using "sublen", it can be more efficient to provide an array,
13200 because only then the caching is used.
13201 array: the data
13202 pos: position in the data to find the match for
13203 size: size of the data
13204 limit: limit length to maximum this value (default should be 258). This allows
13205 finding a shorter dist for that length (= less extra bits). Must be
13206 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
13207 sublen: output array of 259 elements, or null. Has, for each length, the
13208 smallest distance required to reach this length. Only 256 of its 259 values
13209 are used, the first 3 are ignored (the shortest length is 3. It is purely
13210 for convenience that the array is made 3 longer).
13211 */
13212 void ZopfliFindLongestMatch(
13213 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
13214 size_t pos, size_t size, size_t limit,
13215 unsigned short* sublen, unsigned short* distance, unsigned short* length);
13216  
13217 /*
13218 Verifies if length and dist are indeed valid, only used for assertion.
13219 */
13220 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
13221 unsigned short dist, unsigned short length);
13222  
13223 /*
13224 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
13225 with the slow but better "squeeze" implementation.
13226 The result is placed in the ZopfliLZ77Store.
13227 If instart is larger than 0, it uses values before instart as starting
13228 dictionary.
13229 */
13230 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
13231 size_t instart, size_t inend,
13232 ZopfliLZ77Store* store, ZopfliHash* h);
13233  
13234 #endif /* ZOPFLI_LZ77_H_ */
13235  
13236 /* zopfli.h */
13237 /*
13238 Copyright 2011 Google Inc. All Rights Reserved.
13239  
13240 Licensed under the Apache License, Version 2.0 (the "License");
13241 you may not use this file except in compliance with the License.
13242 You may obtain a copy of the License at
13243  
13244 http://www.apache.org/licenses/LICENSE-2.0
13245  
13246 Unless required by applicable law or agreed to in writing, software
13247 distributed under the License is distributed on an "AS IS" BASIS,
13248 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13249 See the License for the specific language governing permissions and
13250 limitations under the License.
13251  
13252 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
13253 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
13254 */
13255  
13256 #ifndef ZOPFLI_ZOPFLI_H_
13257 #define ZOPFLI_ZOPFLI_H_
13258  
13259 #include <stddef.h>
13260 #include <stdlib.h> /* for size_t */
13261  
13262 #ifdef __cplusplus
13263 extern "C" {
13264 #endif
13265  
13266 /*
13267 Options used throughout the program.
13268 */
13269 typedef struct ZopfliOptions {
13270 /* Whether to print output */
13271 int verbose;
13272  
13273 /* Whether to print more detailed output */
13274 int verbose_more;
13275  
13276 /*
13277 Maximum amount of times to rerun forward and backward pass to optimize LZ77
13278 compression cost. Good values: 10, 15 for small files, 5 for files over
13279 several MB in size or it will be too slow.
13280 */
13281 int numiterations;
13282  
13283 /*
13284 If true, splits the data in multiple deflate blocks with optimal choice
13285 for the block boundaries. Block splitting gives better compression. Default:
13286 true (1).
13287 */
13288 int blocksplitting;
13289  
13290 /*
13291 No longer used, left for compatibility.
13292 */
13293 int blocksplittinglast;
13294  
13295 /*
13296 Maximum amount of blocks to split into (0 for unlimited, but this can give
13297 extreme results that hurt compression on some files). Default value: 15.
13298 */
13299 int blocksplittingmax;
13300 } ZopfliOptions;
13301  
13302 /* Initializes options with default values. */
13303 void ZopfliInitOptions(ZopfliOptions* options);
13304  
13305 /* Output format */
13306 typedef enum {
13307 ZOPFLI_FORMAT_GZIP,
13308 ZOPFLI_FORMAT_ZLIB,
13309 ZOPFLI_FORMAT_DEFLATE
13310 } ZopfliFormat;
13311  
13312 /*
13313 Compresses according to the given output format and appends the result to the
13314 output.
13315  
13316 options: global program options
13317 output_type: the output format to use
13318 out: pointer to the dynamic output array to which the result is appended. Must
13319 be freed after use
13320 outsize: pointer to the dynamic output array size
13321 */
13322 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
13323 const unsigned char* in, size_t insize,
13324 unsigned char** out, size_t* outsize);
13325  
13326 #ifdef __cplusplus
13327 } // extern "C"
13328 #endif
13329  
13330 #endif /* ZOPFLI_ZOPFLI_H_ */
13331  
13332  
13333 #ifdef __cplusplus
13334 extern "C" {
13335 #endif
13336  
13337 /*
13338 Compresses according to the deflate specification and append the compressed
13339 result to the output.
13340 This function will usually output multiple deflate blocks. If final is 1, then
13341 the final bit will be set on the last block.
13342  
13343 options: global program options
13344 btype: the deflate block type. Use 2 for best compression.
13345 -0: non compressed blocks (00)
13346 -1: blocks with fixed tree (01)
13347 -2: blocks with dynamic tree (10)
13348 final: whether this is the last section of the input, sets the final bit to the
13349 last deflate block.
13350 in: the input bytes
13351 insize: number of input bytes
13352 bp: bit pointer for the output array. This must initially be 0, and for
13353 consecutive calls must be reused (it can have values from 0-7). This is
13354 because deflate appends blocks as bit-based data, rather than on byte
13355 boundaries.
13356 out: pointer to the dynamic output array to which the result is appended. Must
13357 be freed after use.
13358 outsize: pointer to the dynamic output array size.
13359 */
13360 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
13361 const unsigned char* in, size_t insize,
13362 unsigned char* bp, unsigned char** out, size_t* outsize);
13363  
13364 /*
13365 Like ZopfliDeflate, but allows to specify start and end byte with instart and
13366 inend. Only that part is compressed, but earlier bytes are still used for the
13367 back window.
13368 */
13369 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
13370 const unsigned char* in, size_t instart, size_t inend,
13371 unsigned char* bp, unsigned char** out,
13372 size_t* outsize);
13373  
13374 /*
13375 Calculates block size in bits.
13376 litlens: lz77 lit/lengths
13377 dists: ll77 distances
13378 lstart: start of block
13379 lend: end of block (not inclusive)
13380 */
13381 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
13382 size_t lstart, size_t lend, int btype);
13383  
13384 /*
13385 Calculates block size in bits, automatically using the best btype.
13386 */
13387 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
13388 size_t lstart, size_t lend);
13389  
13390 #ifdef __cplusplus
13391 } // extern "C"
13392 #endif
13393  
13394 #endif /* ZOPFLI_DEFLATE_H_ */
13395  
13396 /* symbols.h */
13397 /*
13398 Copyright 2016 Google Inc. All Rights Reserved.
13399  
13400 Licensed under the Apache License, Version 2.0 (the "License");
13401 you may not use this file except in compliance with the License.
13402 You may obtain a copy of the License at
13403  
13404 http://www.apache.org/licenses/LICENSE-2.0
13405  
13406 Unless required by applicable law or agreed to in writing, software
13407 distributed under the License is distributed on an "AS IS" BASIS,
13408 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13409 See the License for the specific language governing permissions and
13410 limitations under the License.
13411  
13412 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
13413 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
13414 */
13415  
13416 /*
13417 Utilities for using the lz77 symbols of the deflate spec.
13418 */
13419  
13420 #ifndef ZOPFLI_SYMBOLS_H_
13421 #define ZOPFLI_SYMBOLS_H_
13422  
13423 /* __has_builtin available in clang */
13424 #ifdef __has_builtin
13425 # if __has_builtin(__builtin_clz)
13426 # define ZOPFLI_HAS_BUILTIN_CLZ
13427 # endif
13428 /* __builtin_clz available beginning with GCC 3.4 */
13429 #elif __GNUC__ * 100 + __GNUC_MINOR__ >= 304
13430 # define ZOPFLI_HAS_BUILTIN_CLZ
13431 #endif
13432  
13433 /* Gets the amount of extra bits for the given dist, cfr. the DEFLATE spec. */
13434 static int ZopfliGetDistExtraBits(int dist) {
13435 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
13436 if (dist < 5) return 0;
13437 return (31 ^ __builtin_clz(dist - 1)) - 1; /* log2(dist - 1) - 1 */
13438 #else
13439 if (dist < 5) return 0;
13440 else if (dist < 9) return 1;
13441 else if (dist < 17) return 2;
13442 else if (dist < 33) return 3;
13443 else if (dist < 65) return 4;
13444 else if (dist < 129) return 5;
13445 else if (dist < 257) return 6;
13446 else if (dist < 513) return 7;
13447 else if (dist < 1025) return 8;
13448 else if (dist < 2049) return 9;
13449 else if (dist < 4097) return 10;
13450 else if (dist < 8193) return 11;
13451 else if (dist < 16385) return 12;
13452 else return 13;
13453 #endif
13454 }
13455  
13456 /* Gets value of the extra bits for the given dist, cfr. the DEFLATE spec. */
13457 static int ZopfliGetDistExtraBitsValue(int dist) {
13458 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
13459 if (dist < 5) {
13460 return 0;
13461 } else {
13462 int l = 31 ^ __builtin_clz(dist - 1); /* log2(dist - 1) */
13463 return (dist - (1 + (1 << l))) & ((1 << (l - 1)) - 1);
13464 }
13465 #else
13466 if (dist < 5) return 0;
13467 else if (dist < 9) return (dist - 5) & 1;
13468 else if (dist < 17) return (dist - 9) & 3;
13469 else if (dist < 33) return (dist - 17) & 7;
13470 else if (dist < 65) return (dist - 33) & 15;
13471 else if (dist < 129) return (dist - 65) & 31;
13472 else if (dist < 257) return (dist - 129) & 63;
13473 else if (dist < 513) return (dist - 257) & 127;
13474 else if (dist < 1025) return (dist - 513) & 255;
13475 else if (dist < 2049) return (dist - 1025) & 511;
13476 else if (dist < 4097) return (dist - 2049) & 1023;
13477 else if (dist < 8193) return (dist - 4097) & 2047;
13478 else if (dist < 16385) return (dist - 8193) & 4095;
13479 else return (dist - 16385) & 8191;
13480 #endif
13481 }
13482  
13483 /* Gets the symbol for the given dist, cfr. the DEFLATE spec. */
13484 static int ZopfliGetDistSymbol(int dist) {
13485 #ifdef ZOPFLI_HAS_BUILTIN_CLZ
13486 if (dist < 5) {
13487 return dist - 1;
13488 } else {
13489 int l = (31 ^ __builtin_clz(dist - 1)); /* log2(dist - 1) */
13490 int r = ((dist - 1) >> (l - 1)) & 1;
13491 return l * 2 + r;
13492 }
13493 #else
13494 if (dist < 193) {
13495 if (dist < 13) { /* dist 0..13. */
13496 if (dist < 5) return dist - 1;
13497 else if (dist < 7) return 4;
13498 else if (dist < 9) return 5;
13499 else return 6;
13500 } else { /* dist 13..193. */
13501 if (dist < 17) return 7;
13502 else if (dist < 25) return 8;
13503 else if (dist < 33) return 9;
13504 else if (dist < 49) return 10;
13505 else if (dist < 65) return 11;
13506 else if (dist < 97) return 12;
13507 else if (dist < 129) return 13;
13508 else return 14;
13509 }
13510 } else {
13511 if (dist < 2049) { /* dist 193..2049. */
13512 if (dist < 257) return 15;
13513 else if (dist < 385) return 16;
13514 else if (dist < 513) return 17;
13515 else if (dist < 769) return 18;
13516 else if (dist < 1025) return 19;
13517 else if (dist < 1537) return 20;
13518 else return 21;
13519 } else { /* dist 2049..32768. */
13520 if (dist < 3073) return 22;
13521 else if (dist < 4097) return 23;
13522 else if (dist < 6145) return 24;
13523 else if (dist < 8193) return 25;
13524 else if (dist < 12289) return 26;
13525 else if (dist < 16385) return 27;
13526 else if (dist < 24577) return 28;
13527 else return 29;
13528 }
13529 }
13530 #endif
13531 }
13532  
13533 /* Gets the amount of extra bits for the given length, cfr. the DEFLATE spec. */
13534 static int ZopfliGetLengthExtraBits(int l) {
13535 static const int table[259] = {
13536 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
13537 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
13538 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
13539 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
13540 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
13541 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
13542 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
13543 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
13544 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13545 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13546 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13547 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13548 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13549 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13550 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
13551 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0
13552 };
13553 return table[l];
13554 }
13555  
13556 /* Gets value of the extra bits for the given length, cfr. the DEFLATE spec. */
13557 static int ZopfliGetLengthExtraBitsValue(int l) {
13558 static const int table[259] = {
13559 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0,
13560 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5,
13561 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6,
13562 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
13563 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2,
13564 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
13565 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
13566 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
13567 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6,
13568 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
13569 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
13570 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0
13571 };
13572 return table[l];
13573 }
13574  
13575 /*
13576 Gets the symbol for the given length, cfr. the DEFLATE spec.
13577 Returns the symbol in the range [257-285] (inclusive)
13578 */
13579 static int ZopfliGetLengthSymbol(int l) {
13580 static const int table[259] = {
13581 0, 0, 0, 257, 258, 259, 260, 261, 262, 263, 264,
13582 265, 265, 266, 266, 267, 267, 268, 268,
13583 269, 269, 269, 269, 270, 270, 270, 270,
13584 271, 271, 271, 271, 272, 272, 272, 272,
13585 273, 273, 273, 273, 273, 273, 273, 273,
13586 274, 274, 274, 274, 274, 274, 274, 274,
13587 275, 275, 275, 275, 275, 275, 275, 275,
13588 276, 276, 276, 276, 276, 276, 276, 276,
13589 277, 277, 277, 277, 277, 277, 277, 277,
13590 277, 277, 277, 277, 277, 277, 277, 277,
13591 278, 278, 278, 278, 278, 278, 278, 278,
13592 278, 278, 278, 278, 278, 278, 278, 278,
13593 279, 279, 279, 279, 279, 279, 279, 279,
13594 279, 279, 279, 279, 279, 279, 279, 279,
13595 280, 280, 280, 280, 280, 280, 280, 280,
13596 280, 280, 280, 280, 280, 280, 280, 280,
13597 281, 281, 281, 281, 281, 281, 281, 281,
13598 281, 281, 281, 281, 281, 281, 281, 281,
13599 281, 281, 281, 281, 281, 281, 281, 281,
13600 281, 281, 281, 281, 281, 281, 281, 281,
13601 282, 282, 282, 282, 282, 282, 282, 282,
13602 282, 282, 282, 282, 282, 282, 282, 282,
13603 282, 282, 282, 282, 282, 282, 282, 282,
13604 282, 282, 282, 282, 282, 282, 282, 282,
13605 283, 283, 283, 283, 283, 283, 283, 283,
13606 283, 283, 283, 283, 283, 283, 283, 283,
13607 283, 283, 283, 283, 283, 283, 283, 283,
13608 283, 283, 283, 283, 283, 283, 283, 283,
13609 284, 284, 284, 284, 284, 284, 284, 284,
13610 284, 284, 284, 284, 284, 284, 284, 284,
13611 284, 284, 284, 284, 284, 284, 284, 284,
13612 284, 284, 284, 284, 284, 284, 284, 285
13613 };
13614 return table[l];
13615 }
13616  
13617 /* Gets the amount of extra bits for the given length symbol. */
13618 static int ZopfliGetLengthSymbolExtraBits(int s) {
13619 static const int table[29] = {
13620 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
13621 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0
13622 };
13623 return table[s - 257];
13624 }
13625  
13626 /* Gets the amount of extra bits for the given distance symbol. */
13627 static int ZopfliGetDistSymbolExtraBits(int s) {
13628 static const int table[30] = {
13629 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
13630 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
13631 };
13632 return table[s];
13633 }
13634  
13635 #endif /* ZOPFLI_SYMBOLS_H_ */
13636  
13637 /* tree.h */
13638 /*
13639 Copyright 2011 Google Inc. All Rights Reserved.
13640  
13641 Licensed under the Apache License, Version 2.0 (the "License");
13642 you may not use this file except in compliance with the License.
13643 You may obtain a copy of the License at
13644  
13645 http://www.apache.org/licenses/LICENSE-2.0
13646  
13647 Unless required by applicable law or agreed to in writing, software
13648 distributed under the License is distributed on an "AS IS" BASIS,
13649 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13650 See the License for the specific language governing permissions and
13651 limitations under the License.
13652  
13653 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
13654 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
13655 */
13656  
13657 /*
13658 Utilities for creating and using Huffman trees.
13659 */
13660  
13661 #ifndef ZOPFLI_TREE_H_
13662 #define ZOPFLI_TREE_H_
13663  
13664 #include <string.h>
13665  
13666 /*
13667 Calculates the bitlengths for the Huffman tree, based on the counts of each
13668 symbol.
13669 */
13670 void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits,
13671 unsigned *bitlengths);
13672  
13673 /*
13674 Converts a series of Huffman tree bitlengths, to the bit values of the symbols.
13675 */
13676 void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits,
13677 unsigned* symbols);
13678  
13679 /*
13680 Calculates the entropy of each symbol, based on the counts of each symbol. The
13681 result is similar to the result of ZopfliCalculateBitLengths, but with the
13682 actual theoritical bit lengths according to the entropy. Since the resulting
13683 values are fractional, they cannot be used to encode the tree specified by
13684 DEFLATE.
13685 */
13686 void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths);
13687  
13688 #endif /* ZOPFLI_TREE_H_ */
13689  
13690 /* util.h */
13691 /*
13692 Copyright 2011 Google Inc. All Rights Reserved.
13693  
13694 Licensed under the Apache License, Version 2.0 (the "License");
13695 you may not use this file except in compliance with the License.
13696 You may obtain a copy of the License at
13697  
13698 http://www.apache.org/licenses/LICENSE-2.0
13699  
13700 Unless required by applicable law or agreed to in writing, software
13701 distributed under the License is distributed on an "AS IS" BASIS,
13702 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13703 See the License for the specific language governing permissions and
13704 limitations under the License.
13705  
13706 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
13707 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
13708 */
13709  
13710 /*
13711 Several utilities, including: #defines to try different compression results,
13712 basic deflate specification values and generic program options.
13713 */
13714  
13715 #ifndef ZOPFLI_UTIL_H_
13716 #define ZOPFLI_UTIL_H_
13717  
13718 #include <string.h>
13719 #include <stdlib.h>
13720  
13721 /* Minimum and maximum length that can be encoded in deflate. */
13722 #define ZOPFLI_MAX_MATCH 258
13723 #define ZOPFLI_MIN_MATCH 3
13724  
13725 /* Number of distinct literal/length and distance symbols in DEFLATE */
13726 #define ZOPFLI_NUM_LL 288
13727 #define ZOPFLI_NUM_D 32
13728  
13729 /*
13730 The window size for deflate. Must be a power of two. This should be 32768, the
13731 maximum possible by the deflate spec. Anything less hurts compression more than
13732 speed.
13733 */
13734 #define ZOPFLI_WINDOW_SIZE 32768
13735  
13736 /*
13737 The window mask used to wrap indices into the window. This is why the
13738 window size must be a power of two.
13739 */
13740 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
13741  
13742 /*
13743 A block structure of huge, non-smart, blocks to divide the input into, to allow
13744 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
13745 The whole compression algorithm, including the smarter block splitting, will
13746 be executed independently on each huge block.
13747 Dividing into huge blocks hurts compression, but not much relative to the size.
13748 Set it to 0 to disable master blocks.
13749 */
13750 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
13751  
13752 /*
13753 Used to initialize costs for example
13754 */
13755 #define ZOPFLI_LARGE_FLOAT 1e30
13756  
13757 /*
13758 For longest match cache. max 256. Uses huge amounts of memory but makes it
13759 faster. Uses this many times three bytes per single byte of the input data.
13760 This is so because longest match finding has to find the exact distance
13761 that belongs to each length for the best lz77 strategy.
13762 Good values: e.g. 5, 8.
13763 */
13764 #define ZOPFLI_CACHE_LENGTH 8
13765  
13766 /*
13767 limit the max hash chain hits for this hash value. This has an effect only
13768 on files where the hash value is the same very often. On these files, this
13769 gives worse compression (the value should ideally be 32768, which is the
13770 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
13771 faster on some specific files.
13772 Good value: e.g. 8192.
13773 */
13774 #define ZOPFLI_MAX_CHAIN_HITS 8192
13775  
13776 /*
13777 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
13778 consumes a lot of memory but speeds it up. No effect on compression size.
13779 */
13780 #define ZOPFLI_LONGEST_MATCH_CACHE
13781  
13782 /*
13783 Enable to remember amount of successive identical bytes in the hash chain for
13784 finding longest match
13785 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
13786 This has no effect on the compression result, and enabling it increases speed.
13787 */
13788 #define ZOPFLI_HASH_SAME
13789  
13790 /*
13791 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
13792 best length so far is long enough. This is way faster for files with lots of
13793 identical bytes, on which the compressor is otherwise too slow. Regular files
13794 are unaffected or maybe a tiny bit slower.
13795 This has no effect on the compression result, only on speed.
13796 */
13797 #define ZOPFLI_HASH_SAME_HASH
13798  
13799 /*
13800 Enable this, to avoid slowness for files which are a repetition of the same
13801 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
13802 the compression result.
13803 */
13804 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
13805  
13806 /*
13807 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
13808 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
13809 varies from file to file.
13810 */
13811 #define ZOPFLI_LAZY_MATCHING
13812  
13813 /*
13814 Appends value to dynamically allocated memory, doubling its allocation size
13815 whenever needed.
13816  
13817 value: the value to append, type T
13818 data: pointer to the dynamic array to append to, type T**
13819 size: pointer to the size of the array to append to, type size_t*. This is the
13820 size that you consider the array to be, not the internal allocation size.
13821 Precondition: allocated size of data is at least a power of two greater than or
13822 equal than *size.
13823 */
13824 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
13825 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
13826 if (!((*size) & ((*size) - 1))) {\
13827 /*double alloc size if it's a power of two*/\
13828 void** data_void = reinterpret_cast<void**>(data);\
13829 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
13830 : realloc((*data), (*size) * 2 * sizeof(**data));\
13831 }\
13832 (*data)[(*size)] = (value);\
13833 (*size)++;\
13834 }
13835 #else /* C gives problems with strict-aliasing rules for (void**) cast */
13836 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
13837 if (!((*size) & ((*size) - 1))) {\
13838 /*double alloc size if it's a power of two*/\
13839 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
13840 : realloc((*data), (*size) * 2 * sizeof(**data));\
13841 }\
13842 (*data)[(*size)] = (value);\
13843 (*size)++;\
13844 }
13845 #endif
13846  
13847  
13848 #endif /* ZOPFLI_UTIL_H_ */
13849  
13850  
13851 typedef struct SymbolStats {
13852 /* The literal and length symbols. */
13853 size_t litlens[ZOPFLI_NUM_LL];
13854 /* The 32 unique dist symbols, not the 32768 possible dists. */
13855 size_t dists[ZOPFLI_NUM_D];
13856  
13857 /* Length of each lit/len symbol in bits. */
13858 double ll_symbols[ZOPFLI_NUM_LL];
13859 /* Length of each dist symbol in bits. */
13860 double d_symbols[ZOPFLI_NUM_D];
13861 } SymbolStats;
13862  
13863 /* Sets everything to 0. */
13864 4 static void InitStats(SymbolStats* stats) {
13865 4 memset(stats->litlens, 0, ZOPFLI_NUM_LL * sizeof(stats->litlens[0]));
13866 4 memset(stats->dists, 0, ZOPFLI_NUM_D * sizeof(stats->dists[0]));
13867  
13868 4 memset(stats->ll_symbols, 0, ZOPFLI_NUM_LL * sizeof(stats->ll_symbols[0]));
13869 4 memset(stats->d_symbols, 0, ZOPFLI_NUM_D * sizeof(stats->d_symbols[0]));
13870 4 }
13871  
13872 100 static void CopyStats(SymbolStats* source, SymbolStats* dest) {
13873 100 memcpy(dest->litlens, source->litlens,
13874 ZOPFLI_NUM_LL * sizeof(dest->litlens[0]));
13875 100 memcpy(dest->dists, source->dists, ZOPFLI_NUM_D * sizeof(dest->dists[0]));
13876  
13877 100 memcpy(dest->ll_symbols, source->ll_symbols,
13878 ZOPFLI_NUM_LL * sizeof(dest->ll_symbols[0]));
13879 100 memcpy(dest->d_symbols, source->d_symbols,
13880 ZOPFLI_NUM_D * sizeof(dest->d_symbols[0]));
13881 100 }
13882  
13883 /* Adds the bit lengths. */
13884 32 static void AddWeighedStatFreqs(const SymbolStats* stats1, double w1,
13885 const SymbolStats* stats2, double w2,
13886 SymbolStats* result) {
13887 size_t i;
13888 9248 ▶ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_LL; i++) { 9216 32 9248
13889 9216 result->litlens[i] =
13890 9216 (size_t) (stats1->litlens[i] * w1 + stats2->litlens[i] * w2);
13891 }
13892 1056 ▶ for (i = 0; i < ZOPFLI_NUM_D; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_D; i++) { 1024 32 1056
13893 1024 result->dists[i] =
13894 1024 (size_t) (stats1->dists[i] * w1 + stats2->dists[i] * w2);
13895 }
13896 32 result->litlens[256] = 1; /* End symbol. */
13897 32 }
13898  
13899 typedef struct RanState {
13900 unsigned int m_w, m_z;
13901 } RanState;
13902  
13903 4 static void InitRanState(RanState* state) {
13904 4 state->m_w = 1;
13905 4 state->m_z = 2;
13906 4 }
13907  
13908 /* Get random number: "Multiply-With-Carry" generator of G. Marsaglia */
13909 15380 static unsigned int Ran(RanState* state) {
13910 15380 state->m_z = 36969 * (state->m_z & 65535) + (state->m_z >> 16);
13911 15380 state->m_w = 18000 * (state->m_w & 65535) + (state->m_w >> 16);
13912 15380 return (state->m_z << 16) + state->m_w; /* 32-bit result. */
13913 }
13914  
13915 72 static void RandomizeFreqs(RanState* state, size_t* freqs, int n) {
13916 int i;
13917 11592 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 11520 72 11592
13918 11520 ▶ if ((Ran(state) >> 4) % 3 == 0) freqs[i] = freqs[Ran(state) % n];
Branch: 1 branch
BranchTrueFalseTotal
if ((Ran(state) >> 4) % 3 == 0) freqs[i] = freqs[Ran(state) % n]; 3860 7660 11520
13919 }
13920 72 }
13921  
13922 36 static void RandomizeStatFreqs(RanState* state, SymbolStats* stats) {
13923 36 RandomizeFreqs(state, stats->litlens, ZOPFLI_NUM_LL);
13924 36 RandomizeFreqs(state, stats->dists, ZOPFLI_NUM_D);
13925 36 stats->litlens[256] = 1; /* End symbol. */
13926 36 }
13927  
13928 60 static void ClearStatFreqs(SymbolStats* stats) {
13929 size_t i;
13930 17340 ▶ for (i = 0; i < ZOPFLI_NUM_LL; i++) stats->litlens[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_LL; i++) stats->litlens[i] = 0; 17280 60 17340
13931 1980 ▶ for (i = 0; i < ZOPFLI_NUM_D; i++) stats->dists[i] = 0;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ZOPFLI_NUM_D; i++) stats->dists[i] = 0; 1920 60 1980
13932 60 }
13933  
13934 /*
13935 Function that calculates a cost based on a model for the given LZ77 symbol.
13936 litlen: means literal symbol if dist is 0, length otherwise.
13937 */
13938 typedef double CostModelFun(unsigned litlen, unsigned dist, void* context);
13939  
13940 /*
13941 Cost model which should exactly match fixed tree.
13942 type: CostModelFun
13943 */
13944 803726 static double GetCostFixed(unsigned litlen, unsigned dist, void* unused) {
13945 (void)unused;
13946 803726 ▶ if (dist == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist == 0) { 9180 794546 803726
13947 9180 ▶ if (litlen <= 143) return 8;
Branch: 1 branch
BranchTrueFalseTotal
if (litlen <= 143) return 8; 9180 0 9180
13948 0 else return 9;
13949 } else {
13950 794546 int dbits = ZopfliGetDistExtraBits(dist);
13951 794546 int lbits = ZopfliGetLengthExtraBits(litlen);
13952 794546 int lsym = ZopfliGetLengthSymbol(litlen);
13953 794546 int cost = 0;
13954 794546 ▶ if (lsym <= 279) cost += 7;
Branch: 1 branch
BranchTrueFalseTotal
if (lsym <= 279) cost += 7; 43002 751544 794546
13955 751544 else cost += 8;
13956 794546 cost += 5; /* Every dist symbol has length 5. */
13957 794546 return cost + dbits + lbits;
13958 }
13959 }
13960  
13961 /*
13962 Cost model based on symbol statistics.
13963 type: CostModelFun
13964 */
13965 7847149 static double GetCostStat(unsigned litlen, unsigned dist, void* context) {
13966 7847149 SymbolStats* stats = (SymbolStats*)context;
13967 7847149 ▶ if (dist == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (dist == 0) { 137700 7709449 7847149
13968 137700 return stats->ll_symbols[litlen];
13969 } else {
13970 7709449 int lsym = ZopfliGetLengthSymbol(litlen);
13971 7709449 int lbits = ZopfliGetLengthExtraBits(litlen);
13972 7709449 int dsym = ZopfliGetDistSymbol(dist);
13973 7709449 int dbits = ZopfliGetDistExtraBits(dist);
13974 7709449 return lbits + dbits + stats->ll_symbols[lsym] + stats->d_symbols[dsym];
13975 }
13976 }
13977  
13978 /*
13979 Finds the minimum possible cost this cost model can return for valid length and
13980 distance symbols.
13981 */
13982 64 static double GetCostModelMinCost(CostModelFun* costmodel, void* costcontext) {
13983 double mincost;
13984 64 int bestlength = 0; /* length that has lowest cost in the cost model */
13985 64 int bestdist = 0; /* distance that has lowest cost in the cost model */
13986 int i;
13987 /*
13988 Table of distances that have a different distance symbol in the deflate
13989 specification. Each value is the first distance that has a new symbol. Only
13990 different symbols affect the cost model so only these need to be checked.
13991 See RFC 1951 section 3.2.5. Compressed blocks (length and distance codes).
13992 */
13993 static const int dsymbols[30] = {
13994 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513,
13995 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577
13996 };
13997  
13998 64 mincost = ZOPFLI_LARGE_FLOAT;
13999 16448 ▶ for (i = 3; i < 259; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 3; i < 259; i++) { 16384 64 16448
14000 16384 double c = costmodel(i, 1, costcontext);
14001 16384 ▶ if (c < mincost) {
Branch: 1 branch
BranchTrueFalseTotal
if (c < mincost) { 76 16308 16384
14002 76 bestlength = i;
14003 76 mincost = c;
14004 }
14005 }
14006  
14007 64 mincost = ZOPFLI_LARGE_FLOAT;
14008 1984 ▶ for (i = 0; i < 30; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 30; i++) { 1920 64 1984
14009 1920 double c = costmodel(3, dsymbols[i], costcontext);
14010 1920 ▶ if (c < mincost) {
Branch: 1 branch
BranchTrueFalseTotal
if (c < mincost) { 78 1842 1920
14011 78 bestdist = dsymbols[i];
14012 78 mincost = c;
14013 }
14014 }
14015  
14016 64 return costmodel(bestlength, bestdist, costcontext);
14017 }
14018  
14019 146880 static size_t zopfli_min(size_t a, size_t b) {
14020 146880 return a < b ? a : b;
14021 }
14022  
14023 /*
14024 Performs the forward pass for "squeeze". Gets the most optimal length to reach
14025 every byte from a previous byte, using cost calculations.
14026 s: the ZopfliBlockState
14027 in: the input data array
14028 instart: where to start
14029 inend: where to stop (not inclusive)
14030 costmodel: function to calculate the cost of some lit/len/dist pair.
14031 costcontext: abstract context for the costmodel function
14032 length_array: output array of size (inend - instart) which will receive the best
14033 length to reach this byte from a previous byte.
14034 returns the cost that was, according to the costmodel, needed to get to the end.
14035 */
14036 64 static double GetBestLengths(ZopfliBlockState *s,
14037 const unsigned char* in,
14038 size_t instart, size_t inend,
14039 CostModelFun* costmodel, void* costcontext,
14040 unsigned short* length_array,
14041 ZopfliHash* h, float* costs) {
14042 /* Best cost to get here so far. */
14043 64 size_t blocksize = inend - instart;
14044 64 size_t i = 0, k, kend;
14045 unsigned short leng;
14046 unsigned short dist;
14047 unsigned short sublen[259];
14048 64 size_t windowstart = instart > ZOPFLI_WINDOW_SIZE
14049 64 ? instart - ZOPFLI_WINDOW_SIZE : 0;
14050 double result;
14051 64 double mincost = GetCostModelMinCost(costmodel, costcontext);
14052 double mincostaddcostj;
14053  
14054 64 ▶ if (instart == inend) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (instart == inend) return 0; 0 64 64
14055  
14056 64 ZopfliResetHash(ZOPFLI_WINDOW_SIZE, h);
14057 64 ZopfliWarmupHash(in, windowstart, inend, h);
14058 208 ▶ for (i = windowstart; i < instart; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = windowstart; i < instart; i++) { 144 64 208
14059 144 ZopfliUpdateHash(in, i, inend, h);
14060 }
14061  
14062 146944 ▶ for (i = 1; i < blocksize + 1; i++) costs[i] = ZOPFLI_LARGE_FLOAT;
Branch: 1 branch
BranchTrueFalseTotal
for (i = 1; i < blocksize + 1; i++) costs[i] = ZOPFLI_LARGE_FLOAT; 146880 64 146944
14063 64 costs[0] = 0; /* Because it's the start. */
14064 64 length_array[0] = 0;
14065  
14066 146944 ▶ for (i = instart; i < inend; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = instart; i < inend; i++) { 146880 64 146944
14067 146880 size_t j = i - instart; /* Index in the costs array and length_array. */
14068 146880 ZopfliUpdateHash(in, i, inend, h);
14069  
14070 #ifdef ZOPFLI_SHORTCUT_LONG_REPETITIONS
14071 /* If we're in a long repetition of the same character and have more than
14072 ZOPFLI_MAX_MATCH characters before and after our position. */
14073 146880 ▶ if (h->same[i & ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH * 2
Branch: 1 branch
BranchTrueFalseTotal
if (h->same[i & ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH * 2 0 146880 146880
14074 0 ▶ && i > instart + ZOPFLI_MAX_MATCH + 1
Branch: 1 branch
BranchTrueFalseTotal
&& i > instart + ZOPFLI_MAX_MATCH + 1 0 0 0
14075 0 ▶ && i + ZOPFLI_MAX_MATCH * 2 + 1 < inend
Branch: 1 branch
BranchTrueFalseTotal
&& i + ZOPFLI_MAX_MATCH * 2 + 1 < inend 0 0 0
14076 0 ▶ && h->same[(i - ZOPFLI_MAX_MATCH) & ZOPFLI_WINDOW_MASK]
Branch: 1 branch
BranchTrueFalseTotal
&& h->same[(i - ZOPFLI_MAX_MATCH) & ZOPFLI_WINDOW_MASK] 0 0 0
14077 > ZOPFLI_MAX_MATCH) {
14078 0 double symbolcost = costmodel(ZOPFLI_MAX_MATCH, 1, costcontext);
14079 /* Set the length to reach each one to ZOPFLI_MAX_MATCH, and the cost to
14080 the cost corresponding to that length. Doing this, we skip
14081 ZOPFLI_MAX_MATCH values to avoid calling ZopfliFindLongestMatch. */
14082 0 ▶ for (k = 0; k < ZOPFLI_MAX_MATCH; k++) {
Branch: 1 branch
BranchTrueFalseTotal
for (k = 0; k < ZOPFLI_MAX_MATCH; k++) { 0 0 0
14083 0 costs[j + ZOPFLI_MAX_MATCH] = costs[j] + symbolcost;
14084 0 length_array[j + ZOPFLI_MAX_MATCH] = ZOPFLI_MAX_MATCH;
14085 0 i++;
14086 0 j++;
14087 0 ZopfliUpdateHash(in, i, inend, h);
14088 }
14089 }
14090 #endif
14091  
14092 146880 ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, sublen,
14093 &dist, &leng);
14094  
14095 /* Literal. */
14096 146880 ▶ if (i + 1 <= inend) {
Branch: 1 branch
BranchTrueFalseTotal
if (i + 1 <= inend) { 146880 0 146880
14097 146880 double newCost = costmodel(in[i], 0, costcontext) + costs[j];
14098 assert(newCost >= 0);
14099 146880 ▶ if (newCost < costs[j + 1]) {
Branch: 1 branch
BranchTrueFalseTotal
if (newCost < costs[j + 1]) { 1046 145834 146880
14100 1046 costs[j + 1] = newCost;
14101 1046 length_array[j + 1] = 1;
14102 }
14103 }
14104 /* Lengths. */
14105 146880 kend = zopfli_min(leng, inend-i);
14106 146880 mincostaddcostj = mincost + costs[j];
14107 36544704 ▶ for (k = 3; k <= kend; k++) {
Branch: 1 branch
BranchTrueFalseTotal
for (k = 3; k <= kend; k++) { 36397824 146880 36544704
14108 double newCost;
14109  
14110 /* Calling the cost model is expensive, avoid this if we are already at
14111 the minimum possible cost that it can return. */
14112 36397824 ▶ if (costs[j + k] <= mincostaddcostj) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (costs[j + k] <= mincostaddcostj) continue; 27912197 8485627 36397824
14113  
14114 8485627 newCost = costmodel(k, sublen[k], costcontext) + costs[j];
14115 assert(newCost >= 0);
14116 8485627 ▶ if (newCost < costs[j + k]) {
Branch: 1 branch
BranchTrueFalseTotal
if (newCost < costs[j + k]) { 190816 8294811 8485627
14117 assert(k <= ZOPFLI_MAX_MATCH);
14118 190816 costs[j + k] = newCost;
14119 190816 length_array[j + k] = k;
14120 }
14121 }
14122 }
14123  
14124 assert(costs[blocksize] >= 0);
14125 64 result = costs[blocksize];
14126  
14127 64 return result;
14128 }
14129  
14130 /*
14131 Calculates the optimal path of lz77 lengths to use, from the calculated
14132 length_array. The length_array must contain the optimal length to reach that
14133 byte. The path will be filled with the lengths to use, so its data size will be
14134 the amount of lz77 symbols.
14135 */
14136 64 static void TraceBackwards(size_t size, const unsigned short* length_array,
14137 unsigned short** path, size_t* pathsize) {
14138 64 size_t index = size;
14139 64 ▶ if (size == 0) return;
Branch: 1 branch
BranchTrueFalseTotal
if (size == 0) return; 64 0 64
14140 for (;;) {
14141 1024 ▶ ZOPFLI_APPEND_DATA(length_array[index], path, pathsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(length_array[index], path, pathsize); 352 672 1024
ZOPFLI_APPEND_DATA(length_array[index], path, pathsize); 64 288 352
14142 assert(length_array[index] <= index);
14143 assert(length_array[index] <= ZOPFLI_MAX_MATCH);
14144 assert(length_array[index] != 0);
14145 1024 index -= length_array[index];
14146 1024 ▶ if (index == 0) break;
Branch: 1 branch
BranchTrueFalseTotal
if (index == 0) break; 64 960 1024
14147 }
14148  
14149 /* Mirror result. */
14150 560 ▶ for (index = 0; index < *pathsize / 2; index++) {
Branch: 1 branch
BranchTrueFalseTotal
for (index = 0; index < *pathsize / 2; index++) { 496 64 560
14151 496 unsigned short temp = (*path)[index];
14152 496 (*path)[index] = (*path)[*pathsize - index - 1];
14153 496 (*path)[*pathsize - index - 1] = temp;
14154 }
14155 }
14156  
14157 64 static void FollowPath(ZopfliBlockState* s,
14158 const unsigned char* in, size_t instart, size_t inend,
14159 unsigned short* path, size_t pathsize,
14160 ZopfliLZ77Store* store, ZopfliHash *h) {
14161 64 size_t i, j, pos = 0;
14162 64 size_t windowstart = instart > ZOPFLI_WINDOW_SIZE
14163 64 ? instart - ZOPFLI_WINDOW_SIZE : 0;
14164  
14165 64 size_t total_length_test = 0;
14166  
14167 64 ▶ if (instart == inend) return;
Branch: 1 branch
BranchTrueFalseTotal
if (instart == inend) return; 0 64 64
14168  
14169 64 ZopfliResetHash(ZOPFLI_WINDOW_SIZE, h);
14170 64 ZopfliWarmupHash(in, windowstart, inend, h);
14171 208 ▶ for (i = windowstart; i < instart; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = windowstart; i < instart; i++) { 144 64 208
14172 144 ZopfliUpdateHash(in, i, inend, h);
14173 }
14174  
14175 64 pos = instart;
14176 1088 ▶ for (i = 0; i < pathsize; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < pathsize; i++) { 1024 64 1088
14177 1024 unsigned short length = path[i];
14178 unsigned short dummy_length;
14179 unsigned short dist;
14180 assert(pos < inend);
14181  
14182 1024 ZopfliUpdateHash(in, pos, inend, h);
14183  
14184 /* Add to output. */
14185 1024 ▶ if (length >= ZOPFLI_MIN_MATCH) {
Branch: 1 branch
BranchTrueFalseTotal
if (length >= ZOPFLI_MIN_MATCH) { 592 432 1024
14186 /* Get the distance by recalculating longest match. The found length
14187 should match the length from the path. */
14188 592 ZopfliFindLongestMatch(s, h, in, pos, inend, length, 0,
14189 &dist, &dummy_length);
14190 assert(!(dummy_length != length && length > 2 && dummy_length > 2));
14191 592 ZopfliVerifyLenDist(in, inend, pos, dist, length);
14192 592 ZopfliStoreLitLenDist(length, dist, pos, store);
14193 592 total_length_test += length;
14194 } else {
14195 432 length = 1;
14196 432 ZopfliStoreLitLenDist(in[pos], 0, pos, store);
14197 432 total_length_test++;
14198 }
14199  
14200  
14201 assert(pos + length <= inend);
14202 146880 ▶ for (j = 1; j < length; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (j = 1; j < length; j++) { 145856 1024 146880
14203 145856 ZopfliUpdateHash(in, pos + j, inend, h);
14204 }
14205  
14206 1024 pos += length;
14207 }
14208 }
14209  
14210 /* Calculates the entropy of the statistics */
14211 132 static void CalculateStatistics(SymbolStats* stats) {
14212 132 ZopfliCalculateEntropy(stats->litlens, ZOPFLI_NUM_LL, stats->ll_symbols);
14213 132 ZopfliCalculateEntropy(stats->dists, ZOPFLI_NUM_D, stats->d_symbols);
14214 132 }
14215  
14216 /* Appends the symbol statistics from the store. */
14217 64 static void GetStatistics(const ZopfliLZ77Store* store, SymbolStats* stats) {
14218 size_t i;
14219 1088 ▶ for (i = 0; i < store->size; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < store->size; i++) { 1024 64 1088
14220 1024 ▶ if (store->dists[i] == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (store->dists[i] == 0) { 432 592 1024
14221 432 stats->litlens[store->litlens[i]]++;
14222 } else {
14223 592 stats->litlens[ZopfliGetLengthSymbol(store->litlens[i])]++;
14224 592 stats->dists[ZopfliGetDistSymbol(store->dists[i])]++;
14225 }
14226 }
14227 64 stats->litlens[256] = 1; /* End symbol. */
14228  
14229 64 CalculateStatistics(stats);
14230 64 }
14231  
14232 /*
14233 Does a single run for ZopfliLZ77Optimal. For good compression, repeated runs
14234 with updated statistics should be performed.
14235 s: the block state
14236 in: the input data array
14237 instart: where to start
14238 inend: where to stop (not inclusive)
14239 path: pointer to dynamically allocated memory to store the path
14240 pathsize: pointer to the size of the dynamic path array
14241 length_array: array of size (inend - instart) used to store lengths
14242 costmodel: function to use as the cost model for this squeeze run
14243 costcontext: abstract context for the costmodel function
14244 store: place to output the LZ77 data
14245 returns the cost that was, according to the costmodel, needed to get to the end.
14246 This is not the actual cost.
14247 */
14248 64 static double LZ77OptimalRun(ZopfliBlockState* s,
14249 const unsigned char* in, size_t instart, size_t inend,
14250 unsigned short** path, size_t* pathsize,
14251 unsigned short* length_array, CostModelFun* costmodel,
14252 void* costcontext, ZopfliLZ77Store* store,
14253 ZopfliHash* h, float* costs) {
14254 64 double cost = GetBestLengths(s, in, instart, inend, costmodel,
14255 costcontext, length_array, h, costs);
14256 64 free(*path);
14257 64 *path = 0;
14258 64 *pathsize = 0;
14259 64 TraceBackwards(inend - instart, length_array, path, pathsize);
14260 64 FollowPath(s, in, instart, inend, *path, *pathsize, store, h);
14261 assert(cost < ZOPFLI_LARGE_FLOAT);
14262 64 return cost;
14263 }
14264  
14265 4 void ZopfliLZ77Optimal(ZopfliBlockState *s,
14266 const unsigned char* in, size_t instart, size_t inend,
14267 int numiterations,
14268 ZopfliLZ77Store* store) {
14269 /* Dist to get to here with smallest cost. */
14270 4 size_t blocksize = inend - instart;
14271 unsigned short* length_array =
14272 4 (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1));
14273 4 unsigned short* path = 0;
14274 4 size_t pathsize = 0;
14275 ZopfliLZ77Store currentstore;
14276 ZopfliHash hash;
14277 4 ZopfliHash* h = &hash;
14278 SymbolStats stats, beststats, laststats;
14279 int i;
14280 4 float* costs = (float*)malloc(sizeof(float) * (blocksize + 1));
14281 double cost;
14282 4 double bestcost = ZOPFLI_LARGE_FLOAT;
14283 4 double lastcost = 0;
14284 /* Try randomizing the costs a bit once the size stabilizes. */
14285 RanState ran_state;
14286 4 int lastrandomstep = -1;
14287  
14288 4 ▶ if (!costs) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!costs) exit(-1); /* Allocation failed. */ 0 4 4
14289 4 ▶ if (!length_array) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!length_array) exit(-1); /* Allocation failed. */ 0 4 4
14290  
14291 4 InitRanState(&ran_state);
14292 4 InitStats(&stats);
14293 4 ZopfliInitLZ77Store(in, &currentstore);
14294 4 ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
14295  
14296 /* Do regular deflate, then loop multiple shortest path runs, each time using
14297 the statistics of the previous run. */
14298  
14299 /* Initial run. */
14300 4 ZopfliLZ77Greedy(s, in, instart, inend, &currentstore, h);
14301 4 GetStatistics(&currentstore, &stats);
14302  
14303 /* Repeat statistics with each time the cost model from the previous stat
14304 run. */
14305 64 ▶ for (i = 0; i < numiterations; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < numiterations; i++) { 60 4 64
14306 60 ZopfliCleanLZ77Store(&currentstore);
14307 60 ZopfliInitLZ77Store(in, &currentstore);
14308 60 LZ77OptimalRun(s, in, instart, inend, &path, &pathsize,
14309 length_array, GetCostStat, (void*)&stats,
14310 &currentstore, h, costs);
14311 60 cost = ZopfliCalculateBlockSize(&currentstore, 0, currentstore.size, 2);
14312 60 ▶ if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) {
Branch: 3 branches
BranchTrueFalseTotal
if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { 60 0 60
if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { 0 60 60
if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { 0 0 0
14313 0 fprintf(stderr, "Iteration %d: %d bit\n", i, (int) cost);
14314 }
14315 60 ▶ if (cost < bestcost) {
Branch: 1 branch
BranchTrueFalseTotal
if (cost < bestcost) { 4 56 60
14316 /* Copy to the output store. */
14317 4 ZopfliCopyLZ77Store(&currentstore, store);
14318 4 CopyStats(&stats, &beststats);
14319 4 bestcost = cost;
14320 }
14321 60 CopyStats(&stats, &laststats);
14322 60 ClearStatFreqs(&stats);
14323 60 GetStatistics(&currentstore, &stats);
14324 60 ▶ if (lastrandomstep != -1) {
Branch: 1 branch
BranchTrueFalseTotal
if (lastrandomstep != -1) { 32 28 60
14325 /* This makes it converge slower but better. Do it only once the
14326 randomness kicks in so that if the user does few iterations, it gives a
14327 better result sooner. */
14328 32 AddWeighedStatFreqs(&stats, 1.0, &laststats, 0.5, &stats);
14329 32 CalculateStatistics(&stats);
14330 }
14331 60 ▶ if (i > 5 && cost == lastcost) {
Branch: 2 branches
BranchTrueFalseTotal
if (i > 5 && cost == lastcost) { 36 24 60
if (i > 5 && cost == lastcost) { 36 0 36
14332 36 CopyStats(&beststats, &stats);
14333 36 RandomizeStatFreqs(&ran_state, &stats);
14334 36 CalculateStatistics(&stats);
14335 36 lastrandomstep = i;
14336 }
14337 60 lastcost = cost;
14338 }
14339  
14340 4 free(length_array);
14341 4 free(path);
14342 4 free(costs);
14343 4 ZopfliCleanLZ77Store(&currentstore);
14344 4 ZopfliCleanHash(h);
14345 4 }
14346  
14347 4 void ZopfliLZ77OptimalFixed(ZopfliBlockState *s,
14348 const unsigned char* in,
14349 size_t instart, size_t inend,
14350 ZopfliLZ77Store* store)
14351 {
14352 /* Dist to get to here with smallest cost. */
14353 4 size_t blocksize = inend - instart;
14354 unsigned short* length_array =
14355 4 (unsigned short*)malloc(sizeof(unsigned short) * (blocksize + 1));
14356 4 unsigned short* path = 0;
14357 4 size_t pathsize = 0;
14358 ZopfliHash hash;
14359 4 ZopfliHash* h = &hash;
14360 4 float* costs = (float*)malloc(sizeof(float) * (blocksize + 1));
14361  
14362 4 ▶ if (!costs) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!costs) exit(-1); /* Allocation failed. */ 0 4 4
14363 4 ▶ if (!length_array) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!length_array) exit(-1); /* Allocation failed. */ 0 4 4
14364  
14365 4 ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
14366  
14367 4 s->blockstart = instart;
14368 4 s->blockend = inend;
14369  
14370 /* Shortest path for fixed tree This one should give the shortest possible
14371 result for fixed tree, no repeated runs are needed since the tree is known. */
14372 4 LZ77OptimalRun(s, in, instart, inend, &path, &pathsize,
14373 length_array, GetCostFixed, 0, store, h, costs);
14374  
14375 4 free(length_array);
14376 4 free(path);
14377 4 free(costs);
14378 4 ZopfliCleanHash(h);
14379 4 }
14380 /*
14381 Copyright 2011 Google Inc. All Rights Reserved.
14382  
14383 Licensed under the Apache License, Version 2.0 (the "License");
14384 you may not use this file except in compliance with the License.
14385 You may obtain a copy of the License at
14386  
14387 http://www.apache.org/licenses/LICENSE-2.0
14388  
14389 Unless required by applicable law or agreed to in writing, software
14390 distributed under the License is distributed on an "AS IS" BASIS,
14391 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14392 See the License for the specific language governing permissions and
14393 limitations under the License.
14394  
14395 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14396 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14397 */
14398  
14399 /* tree.h */
14400 /*
14401 Copyright 2011 Google Inc. All Rights Reserved.
14402  
14403 Licensed under the Apache License, Version 2.0 (the "License");
14404 you may not use this file except in compliance with the License.
14405 You may obtain a copy of the License at
14406  
14407 http://www.apache.org/licenses/LICENSE-2.0
14408  
14409 Unless required by applicable law or agreed to in writing, software
14410 distributed under the License is distributed on an "AS IS" BASIS,
14411 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14412 See the License for the specific language governing permissions and
14413 limitations under the License.
14414  
14415 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14416 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14417 */
14418  
14419 /*
14420 Utilities for creating and using Huffman trees.
14421 */
14422  
14423 #ifndef ZOPFLI_TREE_H_
14424 #define ZOPFLI_TREE_H_
14425  
14426 #include <string.h>
14427  
14428 /*
14429 Calculates the bitlengths for the Huffman tree, based on the counts of each
14430 symbol.
14431 */
14432 void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits,
14433 unsigned *bitlengths);
14434  
14435 /*
14436 Converts a series of Huffman tree bitlengths, to the bit values of the symbols.
14437 */
14438 void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits,
14439 unsigned* symbols);
14440  
14441 /*
14442 Calculates the entropy of each symbol, based on the counts of each symbol. The
14443 result is similar to the result of ZopfliCalculateBitLengths, but with the
14444 actual theoritical bit lengths according to the entropy. Since the resulting
14445 values are fractional, they cannot be used to encode the tree specified by
14446 DEFLATE.
14447 */
14448 void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths);
14449  
14450 #endif /* ZOPFLI_TREE_H_ */
14451  
14452  
14453 #include <assert.h>
14454 #include <math.h>
14455 #include <stdio.h>
14456 #include <stdlib.h>
14457  
14458 /* katajainen.h */
14459 /*
14460 Copyright 2011 Google Inc. All Rights Reserved.
14461  
14462 Licensed under the Apache License, Version 2.0 (the "License");
14463 you may not use this file except in compliance with the License.
14464 You may obtain a copy of the License at
14465  
14466 http://www.apache.org/licenses/LICENSE-2.0
14467  
14468 Unless required by applicable law or agreed to in writing, software
14469 distributed under the License is distributed on an "AS IS" BASIS,
14470 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14471 See the License for the specific language governing permissions and
14472 limitations under the License.
14473  
14474 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14475 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14476 */
14477  
14478 #ifndef ZOPFLI_KATAJAINEN_H_
14479 #define ZOPFLI_KATAJAINEN_H_
14480  
14481 #include <string.h>
14482  
14483 /*
14484 Outputs minimum-redundancy length-limited code bitlengths for symbols with the
14485 given counts. The bitlengths are limited by maxbits.
14486  
14487 The output is tailored for DEFLATE: symbols that never occur, get a bit length
14488 of 0, and if only a single symbol occurs at least once, its bitlength will be 1,
14489 and not 0 as would theoretically be needed for a single symbol.
14490  
14491 frequencies: The amount of occurrences of each symbol.
14492 n: The amount of symbols.
14493 maxbits: Maximum bit length, inclusive.
14494 bitlengths: Output, the bitlengths for the symbol prefix codes.
14495 return: 0 for OK, non-0 for error.
14496 */
14497 int ZopfliLengthLimitedCodeLengths(
14498 const size_t* frequencies, int n, int maxbits, unsigned* bitlengths);
14499  
14500 #endif /* ZOPFLI_KATAJAINEN_H_ */
14501  
14502 /* util.h */
14503 /*
14504 Copyright 2011 Google Inc. All Rights Reserved.
14505  
14506 Licensed under the Apache License, Version 2.0 (the "License");
14507 you may not use this file except in compliance with the License.
14508 You may obtain a copy of the License at
14509  
14510 http://www.apache.org/licenses/LICENSE-2.0
14511  
14512 Unless required by applicable law or agreed to in writing, software
14513 distributed under the License is distributed on an "AS IS" BASIS,
14514 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14515 See the License for the specific language governing permissions and
14516 limitations under the License.
14517  
14518 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14519 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14520 */
14521  
14522 /*
14523 Several utilities, including: #defines to try different compression results,
14524 basic deflate specification values and generic program options.
14525 */
14526  
14527 #ifndef ZOPFLI_UTIL_H_
14528 #define ZOPFLI_UTIL_H_
14529  
14530 #include <string.h>
14531 #include <stdlib.h>
14532  
14533 /* Minimum and maximum length that can be encoded in deflate. */
14534 #define ZOPFLI_MAX_MATCH 258
14535 #define ZOPFLI_MIN_MATCH 3
14536  
14537 /* Number of distinct literal/length and distance symbols in DEFLATE */
14538 #define ZOPFLI_NUM_LL 288
14539 #define ZOPFLI_NUM_D 32
14540  
14541 /*
14542 The window size for deflate. Must be a power of two. This should be 32768, the
14543 maximum possible by the deflate spec. Anything less hurts compression more than
14544 speed.
14545 */
14546 #define ZOPFLI_WINDOW_SIZE 32768
14547  
14548 /*
14549 The window mask used to wrap indices into the window. This is why the
14550 window size must be a power of two.
14551 */
14552 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
14553  
14554 /*
14555 A block structure of huge, non-smart, blocks to divide the input into, to allow
14556 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
14557 The whole compression algorithm, including the smarter block splitting, will
14558 be executed independently on each huge block.
14559 Dividing into huge blocks hurts compression, but not much relative to the size.
14560 Set it to 0 to disable master blocks.
14561 */
14562 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
14563  
14564 /*
14565 Used to initialize costs for example
14566 */
14567 #define ZOPFLI_LARGE_FLOAT 1e30
14568  
14569 /*
14570 For longest match cache. max 256. Uses huge amounts of memory but makes it
14571 faster. Uses this many times three bytes per single byte of the input data.
14572 This is so because longest match finding has to find the exact distance
14573 that belongs to each length for the best lz77 strategy.
14574 Good values: e.g. 5, 8.
14575 */
14576 #define ZOPFLI_CACHE_LENGTH 8
14577  
14578 /*
14579 limit the max hash chain hits for this hash value. This has an effect only
14580 on files where the hash value is the same very often. On these files, this
14581 gives worse compression (the value should ideally be 32768, which is the
14582 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
14583 faster on some specific files.
14584 Good value: e.g. 8192.
14585 */
14586 #define ZOPFLI_MAX_CHAIN_HITS 8192
14587  
14588 /*
14589 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
14590 consumes a lot of memory but speeds it up. No effect on compression size.
14591 */
14592 #define ZOPFLI_LONGEST_MATCH_CACHE
14593  
14594 /*
14595 Enable to remember amount of successive identical bytes in the hash chain for
14596 finding longest match
14597 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
14598 This has no effect on the compression result, and enabling it increases speed.
14599 */
14600 #define ZOPFLI_HASH_SAME
14601  
14602 /*
14603 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
14604 best length so far is long enough. This is way faster for files with lots of
14605 identical bytes, on which the compressor is otherwise too slow. Regular files
14606 are unaffected or maybe a tiny bit slower.
14607 This has no effect on the compression result, only on speed.
14608 */
14609 #define ZOPFLI_HASH_SAME_HASH
14610  
14611 /*
14612 Enable this, to avoid slowness for files which are a repetition of the same
14613 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
14614 the compression result.
14615 */
14616 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
14617  
14618 /*
14619 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
14620 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
14621 varies from file to file.
14622 */
14623 #define ZOPFLI_LAZY_MATCHING
14624  
14625 /*
14626 Appends value to dynamically allocated memory, doubling its allocation size
14627 whenever needed.
14628  
14629 value: the value to append, type T
14630 data: pointer to the dynamic array to append to, type T**
14631 size: pointer to the size of the array to append to, type size_t*. This is the
14632 size that you consider the array to be, not the internal allocation size.
14633 Precondition: allocated size of data is at least a power of two greater than or
14634 equal than *size.
14635 */
14636 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
14637 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
14638 if (!((*size) & ((*size) - 1))) {\
14639 /*double alloc size if it's a power of two*/\
14640 void** data_void = reinterpret_cast<void**>(data);\
14641 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
14642 : realloc((*data), (*size) * 2 * sizeof(**data));\
14643 }\
14644 (*data)[(*size)] = (value);\
14645 (*size)++;\
14646 }
14647 #else /* C gives problems with strict-aliasing rules for (void**) cast */
14648 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
14649 if (!((*size) & ((*size) - 1))) {\
14650 /*double alloc size if it's a power of two*/\
14651 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
14652 : realloc((*data), (*size) * 2 * sizeof(**data));\
14653 }\
14654 (*data)[(*size)] = (value);\
14655 (*size)++;\
14656 }
14657 #endif
14658  
14659  
14660 #endif /* ZOPFLI_UTIL_H_ */
14661  
14662  
14663 9 void ZopfliLengthsToSymbols(const unsigned* lengths, size_t n, unsigned maxbits,
14664 unsigned* symbols) {
14665 9 size_t* bl_count = (size_t*)malloc(sizeof(size_t) * (maxbits + 1));
14666 9 size_t* next_code = (size_t*)malloc(sizeof(size_t) * (maxbits + 1));
14667 unsigned bits, i;
14668 unsigned code;
14669  
14670 1308 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 1299 9 1308
14671 1299 symbols[i] = 0;
14672 }
14673  
14674 /* 1) Count the number of codes for each code length. Let bl_count[N] be the
14675 number of codes of length N, N >= 1. */
14676 145 ▶ for (bits = 0; bits <= maxbits; bits++) {
Branch: 1 branch
BranchTrueFalseTotal
for (bits = 0; bits <= maxbits; bits++) { 136 9 145
14677 136 bl_count[bits] = 0;
14678 }
14679 1308 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 1299 9 1308
14680 assert(lengths[i] <= maxbits);
14681 1299 bl_count[lengths[i]]++;
14682 }
14683 /* 2) Find the numerical value of the smallest code for each code length. */
14684 9 code = 0;
14685 9 bl_count[0] = 0;
14686 136 ▶ for (bits = 1; bits <= maxbits; bits++) {
Branch: 1 branch
BranchTrueFalseTotal
for (bits = 1; bits <= maxbits; bits++) { 127 9 136
14687 127 code = (code + bl_count[bits-1]) << 1;
14688 127 next_code[bits] = code;
14689 }
14690 /* 3) Assign numerical values to all codes, using consecutive values for all
14691 codes of the same length with the base values determined at step 2. */
14692 1308 ▶ for (i = 0; i < n; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; i++) { 1299 9 1308
14693 1299 unsigned len = lengths[i];
14694 1299 ▶ if (len != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (len != 0) { 970 329 1299
14695 970 symbols[i] = next_code[len];
14696 970 next_code[len]++;
14697 }
14698 }
14699  
14700 9 free(bl_count);
14701 9 free(next_code);
14702 9 }
14703  
14704 264 void ZopfliCalculateEntropy(const size_t* count, size_t n, double* bitlengths) {
14705 static const double kInvLog2 = 1.4426950408889; /* 1.0 / log(2.0) */
14706 264 unsigned sum = 0;
14707 unsigned i;
14708 double log2sum;
14709 42504 ▶ for (i = 0; i < n; ++i) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; ++i) { 42240 264 42504
14710 42240 sum += count[i];
14711 }
14712 264 ▶ log2sum = (sum == 0 ? log(n) : log(sum)) * kInvLog2;
Branch: 1 branch
BranchTrueFalseTotal
log2sum = (sum == 0 ? log(n) : log(sum)) * kInvLog2; 36 228 264
14713 42504 ▶ for (i = 0; i < n; ++i) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < n; ++i) { 42240 264 42504
14714 /* When the count of the symbol is 0, but its cost is requested anyway, it
14715 means the symbol will appear at least once anyway, so give it the cost as if
14716 its count is 1.*/
14717 42240 ▶ if (count[i] == 0) bitlengths[i] = log2sum;
Branch: 1 branch
BranchTrueFalseTotal
if (count[i] == 0) bitlengths[i] = log2sum; 40893 1347 42240
14718 1347 else bitlengths[i] = log2sum - log(count[i]) * kInvLog2;
14719 /* Depending on compiler and architecture, the above subtraction of two
14720 floating point numbers may give a negative result very close to zero
14721 instead of zero (e.g. -5.973954e-17 with gcc 4.1.2 on Ubuntu 11.4). Clamp
14722 it to zero. These floating point imprecisions do not affect the cost model
14723 significantly so this is ok. */
14724 42240 ▶ if (bitlengths[i] < 0 && bitlengths[i] > -1e-5) bitlengths[i] = 0;
Branch: 2 branches
BranchTrueFalseTotal
if (bitlengths[i] < 0 && bitlengths[i] > -1e-5) bitlengths[i] = 0; 0 42240 42240
if (bitlengths[i] < 0 && bitlengths[i] > -1e-5) bitlengths[i] = 0; 0 0 0
14725 assert(bitlengths[i] >= 0);
14726 }
14727 264 }
14728  
14729 5229 void ZopfliCalculateBitLengths(const size_t* count, size_t n, int maxbits,
14730 unsigned* bitlengths) {
14731 5229 int error = ZopfliLengthLimitedCodeLengths(count, n, maxbits, bitlengths);
14732 (void) error;
14733 assert(!error);
14734 5229 }
14735 /*
14736 Copyright 2011 Google Inc. All Rights Reserved.
14737  
14738 Licensed under the Apache License, Version 2.0 (the "License");
14739 you may not use this file except in compliance with the License.
14740 You may obtain a copy of the License at
14741  
14742 http://www.apache.org/licenses/LICENSE-2.0
14743  
14744 Unless required by applicable law or agreed to in writing, software
14745 distributed under the License is distributed on an "AS IS" BASIS,
14746 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14747 See the License for the specific language governing permissions and
14748 limitations under the License.
14749  
14750 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14751 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14752 */
14753  
14754 /* util.h */
14755 /*
14756 Copyright 2011 Google Inc. All Rights Reserved.
14757  
14758 Licensed under the Apache License, Version 2.0 (the "License");
14759 you may not use this file except in compliance with the License.
14760 You may obtain a copy of the License at
14761  
14762 http://www.apache.org/licenses/LICENSE-2.0
14763  
14764 Unless required by applicable law or agreed to in writing, software
14765 distributed under the License is distributed on an "AS IS" BASIS,
14766 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14767 See the License for the specific language governing permissions and
14768 limitations under the License.
14769  
14770 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14771 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14772 */
14773  
14774 /*
14775 Several utilities, including: #defines to try different compression results,
14776 basic deflate specification values and generic program options.
14777 */
14778  
14779 #ifndef ZOPFLI_UTIL_H_
14780 #define ZOPFLI_UTIL_H_
14781  
14782 #include <string.h>
14783 #include <stdlib.h>
14784  
14785 /* Minimum and maximum length that can be encoded in deflate. */
14786 #define ZOPFLI_MAX_MATCH 258
14787 #define ZOPFLI_MIN_MATCH 3
14788  
14789 /* Number of distinct literal/length and distance symbols in DEFLATE */
14790 #define ZOPFLI_NUM_LL 288
14791 #define ZOPFLI_NUM_D 32
14792  
14793 /*
14794 The window size for deflate. Must be a power of two. This should be 32768, the
14795 maximum possible by the deflate spec. Anything less hurts compression more than
14796 speed.
14797 */
14798 #define ZOPFLI_WINDOW_SIZE 32768
14799  
14800 /*
14801 The window mask used to wrap indices into the window. This is why the
14802 window size must be a power of two.
14803 */
14804 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
14805  
14806 /*
14807 A block structure of huge, non-smart, blocks to divide the input into, to allow
14808 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
14809 The whole compression algorithm, including the smarter block splitting, will
14810 be executed independently on each huge block.
14811 Dividing into huge blocks hurts compression, but not much relative to the size.
14812 Set it to 0 to disable master blocks.
14813 */
14814 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
14815  
14816 /*
14817 Used to initialize costs for example
14818 */
14819 #define ZOPFLI_LARGE_FLOAT 1e30
14820  
14821 /*
14822 For longest match cache. max 256. Uses huge amounts of memory but makes it
14823 faster. Uses this many times three bytes per single byte of the input data.
14824 This is so because longest match finding has to find the exact distance
14825 that belongs to each length for the best lz77 strategy.
14826 Good values: e.g. 5, 8.
14827 */
14828 #define ZOPFLI_CACHE_LENGTH 8
14829  
14830 /*
14831 limit the max hash chain hits for this hash value. This has an effect only
14832 on files where the hash value is the same very often. On these files, this
14833 gives worse compression (the value should ideally be 32768, which is the
14834 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
14835 faster on some specific files.
14836 Good value: e.g. 8192.
14837 */
14838 #define ZOPFLI_MAX_CHAIN_HITS 8192
14839  
14840 /*
14841 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
14842 consumes a lot of memory but speeds it up. No effect on compression size.
14843 */
14844 #define ZOPFLI_LONGEST_MATCH_CACHE
14845  
14846 /*
14847 Enable to remember amount of successive identical bytes in the hash chain for
14848 finding longest match
14849 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
14850 This has no effect on the compression result, and enabling it increases speed.
14851 */
14852 #define ZOPFLI_HASH_SAME
14853  
14854 /*
14855 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
14856 best length so far is long enough. This is way faster for files with lots of
14857 identical bytes, on which the compressor is otherwise too slow. Regular files
14858 are unaffected or maybe a tiny bit slower.
14859 This has no effect on the compression result, only on speed.
14860 */
14861 #define ZOPFLI_HASH_SAME_HASH
14862  
14863 /*
14864 Enable this, to avoid slowness for files which are a repetition of the same
14865 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
14866 the compression result.
14867 */
14868 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
14869  
14870 /*
14871 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
14872 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
14873 varies from file to file.
14874 */
14875 #define ZOPFLI_LAZY_MATCHING
14876  
14877 /*
14878 Appends value to dynamically allocated memory, doubling its allocation size
14879 whenever needed.
14880  
14881 value: the value to append, type T
14882 data: pointer to the dynamic array to append to, type T**
14883 size: pointer to the size of the array to append to, type size_t*. This is the
14884 size that you consider the array to be, not the internal allocation size.
14885 Precondition: allocated size of data is at least a power of two greater than or
14886 equal than *size.
14887 */
14888 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
14889 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
14890 if (!((*size) & ((*size) - 1))) {\
14891 /*double alloc size if it's a power of two*/\
14892 void** data_void = reinterpret_cast<void**>(data);\
14893 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
14894 : realloc((*data), (*size) * 2 * sizeof(**data));\
14895 }\
14896 (*data)[(*size)] = (value);\
14897 (*size)++;\
14898 }
14899 #else /* C gives problems with strict-aliasing rules for (void**) cast */
14900 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
14901 if (!((*size) & ((*size) - 1))) {\
14902 /*double alloc size if it's a power of two*/\
14903 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
14904 : realloc((*data), (*size) * 2 * sizeof(**data));\
14905 }\
14906 (*data)[(*size)] = (value);\
14907 (*size)++;\
14908 }
14909 #endif
14910  
14911  
14912 #endif /* ZOPFLI_UTIL_H_ */
14913  
14914  
14915 /* zopfli.h */
14916 /*
14917 Copyright 2011 Google Inc. All Rights Reserved.
14918  
14919 Licensed under the Apache License, Version 2.0 (the "License");
14920 you may not use this file except in compliance with the License.
14921 You may obtain a copy of the License at
14922  
14923 http://www.apache.org/licenses/LICENSE-2.0
14924  
14925 Unless required by applicable law or agreed to in writing, software
14926 distributed under the License is distributed on an "AS IS" BASIS,
14927 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14928 See the License for the specific language governing permissions and
14929 limitations under the License.
14930  
14931 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
14932 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
14933 */
14934  
14935 #ifndef ZOPFLI_ZOPFLI_H_
14936 #define ZOPFLI_ZOPFLI_H_
14937  
14938 #include <stddef.h>
14939 #include <stdlib.h> /* for size_t */
14940  
14941 #ifdef __cplusplus
14942 extern "C" {
14943 #endif
14944  
14945 /*
14946 Options used throughout the program.
14947 */
14948 typedef struct ZopfliOptions {
14949 /* Whether to print output */
14950 int verbose;
14951  
14952 /* Whether to print more detailed output */
14953 int verbose_more;
14954  
14955 /*
14956 Maximum amount of times to rerun forward and backward pass to optimize LZ77
14957 compression cost. Good values: 10, 15 for small files, 5 for files over
14958 several MB in size or it will be too slow.
14959 */
14960 int numiterations;
14961  
14962 /*
14963 If true, splits the data in multiple deflate blocks with optimal choice
14964 for the block boundaries. Block splitting gives better compression. Default:
14965 true (1).
14966 */
14967 int blocksplitting;
14968  
14969 /*
14970 No longer used, left for compatibility.
14971 */
14972 int blocksplittinglast;
14973  
14974 /*
14975 Maximum amount of blocks to split into (0 for unlimited, but this can give
14976 extreme results that hurt compression on some files). Default value: 15.
14977 */
14978 int blocksplittingmax;
14979 } ZopfliOptions;
14980  
14981 /* Initializes options with default values. */
14982 void ZopfliInitOptions(ZopfliOptions* options);
14983  
14984 /* Output format */
14985 typedef enum {
14986 ZOPFLI_FORMAT_GZIP,
14987 ZOPFLI_FORMAT_ZLIB,
14988 ZOPFLI_FORMAT_DEFLATE
14989 } ZopfliFormat;
14990  
14991 /*
14992 Compresses according to the given output format and appends the result to the
14993 output.
14994  
14995 options: global program options
14996 output_type: the output format to use
14997 out: pointer to the dynamic output array to which the result is appended. Must
14998 be freed after use
14999 outsize: pointer to the dynamic output array size
15000 */
15001 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
15002 const unsigned char* in, size_t insize,
15003 unsigned char** out, size_t* outsize);
15004  
15005 #ifdef __cplusplus
15006 } // extern "C"
15007 #endif
15008  
15009 #endif /* ZOPFLI_ZOPFLI_H_ */
15010  
15011  
15012 #include <assert.h>
15013 #include <stdio.h>
15014 #include <stdlib.h>
15015  
15016 3 void ZopfliInitOptions(ZopfliOptions* options) {
15017 3 options->verbose = 0;
15018 3 options->verbose_more = 0;
15019 3 options->numiterations = 15;
15020 3 options->blocksplitting = 1;
15021 3 options->blocksplittinglast = 0;
15022 3 options->blocksplittingmax = 15;
15023 3 }
15024 /*
15025 Copyright 2013 Google Inc. All Rights Reserved.
15026  
15027 Licensed under the Apache License, Version 2.0 (the "License");
15028 you may not use this file except in compliance with the License.
15029 You may obtain a copy of the License at
15030  
15031 http://www.apache.org/licenses/LICENSE-2.0
15032  
15033 Unless required by applicable law or agreed to in writing, software
15034 distributed under the License is distributed on an "AS IS" BASIS,
15035 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15036 See the License for the specific language governing permissions and
15037 limitations under the License.
15038  
15039 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15040 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15041 */
15042  
15043 /* zlib_container.h */
15044 /*
15045 Copyright 2013 Google Inc. All Rights Reserved.
15046  
15047 Licensed under the Apache License, Version 2.0 (the "License");
15048 you may not use this file except in compliance with the License.
15049 You may obtain a copy of the License at
15050  
15051 http://www.apache.org/licenses/LICENSE-2.0
15052  
15053 Unless required by applicable law or agreed to in writing, software
15054 distributed under the License is distributed on an "AS IS" BASIS,
15055 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15056 See the License for the specific language governing permissions and
15057 limitations under the License.
15058  
15059 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15060 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15061 */
15062  
15063 #ifndef ZOPFLI_ZLIB_H_
15064 #define ZOPFLI_ZLIB_H_
15065  
15066 /*
15067 Functions to compress according to the Zlib specification.
15068 */
15069  
15070 /* zopfli.h */
15071 /*
15072 Copyright 2011 Google Inc. All Rights Reserved.
15073  
15074 Licensed under the Apache License, Version 2.0 (the "License");
15075 you may not use this file except in compliance with the License.
15076 You may obtain a copy of the License at
15077  
15078 http://www.apache.org/licenses/LICENSE-2.0
15079  
15080 Unless required by applicable law or agreed to in writing, software
15081 distributed under the License is distributed on an "AS IS" BASIS,
15082 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15083 See the License for the specific language governing permissions and
15084 limitations under the License.
15085  
15086 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15087 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15088 */
15089  
15090 #ifndef ZOPFLI_ZOPFLI_H_
15091 #define ZOPFLI_ZOPFLI_H_
15092  
15093 #include <stddef.h>
15094 #include <stdlib.h> /* for size_t */
15095  
15096 #ifdef __cplusplus
15097 extern "C" {
15098 #endif
15099  
15100 /*
15101 Options used throughout the program.
15102 */
15103 typedef struct ZopfliOptions {
15104 /* Whether to print output */
15105 int verbose;
15106  
15107 /* Whether to print more detailed output */
15108 int verbose_more;
15109  
15110 /*
15111 Maximum amount of times to rerun forward and backward pass to optimize LZ77
15112 compression cost. Good values: 10, 15 for small files, 5 for files over
15113 several MB in size or it will be too slow.
15114 */
15115 int numiterations;
15116  
15117 /*
15118 If true, splits the data in multiple deflate blocks with optimal choice
15119 for the block boundaries. Block splitting gives better compression. Default:
15120 true (1).
15121 */
15122 int blocksplitting;
15123  
15124 /*
15125 No longer used, left for compatibility.
15126 */
15127 int blocksplittinglast;
15128  
15129 /*
15130 Maximum amount of blocks to split into (0 for unlimited, but this can give
15131 extreme results that hurt compression on some files). Default value: 15.
15132 */
15133 int blocksplittingmax;
15134 } ZopfliOptions;
15135  
15136 /* Initializes options with default values. */
15137 void ZopfliInitOptions(ZopfliOptions* options);
15138  
15139 /* Output format */
15140 typedef enum {
15141 ZOPFLI_FORMAT_GZIP,
15142 ZOPFLI_FORMAT_ZLIB,
15143 ZOPFLI_FORMAT_DEFLATE
15144 } ZopfliFormat;
15145  
15146 /*
15147 Compresses according to the given output format and appends the result to the
15148 output.
15149  
15150 options: global program options
15151 output_type: the output format to use
15152 out: pointer to the dynamic output array to which the result is appended. Must
15153 be freed after use
15154 outsize: pointer to the dynamic output array size
15155 */
15156 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
15157 const unsigned char* in, size_t insize,
15158 unsigned char** out, size_t* outsize);
15159  
15160 #ifdef __cplusplus
15161 } // extern "C"
15162 #endif
15163  
15164 #endif /* ZOPFLI_ZOPFLI_H_ */
15165  
15166  
15167 #ifdef __cplusplus
15168 extern "C" {
15169 #endif
15170  
15171 /*
15172 Compresses according to the zlib specification and append the compressed
15173 result to the output.
15174  
15175 options: global program options
15176 out: pointer to the dynamic output array to which the result is appended. Must
15177 be freed after use.
15178 outsize: pointer to the dynamic output array size.
15179 */
15180 void ZopfliZlibCompress(const ZopfliOptions* options,
15181 const unsigned char* in, size_t insize,
15182 unsigned char** out, size_t* outsize);
15183  
15184 #ifdef __cplusplus
15185 } // extern "C"
15186 #endif
15187  
15188 #endif /* ZOPFLI_ZLIB_H_ */
15189  
15190 /* util.h */
15191 /*
15192 Copyright 2011 Google Inc. All Rights Reserved.
15193  
15194 Licensed under the Apache License, Version 2.0 (the "License");
15195 you may not use this file except in compliance with the License.
15196 You may obtain a copy of the License at
15197  
15198 http://www.apache.org/licenses/LICENSE-2.0
15199  
15200 Unless required by applicable law or agreed to in writing, software
15201 distributed under the License is distributed on an "AS IS" BASIS,
15202 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15203 See the License for the specific language governing permissions and
15204 limitations under the License.
15205  
15206 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15207 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15208 */
15209  
15210 /*
15211 Several utilities, including: #defines to try different compression results,
15212 basic deflate specification values and generic program options.
15213 */
15214  
15215 #ifndef ZOPFLI_UTIL_H_
15216 #define ZOPFLI_UTIL_H_
15217  
15218 #include <string.h>
15219 #include <stdlib.h>
15220  
15221 /* Minimum and maximum length that can be encoded in deflate. */
15222 #define ZOPFLI_MAX_MATCH 258
15223 #define ZOPFLI_MIN_MATCH 3
15224  
15225 /* Number of distinct literal/length and distance symbols in DEFLATE */
15226 #define ZOPFLI_NUM_LL 288
15227 #define ZOPFLI_NUM_D 32
15228  
15229 /*
15230 The window size for deflate. Must be a power of two. This should be 32768, the
15231 maximum possible by the deflate spec. Anything less hurts compression more than
15232 speed.
15233 */
15234 #define ZOPFLI_WINDOW_SIZE 32768
15235  
15236 /*
15237 The window mask used to wrap indices into the window. This is why the
15238 window size must be a power of two.
15239 */
15240 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
15241  
15242 /*
15243 A block structure of huge, non-smart, blocks to divide the input into, to allow
15244 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
15245 The whole compression algorithm, including the smarter block splitting, will
15246 be executed independently on each huge block.
15247 Dividing into huge blocks hurts compression, but not much relative to the size.
15248 Set it to 0 to disable master blocks.
15249 */
15250 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
15251  
15252 /*
15253 Used to initialize costs for example
15254 */
15255 #define ZOPFLI_LARGE_FLOAT 1e30
15256  
15257 /*
15258 For longest match cache. max 256. Uses huge amounts of memory but makes it
15259 faster. Uses this many times three bytes per single byte of the input data.
15260 This is so because longest match finding has to find the exact distance
15261 that belongs to each length for the best lz77 strategy.
15262 Good values: e.g. 5, 8.
15263 */
15264 #define ZOPFLI_CACHE_LENGTH 8
15265  
15266 /*
15267 limit the max hash chain hits for this hash value. This has an effect only
15268 on files where the hash value is the same very often. On these files, this
15269 gives worse compression (the value should ideally be 32768, which is the
15270 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
15271 faster on some specific files.
15272 Good value: e.g. 8192.
15273 */
15274 #define ZOPFLI_MAX_CHAIN_HITS 8192
15275  
15276 /*
15277 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
15278 consumes a lot of memory but speeds it up. No effect on compression size.
15279 */
15280 #define ZOPFLI_LONGEST_MATCH_CACHE
15281  
15282 /*
15283 Enable to remember amount of successive identical bytes in the hash chain for
15284 finding longest match
15285 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
15286 This has no effect on the compression result, and enabling it increases speed.
15287 */
15288 #define ZOPFLI_HASH_SAME
15289  
15290 /*
15291 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
15292 best length so far is long enough. This is way faster for files with lots of
15293 identical bytes, on which the compressor is otherwise too slow. Regular files
15294 are unaffected or maybe a tiny bit slower.
15295 This has no effect on the compression result, only on speed.
15296 */
15297 #define ZOPFLI_HASH_SAME_HASH
15298  
15299 /*
15300 Enable this, to avoid slowness for files which are a repetition of the same
15301 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
15302 the compression result.
15303 */
15304 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
15305  
15306 /*
15307 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
15308 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
15309 varies from file to file.
15310 */
15311 #define ZOPFLI_LAZY_MATCHING
15312  
15313 /*
15314 Appends value to dynamically allocated memory, doubling its allocation size
15315 whenever needed.
15316  
15317 value: the value to append, type T
15318 data: pointer to the dynamic array to append to, type T**
15319 size: pointer to the size of the array to append to, type size_t*. This is the
15320 size that you consider the array to be, not the internal allocation size.
15321 Precondition: allocated size of data is at least a power of two greater than or
15322 equal than *size.
15323 */
15324 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
15325 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15326 if (!((*size) & ((*size) - 1))) {\
15327 /*double alloc size if it's a power of two*/\
15328 void** data_void = reinterpret_cast<void**>(data);\
15329 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
15330 : realloc((*data), (*size) * 2 * sizeof(**data));\
15331 }\
15332 (*data)[(*size)] = (value);\
15333 (*size)++;\
15334 }
15335 #else /* C gives problems with strict-aliasing rules for (void**) cast */
15336 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15337 if (!((*size) & ((*size) - 1))) {\
15338 /*double alloc size if it's a power of two*/\
15339 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
15340 : realloc((*data), (*size) * 2 * sizeof(**data));\
15341 }\
15342 (*data)[(*size)] = (value);\
15343 (*size)++;\
15344 }
15345 #endif
15346  
15347  
15348 #endif /* ZOPFLI_UTIL_H_ */
15349  
15350  
15351 #include <stdio.h>
15352  
15353 /* deflate.h */
15354 /*
15355 Copyright 2011 Google Inc. All Rights Reserved.
15356  
15357 Licensed under the Apache License, Version 2.0 (the "License");
15358 you may not use this file except in compliance with the License.
15359 You may obtain a copy of the License at
15360  
15361 http://www.apache.org/licenses/LICENSE-2.0
15362  
15363 Unless required by applicable law or agreed to in writing, software
15364 distributed under the License is distributed on an "AS IS" BASIS,
15365 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15366 See the License for the specific language governing permissions and
15367 limitations under the License.
15368  
15369 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15370 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15371 */
15372  
15373 #ifndef ZOPFLI_DEFLATE_H_
15374 #define ZOPFLI_DEFLATE_H_
15375  
15376 /*
15377 Functions to compress according to the DEFLATE specification, using the
15378 "squeeze" LZ77 compression backend.
15379 */
15380  
15381 /* lz77.h */
15382 /*
15383 Copyright 2011 Google Inc. All Rights Reserved.
15384  
15385 Licensed under the Apache License, Version 2.0 (the "License");
15386 you may not use this file except in compliance with the License.
15387 You may obtain a copy of the License at
15388  
15389 http://www.apache.org/licenses/LICENSE-2.0
15390  
15391 Unless required by applicable law or agreed to in writing, software
15392 distributed under the License is distributed on an "AS IS" BASIS,
15393 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15394 See the License for the specific language governing permissions and
15395 limitations under the License.
15396  
15397 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15398 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15399 */
15400  
15401 /*
15402 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
15403 compression.
15404 */
15405  
15406 #ifndef ZOPFLI_LZ77_H_
15407 #define ZOPFLI_LZ77_H_
15408  
15409 #include <stdlib.h>
15410  
15411 /* cache.h */
15412 /*
15413 Copyright 2011 Google Inc. All Rights Reserved.
15414  
15415 Licensed under the Apache License, Version 2.0 (the "License");
15416 you may not use this file except in compliance with the License.
15417 You may obtain a copy of the License at
15418  
15419 http://www.apache.org/licenses/LICENSE-2.0
15420  
15421 Unless required by applicable law or agreed to in writing, software
15422 distributed under the License is distributed on an "AS IS" BASIS,
15423 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15424 See the License for the specific language governing permissions and
15425 limitations under the License.
15426  
15427 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15428 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15429 */
15430  
15431 /*
15432 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
15433 */
15434  
15435 #ifndef ZOPFLI_CACHE_H_
15436 #define ZOPFLI_CACHE_H_
15437  
15438 /* util.h */
15439 /*
15440 Copyright 2011 Google Inc. All Rights Reserved.
15441  
15442 Licensed under the Apache License, Version 2.0 (the "License");
15443 you may not use this file except in compliance with the License.
15444 You may obtain a copy of the License at
15445  
15446 http://www.apache.org/licenses/LICENSE-2.0
15447  
15448 Unless required by applicable law or agreed to in writing, software
15449 distributed under the License is distributed on an "AS IS" BASIS,
15450 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15451 See the License for the specific language governing permissions and
15452 limitations under the License.
15453  
15454 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15455 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15456 */
15457  
15458 /*
15459 Several utilities, including: #defines to try different compression results,
15460 basic deflate specification values and generic program options.
15461 */
15462  
15463 #ifndef ZOPFLI_UTIL_H_
15464 #define ZOPFLI_UTIL_H_
15465  
15466 #include <string.h>
15467 #include <stdlib.h>
15468  
15469 /* Minimum and maximum length that can be encoded in deflate. */
15470 #define ZOPFLI_MAX_MATCH 258
15471 #define ZOPFLI_MIN_MATCH 3
15472  
15473 /* Number of distinct literal/length and distance symbols in DEFLATE */
15474 #define ZOPFLI_NUM_LL 288
15475 #define ZOPFLI_NUM_D 32
15476  
15477 /*
15478 The window size for deflate. Must be a power of two. This should be 32768, the
15479 maximum possible by the deflate spec. Anything less hurts compression more than
15480 speed.
15481 */
15482 #define ZOPFLI_WINDOW_SIZE 32768
15483  
15484 /*
15485 The window mask used to wrap indices into the window. This is why the
15486 window size must be a power of two.
15487 */
15488 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
15489  
15490 /*
15491 A block structure of huge, non-smart, blocks to divide the input into, to allow
15492 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
15493 The whole compression algorithm, including the smarter block splitting, will
15494 be executed independently on each huge block.
15495 Dividing into huge blocks hurts compression, but not much relative to the size.
15496 Set it to 0 to disable master blocks.
15497 */
15498 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
15499  
15500 /*
15501 Used to initialize costs for example
15502 */
15503 #define ZOPFLI_LARGE_FLOAT 1e30
15504  
15505 /*
15506 For longest match cache. max 256. Uses huge amounts of memory but makes it
15507 faster. Uses this many times three bytes per single byte of the input data.
15508 This is so because longest match finding has to find the exact distance
15509 that belongs to each length for the best lz77 strategy.
15510 Good values: e.g. 5, 8.
15511 */
15512 #define ZOPFLI_CACHE_LENGTH 8
15513  
15514 /*
15515 limit the max hash chain hits for this hash value. This has an effect only
15516 on files where the hash value is the same very often. On these files, this
15517 gives worse compression (the value should ideally be 32768, which is the
15518 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
15519 faster on some specific files.
15520 Good value: e.g. 8192.
15521 */
15522 #define ZOPFLI_MAX_CHAIN_HITS 8192
15523  
15524 /*
15525 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
15526 consumes a lot of memory but speeds it up. No effect on compression size.
15527 */
15528 #define ZOPFLI_LONGEST_MATCH_CACHE
15529  
15530 /*
15531 Enable to remember amount of successive identical bytes in the hash chain for
15532 finding longest match
15533 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
15534 This has no effect on the compression result, and enabling it increases speed.
15535 */
15536 #define ZOPFLI_HASH_SAME
15537  
15538 /*
15539 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
15540 best length so far is long enough. This is way faster for files with lots of
15541 identical bytes, on which the compressor is otherwise too slow. Regular files
15542 are unaffected or maybe a tiny bit slower.
15543 This has no effect on the compression result, only on speed.
15544 */
15545 #define ZOPFLI_HASH_SAME_HASH
15546  
15547 /*
15548 Enable this, to avoid slowness for files which are a repetition of the same
15549 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
15550 the compression result.
15551 */
15552 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
15553  
15554 /*
15555 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
15556 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
15557 varies from file to file.
15558 */
15559 #define ZOPFLI_LAZY_MATCHING
15560  
15561 /*
15562 Appends value to dynamically allocated memory, doubling its allocation size
15563 whenever needed.
15564  
15565 value: the value to append, type T
15566 data: pointer to the dynamic array to append to, type T**
15567 size: pointer to the size of the array to append to, type size_t*. This is the
15568 size that you consider the array to be, not the internal allocation size.
15569 Precondition: allocated size of data is at least a power of two greater than or
15570 equal than *size.
15571 */
15572 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
15573 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15574 if (!((*size) & ((*size) - 1))) {\
15575 /*double alloc size if it's a power of two*/\
15576 void** data_void = reinterpret_cast<void**>(data);\
15577 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
15578 : realloc((*data), (*size) * 2 * sizeof(**data));\
15579 }\
15580 (*data)[(*size)] = (value);\
15581 (*size)++;\
15582 }
15583 #else /* C gives problems with strict-aliasing rules for (void**) cast */
15584 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15585 if (!((*size) & ((*size) - 1))) {\
15586 /*double alloc size if it's a power of two*/\
15587 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
15588 : realloc((*data), (*size) * 2 * sizeof(**data));\
15589 }\
15590 (*data)[(*size)] = (value);\
15591 (*size)++;\
15592 }
15593 #endif
15594  
15595  
15596 #endif /* ZOPFLI_UTIL_H_ */
15597  
15598  
15599 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
15600  
15601 /*
15602 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
15603 values.
15604 This is needed because the squeeze runs will ask these values multiple times for
15605 the same position.
15606 Uses large amounts of memory, since it has to remember the distance belonging
15607 to every possible shorter-than-the-best length (the so called "sublen" array).
15608 */
15609 typedef struct ZopfliLongestMatchCache {
15610 unsigned short* length;
15611 unsigned short* dist;
15612 unsigned char* sublen;
15613 } ZopfliLongestMatchCache;
15614  
15615 /* Initializes the ZopfliLongestMatchCache. */
15616 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
15617  
15618 /* Frees up the memory of the ZopfliLongestMatchCache. */
15619 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
15620  
15621 /* Stores sublen array in the cache. */
15622 void ZopfliSublenToCache(const unsigned short* sublen,
15623 size_t pos, size_t length,
15624 ZopfliLongestMatchCache* lmc);
15625  
15626 /* Extracts sublen array from the cache. */
15627 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
15628 size_t pos, size_t length,
15629 unsigned short* sublen);
15630 /* Returns the length up to which could be stored in the cache. */
15631 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
15632 size_t pos, size_t length);
15633  
15634 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
15635  
15636 #endif /* ZOPFLI_CACHE_H_ */
15637  
15638 /* hash.h */
15639 /*
15640 Copyright 2011 Google Inc. All Rights Reserved.
15641  
15642 Licensed under the Apache License, Version 2.0 (the "License");
15643 you may not use this file except in compliance with the License.
15644 You may obtain a copy of the License at
15645  
15646 http://www.apache.org/licenses/LICENSE-2.0
15647  
15648 Unless required by applicable law or agreed to in writing, software
15649 distributed under the License is distributed on an "AS IS" BASIS,
15650 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15651 See the License for the specific language governing permissions and
15652 limitations under the License.
15653  
15654 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15655 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15656 */
15657  
15658 /*
15659 The hash for ZopfliFindLongestMatch of lz77.c.
15660 */
15661  
15662 #ifndef ZOPFLI_HASH_H_
15663 #define ZOPFLI_HASH_H_
15664  
15665 /* util.h */
15666 /*
15667 Copyright 2011 Google Inc. All Rights Reserved.
15668  
15669 Licensed under the Apache License, Version 2.0 (the "License");
15670 you may not use this file except in compliance with the License.
15671 You may obtain a copy of the License at
15672  
15673 http://www.apache.org/licenses/LICENSE-2.0
15674  
15675 Unless required by applicable law or agreed to in writing, software
15676 distributed under the License is distributed on an "AS IS" BASIS,
15677 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15678 See the License for the specific language governing permissions and
15679 limitations under the License.
15680  
15681 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15682 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15683 */
15684  
15685 /*
15686 Several utilities, including: #defines to try different compression results,
15687 basic deflate specification values and generic program options.
15688 */
15689  
15690 #ifndef ZOPFLI_UTIL_H_
15691 #define ZOPFLI_UTIL_H_
15692  
15693 #include <string.h>
15694 #include <stdlib.h>
15695  
15696 /* Minimum and maximum length that can be encoded in deflate. */
15697 #define ZOPFLI_MAX_MATCH 258
15698 #define ZOPFLI_MIN_MATCH 3
15699  
15700 /* Number of distinct literal/length and distance symbols in DEFLATE */
15701 #define ZOPFLI_NUM_LL 288
15702 #define ZOPFLI_NUM_D 32
15703  
15704 /*
15705 The window size for deflate. Must be a power of two. This should be 32768, the
15706 maximum possible by the deflate spec. Anything less hurts compression more than
15707 speed.
15708 */
15709 #define ZOPFLI_WINDOW_SIZE 32768
15710  
15711 /*
15712 The window mask used to wrap indices into the window. This is why the
15713 window size must be a power of two.
15714 */
15715 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
15716  
15717 /*
15718 A block structure of huge, non-smart, blocks to divide the input into, to allow
15719 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
15720 The whole compression algorithm, including the smarter block splitting, will
15721 be executed independently on each huge block.
15722 Dividing into huge blocks hurts compression, but not much relative to the size.
15723 Set it to 0 to disable master blocks.
15724 */
15725 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
15726  
15727 /*
15728 Used to initialize costs for example
15729 */
15730 #define ZOPFLI_LARGE_FLOAT 1e30
15731  
15732 /*
15733 For longest match cache. max 256. Uses huge amounts of memory but makes it
15734 faster. Uses this many times three bytes per single byte of the input data.
15735 This is so because longest match finding has to find the exact distance
15736 that belongs to each length for the best lz77 strategy.
15737 Good values: e.g. 5, 8.
15738 */
15739 #define ZOPFLI_CACHE_LENGTH 8
15740  
15741 /*
15742 limit the max hash chain hits for this hash value. This has an effect only
15743 on files where the hash value is the same very often. On these files, this
15744 gives worse compression (the value should ideally be 32768, which is the
15745 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
15746 faster on some specific files.
15747 Good value: e.g. 8192.
15748 */
15749 #define ZOPFLI_MAX_CHAIN_HITS 8192
15750  
15751 /*
15752 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
15753 consumes a lot of memory but speeds it up. No effect on compression size.
15754 */
15755 #define ZOPFLI_LONGEST_MATCH_CACHE
15756  
15757 /*
15758 Enable to remember amount of successive identical bytes in the hash chain for
15759 finding longest match
15760 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
15761 This has no effect on the compression result, and enabling it increases speed.
15762 */
15763 #define ZOPFLI_HASH_SAME
15764  
15765 /*
15766 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
15767 best length so far is long enough. This is way faster for files with lots of
15768 identical bytes, on which the compressor is otherwise too slow. Regular files
15769 are unaffected or maybe a tiny bit slower.
15770 This has no effect on the compression result, only on speed.
15771 */
15772 #define ZOPFLI_HASH_SAME_HASH
15773  
15774 /*
15775 Enable this, to avoid slowness for files which are a repetition of the same
15776 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
15777 the compression result.
15778 */
15779 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
15780  
15781 /*
15782 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
15783 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
15784 varies from file to file.
15785 */
15786 #define ZOPFLI_LAZY_MATCHING
15787  
15788 /*
15789 Appends value to dynamically allocated memory, doubling its allocation size
15790 whenever needed.
15791  
15792 value: the value to append, type T
15793 data: pointer to the dynamic array to append to, type T**
15794 size: pointer to the size of the array to append to, type size_t*. This is the
15795 size that you consider the array to be, not the internal allocation size.
15796 Precondition: allocated size of data is at least a power of two greater than or
15797 equal than *size.
15798 */
15799 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
15800 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15801 if (!((*size) & ((*size) - 1))) {\
15802 /*double alloc size if it's a power of two*/\
15803 void** data_void = reinterpret_cast<void**>(data);\
15804 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
15805 : realloc((*data), (*size) * 2 * sizeof(**data));\
15806 }\
15807 (*data)[(*size)] = (value);\
15808 (*size)++;\
15809 }
15810 #else /* C gives problems with strict-aliasing rules for (void**) cast */
15811 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
15812 if (!((*size) & ((*size) - 1))) {\
15813 /*double alloc size if it's a power of two*/\
15814 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
15815 : realloc((*data), (*size) * 2 * sizeof(**data));\
15816 }\
15817 (*data)[(*size)] = (value);\
15818 (*size)++;\
15819 }
15820 #endif
15821  
15822  
15823 #endif /* ZOPFLI_UTIL_H_ */
15824  
15825  
15826 typedef struct ZopfliHash {
15827 int* head; /* Hash value to index of its most recent occurrence. */
15828 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
15829 int* hashval; /* Index to hash value at this index. */
15830 int val; /* Current hash value. */
15831  
15832 #ifdef ZOPFLI_HASH_SAME_HASH
15833 /* Fields with similar purpose as the above hash, but for the second hash with
15834 a value that is calculated differently. */
15835 int* head2; /* Hash value to index of its most recent occurrence. */
15836 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
15837 int* hashval2; /* Index to hash value at this index. */
15838 int val2; /* Current hash value. */
15839 #endif
15840  
15841 #ifdef ZOPFLI_HASH_SAME
15842 unsigned short* same; /* Amount of repetitions of same byte after this .*/
15843 #endif
15844 } ZopfliHash;
15845  
15846 /* Allocates ZopfliHash memory. */
15847 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
15848  
15849 /* Resets all fields of ZopfliHash. */
15850 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
15851  
15852 /* Frees ZopfliHash memory. */
15853 void ZopfliCleanHash(ZopfliHash* h);
15854  
15855 /*
15856 Updates the hash values based on the current position in the array. All calls
15857 to this must be made for consecutive bytes.
15858 */
15859 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
15860 ZopfliHash* h);
15861  
15862 /*
15863 Prepopulates hash:
15864 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
15865 correctly.
15866 */
15867 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
15868 ZopfliHash* h);
15869  
15870 #endif /* ZOPFLI_HASH_H_ */
15871  
15872 /* zopfli.h */
15873 /*
15874 Copyright 2011 Google Inc. All Rights Reserved.
15875  
15876 Licensed under the Apache License, Version 2.0 (the "License");
15877 you may not use this file except in compliance with the License.
15878 You may obtain a copy of the License at
15879  
15880 http://www.apache.org/licenses/LICENSE-2.0
15881  
15882 Unless required by applicable law or agreed to in writing, software
15883 distributed under the License is distributed on an "AS IS" BASIS,
15884 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15885 See the License for the specific language governing permissions and
15886 limitations under the License.
15887  
15888 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
15889 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
15890 */
15891  
15892 #ifndef ZOPFLI_ZOPFLI_H_
15893 #define ZOPFLI_ZOPFLI_H_
15894  
15895 #include <stddef.h>
15896 #include <stdlib.h> /* for size_t */
15897  
15898 #ifdef __cplusplus
15899 extern "C" {
15900 #endif
15901  
15902 /*
15903 Options used throughout the program.
15904 */
15905 typedef struct ZopfliOptions {
15906 /* Whether to print output */
15907 int verbose;
15908  
15909 /* Whether to print more detailed output */
15910 int verbose_more;
15911  
15912 /*
15913 Maximum amount of times to rerun forward and backward pass to optimize LZ77
15914 compression cost. Good values: 10, 15 for small files, 5 for files over
15915 several MB in size or it will be too slow.
15916 */
15917 int numiterations;
15918  
15919 /*
15920 If true, splits the data in multiple deflate blocks with optimal choice
15921 for the block boundaries. Block splitting gives better compression. Default:
15922 true (1).
15923 */
15924 int blocksplitting;
15925  
15926 /*
15927 No longer used, left for compatibility.
15928 */
15929 int blocksplittinglast;
15930  
15931 /*
15932 Maximum amount of blocks to split into (0 for unlimited, but this can give
15933 extreme results that hurt compression on some files). Default value: 15.
15934 */
15935 int blocksplittingmax;
15936 } ZopfliOptions;
15937  
15938 /* Initializes options with default values. */
15939 void ZopfliInitOptions(ZopfliOptions* options);
15940  
15941 /* Output format */
15942 typedef enum {
15943 ZOPFLI_FORMAT_GZIP,
15944 ZOPFLI_FORMAT_ZLIB,
15945 ZOPFLI_FORMAT_DEFLATE
15946 } ZopfliFormat;
15947  
15948 /*
15949 Compresses according to the given output format and appends the result to the
15950 output.
15951  
15952 options: global program options
15953 output_type: the output format to use
15954 out: pointer to the dynamic output array to which the result is appended. Must
15955 be freed after use
15956 outsize: pointer to the dynamic output array size
15957 */
15958 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
15959 const unsigned char* in, size_t insize,
15960 unsigned char** out, size_t* outsize);
15961  
15962 #ifdef __cplusplus
15963 } // extern "C"
15964 #endif
15965  
15966 #endif /* ZOPFLI_ZOPFLI_H_ */
15967  
15968  
15969 /*
15970 Stores lit/length and dist pairs for LZ77.
15971 Parameter litlens: Contains the literal symbols or length values.
15972 Parameter dists: Contains the distances. A value is 0 to indicate that there is
15973 no dist and the corresponding litlens value is a literal instead of a length.
15974 Parameter size: The size of both the litlens and dists arrays.
15975 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
15976 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
15977  
15978 */
15979 typedef struct ZopfliLZ77Store {
15980 unsigned short* litlens; /* Lit or len. */
15981 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
15982 if > 0: length in corresponding litlens, this is the distance. */
15983 size_t size;
15984  
15985 const unsigned char* data; /* original data */
15986 size_t* pos; /* position in data where this LZ77 command begins */
15987  
15988 unsigned short* ll_symbol;
15989 unsigned short* d_symbol;
15990  
15991 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
15992 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
15993 precise histogram at every N symbols, and the rest can be calculated by
15994 looping through the actual symbols of this chunk. */
15995 size_t* ll_counts;
15996 size_t* d_counts;
15997 } ZopfliLZ77Store;
15998  
15999 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
16000 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
16001 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
16002 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
16003 size_t pos, ZopfliLZ77Store* store);
16004 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
16005 ZopfliLZ77Store* target);
16006 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
16007 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
16008 size_t lstart, size_t lend);
16009 /* Gets the histogram of lit/len and dist symbols in the given range, using the
16010 cumulative histograms, so faster than adding one by one for large range. Does
16011 not add the one end symbol of value 256. */
16012 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
16013 size_t lstart, size_t lend,
16014 size_t* ll_counts, size_t* d_counts);
16015  
16016 /*
16017 Some state information for compressing a block.
16018 This is currently a bit under-used (with mainly only the longest match cache),
16019 but is kept for easy future expansion.
16020 */
16021 typedef struct ZopfliBlockState {
16022 const ZopfliOptions* options;
16023  
16024 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
16025 /* Cache for length/distance pairs found so far. */
16026 ZopfliLongestMatchCache* lmc;
16027 #endif
16028  
16029 /* The start (inclusive) and end (not inclusive) of the current block. */
16030 size_t blockstart;
16031 size_t blockend;
16032 } ZopfliBlockState;
16033  
16034 void ZopfliInitBlockState(const ZopfliOptions* options,
16035 size_t blockstart, size_t blockend, int add_lmc,
16036 ZopfliBlockState* s);
16037 void ZopfliCleanBlockState(ZopfliBlockState* s);
16038  
16039 /*
16040 Finds the longest match (length and corresponding distance) for LZ77
16041 compression.
16042 Even when not using "sublen", it can be more efficient to provide an array,
16043 because only then the caching is used.
16044 array: the data
16045 pos: position in the data to find the match for
16046 size: size of the data
16047 limit: limit length to maximum this value (default should be 258). This allows
16048 finding a shorter dist for that length (= less extra bits). Must be
16049 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
16050 sublen: output array of 259 elements, or null. Has, for each length, the
16051 smallest distance required to reach this length. Only 256 of its 259 values
16052 are used, the first 3 are ignored (the shortest length is 3. It is purely
16053 for convenience that the array is made 3 longer).
16054 */
16055 void ZopfliFindLongestMatch(
16056 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
16057 size_t pos, size_t size, size_t limit,
16058 unsigned short* sublen, unsigned short* distance, unsigned short* length);
16059  
16060 /*
16061 Verifies if length and dist are indeed valid, only used for assertion.
16062 */
16063 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
16064 unsigned short dist, unsigned short length);
16065  
16066 /*
16067 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
16068 with the slow but better "squeeze" implementation.
16069 The result is placed in the ZopfliLZ77Store.
16070 If instart is larger than 0, it uses values before instart as starting
16071 dictionary.
16072 */
16073 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
16074 size_t instart, size_t inend,
16075 ZopfliLZ77Store* store, ZopfliHash* h);
16076  
16077 #endif /* ZOPFLI_LZ77_H_ */
16078  
16079 /* zopfli.h */
16080 /*
16081 Copyright 2011 Google Inc. All Rights Reserved.
16082  
16083 Licensed under the Apache License, Version 2.0 (the "License");
16084 you may not use this file except in compliance with the License.
16085 You may obtain a copy of the License at
16086  
16087 http://www.apache.org/licenses/LICENSE-2.0
16088  
16089 Unless required by applicable law or agreed to in writing, software
16090 distributed under the License is distributed on an "AS IS" BASIS,
16091 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16092 See the License for the specific language governing permissions and
16093 limitations under the License.
16094  
16095 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16096 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16097 */
16098  
16099 #ifndef ZOPFLI_ZOPFLI_H_
16100 #define ZOPFLI_ZOPFLI_H_
16101  
16102 #include <stddef.h>
16103 #include <stdlib.h> /* for size_t */
16104  
16105 #ifdef __cplusplus
16106 extern "C" {
16107 #endif
16108  
16109 /*
16110 Options used throughout the program.
16111 */
16112 typedef struct ZopfliOptions {
16113 /* Whether to print output */
16114 int verbose;
16115  
16116 /* Whether to print more detailed output */
16117 int verbose_more;
16118  
16119 /*
16120 Maximum amount of times to rerun forward and backward pass to optimize LZ77
16121 compression cost. Good values: 10, 15 for small files, 5 for files over
16122 several MB in size or it will be too slow.
16123 */
16124 int numiterations;
16125  
16126 /*
16127 If true, splits the data in multiple deflate blocks with optimal choice
16128 for the block boundaries. Block splitting gives better compression. Default:
16129 true (1).
16130 */
16131 int blocksplitting;
16132  
16133 /*
16134 No longer used, left for compatibility.
16135 */
16136 int blocksplittinglast;
16137  
16138 /*
16139 Maximum amount of blocks to split into (0 for unlimited, but this can give
16140 extreme results that hurt compression on some files). Default value: 15.
16141 */
16142 int blocksplittingmax;
16143 } ZopfliOptions;
16144  
16145 /* Initializes options with default values. */
16146 void ZopfliInitOptions(ZopfliOptions* options);
16147  
16148 /* Output format */
16149 typedef enum {
16150 ZOPFLI_FORMAT_GZIP,
16151 ZOPFLI_FORMAT_ZLIB,
16152 ZOPFLI_FORMAT_DEFLATE
16153 } ZopfliFormat;
16154  
16155 /*
16156 Compresses according to the given output format and appends the result to the
16157 output.
16158  
16159 options: global program options
16160 output_type: the output format to use
16161 out: pointer to the dynamic output array to which the result is appended. Must
16162 be freed after use
16163 outsize: pointer to the dynamic output array size
16164 */
16165 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
16166 const unsigned char* in, size_t insize,
16167 unsigned char** out, size_t* outsize);
16168  
16169 #ifdef __cplusplus
16170 } // extern "C"
16171 #endif
16172  
16173 #endif /* ZOPFLI_ZOPFLI_H_ */
16174  
16175  
16176 #ifdef __cplusplus
16177 extern "C" {
16178 #endif
16179  
16180 /*
16181 Compresses according to the deflate specification and append the compressed
16182 result to the output.
16183 This function will usually output multiple deflate blocks. If final is 1, then
16184 the final bit will be set on the last block.
16185  
16186 options: global program options
16187 btype: the deflate block type. Use 2 for best compression.
16188 -0: non compressed blocks (00)
16189 -1: blocks with fixed tree (01)
16190 -2: blocks with dynamic tree (10)
16191 final: whether this is the last section of the input, sets the final bit to the
16192 last deflate block.
16193 in: the input bytes
16194 insize: number of input bytes
16195 bp: bit pointer for the output array. This must initially be 0, and for
16196 consecutive calls must be reused (it can have values from 0-7). This is
16197 because deflate appends blocks as bit-based data, rather than on byte
16198 boundaries.
16199 out: pointer to the dynamic output array to which the result is appended. Must
16200 be freed after use.
16201 outsize: pointer to the dynamic output array size.
16202 */
16203 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
16204 const unsigned char* in, size_t insize,
16205 unsigned char* bp, unsigned char** out, size_t* outsize);
16206  
16207 /*
16208 Like ZopfliDeflate, but allows to specify start and end byte with instart and
16209 inend. Only that part is compressed, but earlier bytes are still used for the
16210 back window.
16211 */
16212 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
16213 const unsigned char* in, size_t instart, size_t inend,
16214 unsigned char* bp, unsigned char** out,
16215 size_t* outsize);
16216  
16217 /*
16218 Calculates block size in bits.
16219 litlens: lz77 lit/lengths
16220 dists: ll77 distances
16221 lstart: start of block
16222 lend: end of block (not inclusive)
16223 */
16224 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
16225 size_t lstart, size_t lend, int btype);
16226  
16227 /*
16228 Calculates block size in bits, automatically using the best btype.
16229 */
16230 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
16231 size_t lstart, size_t lend);
16232  
16233 #ifdef __cplusplus
16234 } // extern "C"
16235 #endif
16236  
16237 #endif /* ZOPFLI_DEFLATE_H_ */
16238  
16239  
16240  
16241 /* Calculates the adler32 checksum of the data */
16242 0 static unsigned adler32(const unsigned char* data, size_t size)
16243 {
16244 static const unsigned sums_overflow = 5550;
16245 0 unsigned s1 = 1;
16246 0 unsigned s2 = 1 >> 16;
16247  
16248 0 ▶ while (size > 0) {
Branch: 1 branch
BranchTrueFalseTotal
while (size > 0) { 0 0 0
16249 0 size_t amount = size > sums_overflow ? sums_overflow : size;
16250 0 size -= amount;
16251 0 ▶ while (amount > 0) {
Branch: 1 branch
BranchTrueFalseTotal
while (amount > 0) { 0 0 0
16252 0 s1 += (*data++);
16253 0 s2 += s1;
16254 0 amount--;
16255 }
16256 0 s1 %= 65521;
16257 0 s2 %= 65521;
16258 }
16259  
16260 0 return (s2 << 16) | s1;
16261 }
16262  
16263 0 void ZopfliZlibCompress(const ZopfliOptions* options,
16264 const unsigned char* in, size_t insize,
16265 unsigned char** out, size_t* outsize) {
16266 0 unsigned char bitpointer = 0;
16267 0 unsigned checksum = adler32(in, (unsigned)insize);
16268 0 unsigned cmf = 120; /* CM 8, CINFO 7. See zlib spec.*/
16269 0 unsigned flevel = 3;
16270 0 unsigned fdict = 0;
16271 0 unsigned cmfflg = 256 * cmf + fdict * 32 + flevel * 64;
16272 0 unsigned fcheck = 31 - cmfflg % 31;
16273 0 cmfflg += fcheck;
16274  
16275 0 ▶ ZOPFLI_APPEND_DATA(cmfflg / 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(cmfflg / 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(cmfflg / 256, out, outsize); 0 0 0
16276 0 ▶ ZOPFLI_APPEND_DATA(cmfflg % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(cmfflg % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(cmfflg % 256, out, outsize); 0 0 0
16277  
16278 0 ZopfliDeflate(options, 2 /* dynamic block */, 1 /* final */,
16279 in, insize, &bitpointer, out, outsize);
16280  
16281 0 ▶ ZOPFLI_APPEND_DATA((checksum >> 24) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((checksum >> 24) % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA((checksum >> 24) % 256, out, outsize); 0 0 0
16282 0 ▶ ZOPFLI_APPEND_DATA((checksum >> 16) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((checksum >> 16) % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA((checksum >> 16) % 256, out, outsize); 0 0 0
16283 0 ▶ ZOPFLI_APPEND_DATA((checksum >> 8) % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA((checksum >> 8) % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA((checksum >> 8) % 256, out, outsize); 0 0 0
16284 0 ▶ ZOPFLI_APPEND_DATA(checksum % 256, out, outsize);
Branch: 2 branches
BranchTrueFalseTotal
ZOPFLI_APPEND_DATA(checksum % 256, out, outsize); 0 0 0
ZOPFLI_APPEND_DATA(checksum % 256, out, outsize); 0 0 0
16285  
16286 0 ▶ if (options->verbose) {
Branch: 1 branch
BranchTrueFalseTotal
if (options->verbose) { 0 0 0
16287 0 fprintf(stderr,
16288 "Original Size: %d, Zlib: %d, Compression: %f%% Removed\n",
16289 0 (int)insize, (int)*outsize,
16290 0 100.0 * (double)(insize - *outsize) / (double)insize);
16291 }
16292 0 }
16293 /*
16294 Copyright 2011 Google Inc. All Rights Reserved.
16295  
16296 Licensed under the Apache License, Version 2.0 (the "License");
16297 you may not use this file except in compliance with the License.
16298 You may obtain a copy of the License at
16299  
16300 http://www.apache.org/licenses/LICENSE-2.0
16301  
16302 Unless required by applicable law or agreed to in writing, software
16303 distributed under the License is distributed on an "AS IS" BASIS,
16304 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16305 See the License for the specific language governing permissions and
16306 limitations under the License.
16307  
16308 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16309 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16310 */
16311  
16312 /*
16313 Zopfli compressor program. It can output gzip-, zlib- or deflate-compatible
16314 data. By default it creates a .gz file. This tool can only compress, not
16315 decompress. Decompression can be done by any standard gzip, zlib or deflate
16316 decompressor.
16317 */
16318  
16319 #include <assert.h>
16320 #include <stdio.h>
16321 #include <stdlib.h>
16322 #include <string.h>
16323  
16324 /* deflate.h */
16325 /*
16326 Copyright 2011 Google Inc. All Rights Reserved.
16327  
16328 Licensed under the Apache License, Version 2.0 (the "License");
16329 you may not use this file except in compliance with the License.
16330 You may obtain a copy of the License at
16331  
16332 http://www.apache.org/licenses/LICENSE-2.0
16333  
16334 Unless required by applicable law or agreed to in writing, software
16335 distributed under the License is distributed on an "AS IS" BASIS,
16336 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16337 See the License for the specific language governing permissions and
16338 limitations under the License.
16339  
16340 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16341 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16342 */
16343  
16344 #ifndef ZOPFLI_DEFLATE_H_
16345 #define ZOPFLI_DEFLATE_H_
16346  
16347 /*
16348 Functions to compress according to the DEFLATE specification, using the
16349 "squeeze" LZ77 compression backend.
16350 */
16351  
16352 /* lz77.h */
16353 /*
16354 Copyright 2011 Google Inc. All Rights Reserved.
16355  
16356 Licensed under the Apache License, Version 2.0 (the "License");
16357 you may not use this file except in compliance with the License.
16358 You may obtain a copy of the License at
16359  
16360 http://www.apache.org/licenses/LICENSE-2.0
16361  
16362 Unless required by applicable law or agreed to in writing, software
16363 distributed under the License is distributed on an "AS IS" BASIS,
16364 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16365 See the License for the specific language governing permissions and
16366 limitations under the License.
16367  
16368 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16369 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16370 */
16371  
16372 /*
16373 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
16374 compression.
16375 */
16376  
16377 #ifndef ZOPFLI_LZ77_H_
16378 #define ZOPFLI_LZ77_H_
16379  
16380 #include <stdlib.h>
16381  
16382 /* cache.h */
16383 /*
16384 Copyright 2011 Google Inc. All Rights Reserved.
16385  
16386 Licensed under the Apache License, Version 2.0 (the "License");
16387 you may not use this file except in compliance with the License.
16388 You may obtain a copy of the License at
16389  
16390 http://www.apache.org/licenses/LICENSE-2.0
16391  
16392 Unless required by applicable law or agreed to in writing, software
16393 distributed under the License is distributed on an "AS IS" BASIS,
16394 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16395 See the License for the specific language governing permissions and
16396 limitations under the License.
16397  
16398 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16399 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16400 */
16401  
16402 /*
16403 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
16404 */
16405  
16406 #ifndef ZOPFLI_CACHE_H_
16407 #define ZOPFLI_CACHE_H_
16408  
16409 /* util.h */
16410 /*
16411 Copyright 2011 Google Inc. All Rights Reserved.
16412  
16413 Licensed under the Apache License, Version 2.0 (the "License");
16414 you may not use this file except in compliance with the License.
16415 You may obtain a copy of the License at
16416  
16417 http://www.apache.org/licenses/LICENSE-2.0
16418  
16419 Unless required by applicable law or agreed to in writing, software
16420 distributed under the License is distributed on an "AS IS" BASIS,
16421 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16422 See the License for the specific language governing permissions and
16423 limitations under the License.
16424  
16425 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16426 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16427 */
16428  
16429 /*
16430 Several utilities, including: #defines to try different compression results,
16431 basic deflate specification values and generic program options.
16432 */
16433  
16434 #ifndef ZOPFLI_UTIL_H_
16435 #define ZOPFLI_UTIL_H_
16436  
16437 #include <string.h>
16438 #include <stdlib.h>
16439  
16440 /* Minimum and maximum length that can be encoded in deflate. */
16441 #define ZOPFLI_MAX_MATCH 258
16442 #define ZOPFLI_MIN_MATCH 3
16443  
16444 /* Number of distinct literal/length and distance symbols in DEFLATE */
16445 #define ZOPFLI_NUM_LL 288
16446 #define ZOPFLI_NUM_D 32
16447  
16448 /*
16449 The window size for deflate. Must be a power of two. This should be 32768, the
16450 maximum possible by the deflate spec. Anything less hurts compression more than
16451 speed.
16452 */
16453 #define ZOPFLI_WINDOW_SIZE 32768
16454  
16455 /*
16456 The window mask used to wrap indices into the window. This is why the
16457 window size must be a power of two.
16458 */
16459 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
16460  
16461 /*
16462 A block structure of huge, non-smart, blocks to divide the input into, to allow
16463 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
16464 The whole compression algorithm, including the smarter block splitting, will
16465 be executed independently on each huge block.
16466 Dividing into huge blocks hurts compression, but not much relative to the size.
16467 Set it to 0 to disable master blocks.
16468 */
16469 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
16470  
16471 /*
16472 Used to initialize costs for example
16473 */
16474 #define ZOPFLI_LARGE_FLOAT 1e30
16475  
16476 /*
16477 For longest match cache. max 256. Uses huge amounts of memory but makes it
16478 faster. Uses this many times three bytes per single byte of the input data.
16479 This is so because longest match finding has to find the exact distance
16480 that belongs to each length for the best lz77 strategy.
16481 Good values: e.g. 5, 8.
16482 */
16483 #define ZOPFLI_CACHE_LENGTH 8
16484  
16485 /*
16486 limit the max hash chain hits for this hash value. This has an effect only
16487 on files where the hash value is the same very often. On these files, this
16488 gives worse compression (the value should ideally be 32768, which is the
16489 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
16490 faster on some specific files.
16491 Good value: e.g. 8192.
16492 */
16493 #define ZOPFLI_MAX_CHAIN_HITS 8192
16494  
16495 /*
16496 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
16497 consumes a lot of memory but speeds it up. No effect on compression size.
16498 */
16499 #define ZOPFLI_LONGEST_MATCH_CACHE
16500  
16501 /*
16502 Enable to remember amount of successive identical bytes in the hash chain for
16503 finding longest match
16504 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
16505 This has no effect on the compression result, and enabling it increases speed.
16506 */
16507 #define ZOPFLI_HASH_SAME
16508  
16509 /*
16510 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
16511 best length so far is long enough. This is way faster for files with lots of
16512 identical bytes, on which the compressor is otherwise too slow. Regular files
16513 are unaffected or maybe a tiny bit slower.
16514 This has no effect on the compression result, only on speed.
16515 */
16516 #define ZOPFLI_HASH_SAME_HASH
16517  
16518 /*
16519 Enable this, to avoid slowness for files which are a repetition of the same
16520 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
16521 the compression result.
16522 */
16523 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
16524  
16525 /*
16526 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
16527 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
16528 varies from file to file.
16529 */
16530 #define ZOPFLI_LAZY_MATCHING
16531  
16532 /*
16533 Appends value to dynamically allocated memory, doubling its allocation size
16534 whenever needed.
16535  
16536 value: the value to append, type T
16537 data: pointer to the dynamic array to append to, type T**
16538 size: pointer to the size of the array to append to, type size_t*. This is the
16539 size that you consider the array to be, not the internal allocation size.
16540 Precondition: allocated size of data is at least a power of two greater than or
16541 equal than *size.
16542 */
16543 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
16544 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
16545 if (!((*size) & ((*size) - 1))) {\
16546 /*double alloc size if it's a power of two*/\
16547 void** data_void = reinterpret_cast<void**>(data);\
16548 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
16549 : realloc((*data), (*size) * 2 * sizeof(**data));\
16550 }\
16551 (*data)[(*size)] = (value);\
16552 (*size)++;\
16553 }
16554 #else /* C gives problems with strict-aliasing rules for (void**) cast */
16555 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
16556 if (!((*size) & ((*size) - 1))) {\
16557 /*double alloc size if it's a power of two*/\
16558 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
16559 : realloc((*data), (*size) * 2 * sizeof(**data));\
16560 }\
16561 (*data)[(*size)] = (value);\
16562 (*size)++;\
16563 }
16564 #endif
16565  
16566  
16567 #endif /* ZOPFLI_UTIL_H_ */
16568  
16569  
16570 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
16571  
16572 /*
16573 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
16574 values.
16575 This is needed because the squeeze runs will ask these values multiple times for
16576 the same position.
16577 Uses large amounts of memory, since it has to remember the distance belonging
16578 to every possible shorter-than-the-best length (the so called "sublen" array).
16579 */
16580 typedef struct ZopfliLongestMatchCache {
16581 unsigned short* length;
16582 unsigned short* dist;
16583 unsigned char* sublen;
16584 } ZopfliLongestMatchCache;
16585  
16586 /* Initializes the ZopfliLongestMatchCache. */
16587 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
16588  
16589 /* Frees up the memory of the ZopfliLongestMatchCache. */
16590 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
16591  
16592 /* Stores sublen array in the cache. */
16593 void ZopfliSublenToCache(const unsigned short* sublen,
16594 size_t pos, size_t length,
16595 ZopfliLongestMatchCache* lmc);
16596  
16597 /* Extracts sublen array from the cache. */
16598 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
16599 size_t pos, size_t length,
16600 unsigned short* sublen);
16601 /* Returns the length up to which could be stored in the cache. */
16602 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
16603 size_t pos, size_t length);
16604  
16605 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
16606  
16607 #endif /* ZOPFLI_CACHE_H_ */
16608  
16609 /* hash.h */
16610 /*
16611 Copyright 2011 Google Inc. All Rights Reserved.
16612  
16613 Licensed under the Apache License, Version 2.0 (the "License");
16614 you may not use this file except in compliance with the License.
16615 You may obtain a copy of the License at
16616  
16617 http://www.apache.org/licenses/LICENSE-2.0
16618  
16619 Unless required by applicable law or agreed to in writing, software
16620 distributed under the License is distributed on an "AS IS" BASIS,
16621 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16622 See the License for the specific language governing permissions and
16623 limitations under the License.
16624  
16625 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16626 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16627 */
16628  
16629 /*
16630 The hash for ZopfliFindLongestMatch of lz77.c.
16631 */
16632  
16633 #ifndef ZOPFLI_HASH_H_
16634 #define ZOPFLI_HASH_H_
16635  
16636 /* util.h */
16637 /*
16638 Copyright 2011 Google Inc. All Rights Reserved.
16639  
16640 Licensed under the Apache License, Version 2.0 (the "License");
16641 you may not use this file except in compliance with the License.
16642 You may obtain a copy of the License at
16643  
16644 http://www.apache.org/licenses/LICENSE-2.0
16645  
16646 Unless required by applicable law or agreed to in writing, software
16647 distributed under the License is distributed on an "AS IS" BASIS,
16648 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16649 See the License for the specific language governing permissions and
16650 limitations under the License.
16651  
16652 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16653 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16654 */
16655  
16656 /*
16657 Several utilities, including: #defines to try different compression results,
16658 basic deflate specification values and generic program options.
16659 */
16660  
16661 #ifndef ZOPFLI_UTIL_H_
16662 #define ZOPFLI_UTIL_H_
16663  
16664 #include <string.h>
16665 #include <stdlib.h>
16666  
16667 /* Minimum and maximum length that can be encoded in deflate. */
16668 #define ZOPFLI_MAX_MATCH 258
16669 #define ZOPFLI_MIN_MATCH 3
16670  
16671 /* Number of distinct literal/length and distance symbols in DEFLATE */
16672 #define ZOPFLI_NUM_LL 288
16673 #define ZOPFLI_NUM_D 32
16674  
16675 /*
16676 The window size for deflate. Must be a power of two. This should be 32768, the
16677 maximum possible by the deflate spec. Anything less hurts compression more than
16678 speed.
16679 */
16680 #define ZOPFLI_WINDOW_SIZE 32768
16681  
16682 /*
16683 The window mask used to wrap indices into the window. This is why the
16684 window size must be a power of two.
16685 */
16686 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
16687  
16688 /*
16689 A block structure of huge, non-smart, blocks to divide the input into, to allow
16690 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
16691 The whole compression algorithm, including the smarter block splitting, will
16692 be executed independently on each huge block.
16693 Dividing into huge blocks hurts compression, but not much relative to the size.
16694 Set it to 0 to disable master blocks.
16695 */
16696 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
16697  
16698 /*
16699 Used to initialize costs for example
16700 */
16701 #define ZOPFLI_LARGE_FLOAT 1e30
16702  
16703 /*
16704 For longest match cache. max 256. Uses huge amounts of memory but makes it
16705 faster. Uses this many times three bytes per single byte of the input data.
16706 This is so because longest match finding has to find the exact distance
16707 that belongs to each length for the best lz77 strategy.
16708 Good values: e.g. 5, 8.
16709 */
16710 #define ZOPFLI_CACHE_LENGTH 8
16711  
16712 /*
16713 limit the max hash chain hits for this hash value. This has an effect only
16714 on files where the hash value is the same very often. On these files, this
16715 gives worse compression (the value should ideally be 32768, which is the
16716 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
16717 faster on some specific files.
16718 Good value: e.g. 8192.
16719 */
16720 #define ZOPFLI_MAX_CHAIN_HITS 8192
16721  
16722 /*
16723 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
16724 consumes a lot of memory but speeds it up. No effect on compression size.
16725 */
16726 #define ZOPFLI_LONGEST_MATCH_CACHE
16727  
16728 /*
16729 Enable to remember amount of successive identical bytes in the hash chain for
16730 finding longest match
16731 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
16732 This has no effect on the compression result, and enabling it increases speed.
16733 */
16734 #define ZOPFLI_HASH_SAME
16735  
16736 /*
16737 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
16738 best length so far is long enough. This is way faster for files with lots of
16739 identical bytes, on which the compressor is otherwise too slow. Regular files
16740 are unaffected or maybe a tiny bit slower.
16741 This has no effect on the compression result, only on speed.
16742 */
16743 #define ZOPFLI_HASH_SAME_HASH
16744  
16745 /*
16746 Enable this, to avoid slowness for files which are a repetition of the same
16747 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
16748 the compression result.
16749 */
16750 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
16751  
16752 /*
16753 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
16754 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
16755 varies from file to file.
16756 */
16757 #define ZOPFLI_LAZY_MATCHING
16758  
16759 /*
16760 Appends value to dynamically allocated memory, doubling its allocation size
16761 whenever needed.
16762  
16763 value: the value to append, type T
16764 data: pointer to the dynamic array to append to, type T**
16765 size: pointer to the size of the array to append to, type size_t*. This is the
16766 size that you consider the array to be, not the internal allocation size.
16767 Precondition: allocated size of data is at least a power of two greater than or
16768 equal than *size.
16769 */
16770 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
16771 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
16772 if (!((*size) & ((*size) - 1))) {\
16773 /*double alloc size if it's a power of two*/\
16774 void** data_void = reinterpret_cast<void**>(data);\
16775 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
16776 : realloc((*data), (*size) * 2 * sizeof(**data));\
16777 }\
16778 (*data)[(*size)] = (value);\
16779 (*size)++;\
16780 }
16781 #else /* C gives problems with strict-aliasing rules for (void**) cast */
16782 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
16783 if (!((*size) & ((*size) - 1))) {\
16784 /*double alloc size if it's a power of two*/\
16785 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
16786 : realloc((*data), (*size) * 2 * sizeof(**data));\
16787 }\
16788 (*data)[(*size)] = (value);\
16789 (*size)++;\
16790 }
16791 #endif
16792  
16793  
16794 #endif /* ZOPFLI_UTIL_H_ */
16795  
16796  
16797 typedef struct ZopfliHash {
16798 int* head; /* Hash value to index of its most recent occurrence. */
16799 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
16800 int* hashval; /* Index to hash value at this index. */
16801 int val; /* Current hash value. */
16802  
16803 #ifdef ZOPFLI_HASH_SAME_HASH
16804 /* Fields with similar purpose as the above hash, but for the second hash with
16805 a value that is calculated differently. */
16806 int* head2; /* Hash value to index of its most recent occurrence. */
16807 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
16808 int* hashval2; /* Index to hash value at this index. */
16809 int val2; /* Current hash value. */
16810 #endif
16811  
16812 #ifdef ZOPFLI_HASH_SAME
16813 unsigned short* same; /* Amount of repetitions of same byte after this .*/
16814 #endif
16815 } ZopfliHash;
16816  
16817 /* Allocates ZopfliHash memory. */
16818 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
16819  
16820 /* Resets all fields of ZopfliHash. */
16821 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
16822  
16823 /* Frees ZopfliHash memory. */
16824 void ZopfliCleanHash(ZopfliHash* h);
16825  
16826 /*
16827 Updates the hash values based on the current position in the array. All calls
16828 to this must be made for consecutive bytes.
16829 */
16830 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
16831 ZopfliHash* h);
16832  
16833 /*
16834 Prepopulates hash:
16835 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
16836 correctly.
16837 */
16838 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
16839 ZopfliHash* h);
16840  
16841 #endif /* ZOPFLI_HASH_H_ */
16842  
16843 /* zopfli.h */
16844 /*
16845 Copyright 2011 Google Inc. All Rights Reserved.
16846  
16847 Licensed under the Apache License, Version 2.0 (the "License");
16848 you may not use this file except in compliance with the License.
16849 You may obtain a copy of the License at
16850  
16851 http://www.apache.org/licenses/LICENSE-2.0
16852  
16853 Unless required by applicable law or agreed to in writing, software
16854 distributed under the License is distributed on an "AS IS" BASIS,
16855 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16856 See the License for the specific language governing permissions and
16857 limitations under the License.
16858  
16859 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
16860 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
16861 */
16862  
16863 #ifndef ZOPFLI_ZOPFLI_H_
16864 #define ZOPFLI_ZOPFLI_H_
16865  
16866 #include <stddef.h>
16867 #include <stdlib.h> /* for size_t */
16868  
16869 #ifdef __cplusplus
16870 extern "C" {
16871 #endif
16872  
16873 /*
16874 Options used throughout the program.
16875 */
16876 typedef struct ZopfliOptions {
16877 /* Whether to print output */
16878 int verbose;
16879  
16880 /* Whether to print more detailed output */
16881 int verbose_more;
16882  
16883 /*
16884 Maximum amount of times to rerun forward and backward pass to optimize LZ77
16885 compression cost. Good values: 10, 15 for small files, 5 for files over
16886 several MB in size or it will be too slow.
16887 */
16888 int numiterations;
16889  
16890 /*
16891 If true, splits the data in multiple deflate blocks with optimal choice
16892 for the block boundaries. Block splitting gives better compression. Default:
16893 true (1).
16894 */
16895 int blocksplitting;
16896  
16897 /*
16898 No longer used, left for compatibility.
16899 */
16900 int blocksplittinglast;
16901  
16902 /*
16903 Maximum amount of blocks to split into (0 for unlimited, but this can give
16904 extreme results that hurt compression on some files). Default value: 15.
16905 */
16906 int blocksplittingmax;
16907 } ZopfliOptions;
16908  
16909 /* Initializes options with default values. */
16910 void ZopfliInitOptions(ZopfliOptions* options);
16911  
16912 /* Output format */
16913 typedef enum {
16914 ZOPFLI_FORMAT_GZIP,
16915 ZOPFLI_FORMAT_ZLIB,
16916 ZOPFLI_FORMAT_DEFLATE
16917 } ZopfliFormat;
16918  
16919 /*
16920 Compresses according to the given output format and appends the result to the
16921 output.
16922  
16923 options: global program options
16924 output_type: the output format to use
16925 out: pointer to the dynamic output array to which the result is appended. Must
16926 be freed after use
16927 outsize: pointer to the dynamic output array size
16928 */
16929 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
16930 const unsigned char* in, size_t insize,
16931 unsigned char** out, size_t* outsize);
16932  
16933 #ifdef __cplusplus
16934 } // extern "C"
16935 #endif
16936  
16937 #endif /* ZOPFLI_ZOPFLI_H_ */
16938  
16939  
16940 /*
16941 Stores lit/length and dist pairs for LZ77.
16942 Parameter litlens: Contains the literal symbols or length values.
16943 Parameter dists: Contains the distances. A value is 0 to indicate that there is
16944 no dist and the corresponding litlens value is a literal instead of a length.
16945 Parameter size: The size of both the litlens and dists arrays.
16946 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
16947 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
16948  
16949 */
16950 typedef struct ZopfliLZ77Store {
16951 unsigned short* litlens; /* Lit or len. */
16952 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
16953 if > 0: length in corresponding litlens, this is the distance. */
16954 size_t size;
16955  
16956 const unsigned char* data; /* original data */
16957 size_t* pos; /* position in data where this LZ77 command begins */
16958  
16959 unsigned short* ll_symbol;
16960 unsigned short* d_symbol;
16961  
16962 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
16963 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
16964 precise histogram at every N symbols, and the rest can be calculated by
16965 looping through the actual symbols of this chunk. */
16966 size_t* ll_counts;
16967 size_t* d_counts;
16968 } ZopfliLZ77Store;
16969  
16970 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
16971 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
16972 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
16973 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
16974 size_t pos, ZopfliLZ77Store* store);
16975 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
16976 ZopfliLZ77Store* target);
16977 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
16978 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
16979 size_t lstart, size_t lend);
16980 /* Gets the histogram of lit/len and dist symbols in the given range, using the
16981 cumulative histograms, so faster than adding one by one for large range. Does
16982 not add the one end symbol of value 256. */
16983 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
16984 size_t lstart, size_t lend,
16985 size_t* ll_counts, size_t* d_counts);
16986  
16987 /*
16988 Some state information for compressing a block.
16989 This is currently a bit under-used (with mainly only the longest match cache),
16990 but is kept for easy future expansion.
16991 */
16992 typedef struct ZopfliBlockState {
16993 const ZopfliOptions* options;
16994  
16995 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
16996 /* Cache for length/distance pairs found so far. */
16997 ZopfliLongestMatchCache* lmc;
16998 #endif
16999  
17000 /* The start (inclusive) and end (not inclusive) of the current block. */
17001 size_t blockstart;
17002 size_t blockend;
17003 } ZopfliBlockState;
17004  
17005 void ZopfliInitBlockState(const ZopfliOptions* options,
17006 size_t blockstart, size_t blockend, int add_lmc,
17007 ZopfliBlockState* s);
17008 void ZopfliCleanBlockState(ZopfliBlockState* s);
17009  
17010 /*
17011 Finds the longest match (length and corresponding distance) for LZ77
17012 compression.
17013 Even when not using "sublen", it can be more efficient to provide an array,
17014 because only then the caching is used.
17015 array: the data
17016 pos: position in the data to find the match for
17017 size: size of the data
17018 limit: limit length to maximum this value (default should be 258). This allows
17019 finding a shorter dist for that length (= less extra bits). Must be
17020 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
17021 sublen: output array of 259 elements, or null. Has, for each length, the
17022 smallest distance required to reach this length. Only 256 of its 259 values
17023 are used, the first 3 are ignored (the shortest length is 3. It is purely
17024 for convenience that the array is made 3 longer).
17025 */
17026 void ZopfliFindLongestMatch(
17027 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
17028 size_t pos, size_t size, size_t limit,
17029 unsigned short* sublen, unsigned short* distance, unsigned short* length);
17030  
17031 /*
17032 Verifies if length and dist are indeed valid, only used for assertion.
17033 */
17034 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
17035 unsigned short dist, unsigned short length);
17036  
17037 /*
17038 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
17039 with the slow but better "squeeze" implementation.
17040 The result is placed in the ZopfliLZ77Store.
17041 If instart is larger than 0, it uses values before instart as starting
17042 dictionary.
17043 */
17044 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
17045 size_t instart, size_t inend,
17046 ZopfliLZ77Store* store, ZopfliHash* h);
17047  
17048 #endif /* ZOPFLI_LZ77_H_ */
17049  
17050 /* zopfli.h */
17051 /*
17052 Copyright 2011 Google Inc. All Rights Reserved.
17053  
17054 Licensed under the Apache License, Version 2.0 (the "License");
17055 you may not use this file except in compliance with the License.
17056 You may obtain a copy of the License at
17057  
17058 http://www.apache.org/licenses/LICENSE-2.0
17059  
17060 Unless required by applicable law or agreed to in writing, software
17061 distributed under the License is distributed on an "AS IS" BASIS,
17062 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17063 See the License for the specific language governing permissions and
17064 limitations under the License.
17065  
17066 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17067 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17068 */
17069  
17070 #ifndef ZOPFLI_ZOPFLI_H_
17071 #define ZOPFLI_ZOPFLI_H_
17072  
17073 #include <stddef.h>
17074 #include <stdlib.h> /* for size_t */
17075  
17076 #ifdef __cplusplus
17077 extern "C" {
17078 #endif
17079  
17080 /*
17081 Options used throughout the program.
17082 */
17083 typedef struct ZopfliOptions {
17084 /* Whether to print output */
17085 int verbose;
17086  
17087 /* Whether to print more detailed output */
17088 int verbose_more;
17089  
17090 /*
17091 Maximum amount of times to rerun forward and backward pass to optimize LZ77
17092 compression cost. Good values: 10, 15 for small files, 5 for files over
17093 several MB in size or it will be too slow.
17094 */
17095 int numiterations;
17096  
17097 /*
17098 If true, splits the data in multiple deflate blocks with optimal choice
17099 for the block boundaries. Block splitting gives better compression. Default:
17100 true (1).
17101 */
17102 int blocksplitting;
17103  
17104 /*
17105 No longer used, left for compatibility.
17106 */
17107 int blocksplittinglast;
17108  
17109 /*
17110 Maximum amount of blocks to split into (0 for unlimited, but this can give
17111 extreme results that hurt compression on some files). Default value: 15.
17112 */
17113 int blocksplittingmax;
17114 } ZopfliOptions;
17115  
17116 /* Initializes options with default values. */
17117 void ZopfliInitOptions(ZopfliOptions* options);
17118  
17119 /* Output format */
17120 typedef enum {
17121 ZOPFLI_FORMAT_GZIP,
17122 ZOPFLI_FORMAT_ZLIB,
17123 ZOPFLI_FORMAT_DEFLATE
17124 } ZopfliFormat;
17125  
17126 /*
17127 Compresses according to the given output format and appends the result to the
17128 output.
17129  
17130 options: global program options
17131 output_type: the output format to use
17132 out: pointer to the dynamic output array to which the result is appended. Must
17133 be freed after use
17134 outsize: pointer to the dynamic output array size
17135 */
17136 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
17137 const unsigned char* in, size_t insize,
17138 unsigned char** out, size_t* outsize);
17139  
17140 #ifdef __cplusplus
17141 } // extern "C"
17142 #endif
17143  
17144 #endif /* ZOPFLI_ZOPFLI_H_ */
17145  
17146  
17147 #ifdef __cplusplus
17148 extern "C" {
17149 #endif
17150  
17151 /*
17152 Compresses according to the deflate specification and append the compressed
17153 result to the output.
17154 This function will usually output multiple deflate blocks. If final is 1, then
17155 the final bit will be set on the last block.
17156  
17157 options: global program options
17158 btype: the deflate block type. Use 2 for best compression.
17159 -0: non compressed blocks (00)
17160 -1: blocks with fixed tree (01)
17161 -2: blocks with dynamic tree (10)
17162 final: whether this is the last section of the input, sets the final bit to the
17163 last deflate block.
17164 in: the input bytes
17165 insize: number of input bytes
17166 bp: bit pointer for the output array. This must initially be 0, and for
17167 consecutive calls must be reused (it can have values from 0-7). This is
17168 because deflate appends blocks as bit-based data, rather than on byte
17169 boundaries.
17170 out: pointer to the dynamic output array to which the result is appended. Must
17171 be freed after use.
17172 outsize: pointer to the dynamic output array size.
17173 */
17174 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
17175 const unsigned char* in, size_t insize,
17176 unsigned char* bp, unsigned char** out, size_t* outsize);
17177  
17178 /*
17179 Like ZopfliDeflate, but allows to specify start and end byte with instart and
17180 inend. Only that part is compressed, but earlier bytes are still used for the
17181 back window.
17182 */
17183 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
17184 const unsigned char* in, size_t instart, size_t inend,
17185 unsigned char* bp, unsigned char** out,
17186 size_t* outsize);
17187  
17188 /*
17189 Calculates block size in bits.
17190 litlens: lz77 lit/lengths
17191 dists: ll77 distances
17192 lstart: start of block
17193 lend: end of block (not inclusive)
17194 */
17195 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
17196 size_t lstart, size_t lend, int btype);
17197  
17198 /*
17199 Calculates block size in bits, automatically using the best btype.
17200 */
17201 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
17202 size_t lstart, size_t lend);
17203  
17204 #ifdef __cplusplus
17205 } // extern "C"
17206 #endif
17207  
17208 #endif /* ZOPFLI_DEFLATE_H_ */
17209  
17210 /* gzip_container.h */
17211 /*
17212 Copyright 2013 Google Inc. All Rights Reserved.
17213  
17214 Licensed under the Apache License, Version 2.0 (the "License");
17215 you may not use this file except in compliance with the License.
17216 You may obtain a copy of the License at
17217  
17218 http://www.apache.org/licenses/LICENSE-2.0
17219  
17220 Unless required by applicable law or agreed to in writing, software
17221 distributed under the License is distributed on an "AS IS" BASIS,
17222 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17223 See the License for the specific language governing permissions and
17224 limitations under the License.
17225  
17226 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17227 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17228 */
17229  
17230 #ifndef ZOPFLI_GZIP_H_
17231 #define ZOPFLI_GZIP_H_
17232  
17233 /*
17234 Functions to compress according to the Gzip specification.
17235 */
17236  
17237 /* zopfli.h */
17238 /*
17239 Copyright 2011 Google Inc. All Rights Reserved.
17240  
17241 Licensed under the Apache License, Version 2.0 (the "License");
17242 you may not use this file except in compliance with the License.
17243 You may obtain a copy of the License at
17244  
17245 http://www.apache.org/licenses/LICENSE-2.0
17246  
17247 Unless required by applicable law or agreed to in writing, software
17248 distributed under the License is distributed on an "AS IS" BASIS,
17249 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17250 See the License for the specific language governing permissions and
17251 limitations under the License.
17252  
17253 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17254 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17255 */
17256  
17257 #ifndef ZOPFLI_ZOPFLI_H_
17258 #define ZOPFLI_ZOPFLI_H_
17259  
17260 #include <stddef.h>
17261 #include <stdlib.h> /* for size_t */
17262  
17263 #ifdef __cplusplus
17264 extern "C" {
17265 #endif
17266  
17267 /*
17268 Options used throughout the program.
17269 */
17270 typedef struct ZopfliOptions {
17271 /* Whether to print output */
17272 int verbose;
17273  
17274 /* Whether to print more detailed output */
17275 int verbose_more;
17276  
17277 /*
17278 Maximum amount of times to rerun forward and backward pass to optimize LZ77
17279 compression cost. Good values: 10, 15 for small files, 5 for files over
17280 several MB in size or it will be too slow.
17281 */
17282 int numiterations;
17283  
17284 /*
17285 If true, splits the data in multiple deflate blocks with optimal choice
17286 for the block boundaries. Block splitting gives better compression. Default:
17287 true (1).
17288 */
17289 int blocksplitting;
17290  
17291 /*
17292 No longer used, left for compatibility.
17293 */
17294 int blocksplittinglast;
17295  
17296 /*
17297 Maximum amount of blocks to split into (0 for unlimited, but this can give
17298 extreme results that hurt compression on some files). Default value: 15.
17299 */
17300 int blocksplittingmax;
17301 } ZopfliOptions;
17302  
17303 /* Initializes options with default values. */
17304 void ZopfliInitOptions(ZopfliOptions* options);
17305  
17306 /* Output format */
17307 typedef enum {
17308 ZOPFLI_FORMAT_GZIP,
17309 ZOPFLI_FORMAT_ZLIB,
17310 ZOPFLI_FORMAT_DEFLATE
17311 } ZopfliFormat;
17312  
17313 /*
17314 Compresses according to the given output format and appends the result to the
17315 output.
17316  
17317 options: global program options
17318 output_type: the output format to use
17319 out: pointer to the dynamic output array to which the result is appended. Must
17320 be freed after use
17321 outsize: pointer to the dynamic output array size
17322 */
17323 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
17324 const unsigned char* in, size_t insize,
17325 unsigned char** out, size_t* outsize);
17326  
17327 #ifdef __cplusplus
17328 } // extern "C"
17329 #endif
17330  
17331 #endif /* ZOPFLI_ZOPFLI_H_ */
17332  
17333  
17334 #ifdef __cplusplus
17335 extern "C" {
17336 #endif
17337  
17338 /*
17339 Compresses according to the gzip specification and append the compressed
17340 result to the output.
17341  
17342 options: global program options
17343 out: pointer to the dynamic output array to which the result is appended. Must
17344 be freed after use.
17345 outsize: pointer to the dynamic output array size.
17346 */
17347 void ZopfliGzipCompress(const ZopfliOptions* options,
17348 const unsigned char* in, size_t insize,
17349 unsigned char** out, size_t* outsize);
17350  
17351 #ifdef __cplusplus
17352 } // extern "C"
17353 #endif
17354  
17355 #endif /* ZOPFLI_GZIP_H_ */
17356  
17357 /* zlib_container.h */
17358 /*
17359 Copyright 2013 Google Inc. All Rights Reserved.
17360  
17361 Licensed under the Apache License, Version 2.0 (the "License");
17362 you may not use this file except in compliance with the License.
17363 You may obtain a copy of the License at
17364  
17365 http://www.apache.org/licenses/LICENSE-2.0
17366  
17367 Unless required by applicable law or agreed to in writing, software
17368 distributed under the License is distributed on an "AS IS" BASIS,
17369 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17370 See the License for the specific language governing permissions and
17371 limitations under the License.
17372  
17373 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17374 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17375 */
17376  
17377 #ifndef ZOPFLI_ZLIB_H_
17378 #define ZOPFLI_ZLIB_H_
17379  
17380 /*
17381 Functions to compress according to the Zlib specification.
17382 */
17383  
17384 /* zopfli.h */
17385 /*
17386 Copyright 2011 Google Inc. All Rights Reserved.
17387  
17388 Licensed under the Apache License, Version 2.0 (the "License");
17389 you may not use this file except in compliance with the License.
17390 You may obtain a copy of the License at
17391  
17392 http://www.apache.org/licenses/LICENSE-2.0
17393  
17394 Unless required by applicable law or agreed to in writing, software
17395 distributed under the License is distributed on an "AS IS" BASIS,
17396 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17397 See the License for the specific language governing permissions and
17398 limitations under the License.
17399  
17400 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17401 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17402 */
17403  
17404 #ifndef ZOPFLI_ZOPFLI_H_
17405 #define ZOPFLI_ZOPFLI_H_
17406  
17407 #include <stddef.h>
17408 #include <stdlib.h> /* for size_t */
17409  
17410 #ifdef __cplusplus
17411 extern "C" {
17412 #endif
17413  
17414 /*
17415 Options used throughout the program.
17416 */
17417 typedef struct ZopfliOptions {
17418 /* Whether to print output */
17419 int verbose;
17420  
17421 /* Whether to print more detailed output */
17422 int verbose_more;
17423  
17424 /*
17425 Maximum amount of times to rerun forward and backward pass to optimize LZ77
17426 compression cost. Good values: 10, 15 for small files, 5 for files over
17427 several MB in size or it will be too slow.
17428 */
17429 int numiterations;
17430  
17431 /*
17432 If true, splits the data in multiple deflate blocks with optimal choice
17433 for the block boundaries. Block splitting gives better compression. Default:
17434 true (1).
17435 */
17436 int blocksplitting;
17437  
17438 /*
17439 No longer used, left for compatibility.
17440 */
17441 int blocksplittinglast;
17442  
17443 /*
17444 Maximum amount of blocks to split into (0 for unlimited, but this can give
17445 extreme results that hurt compression on some files). Default value: 15.
17446 */
17447 int blocksplittingmax;
17448 } ZopfliOptions;
17449  
17450 /* Initializes options with default values. */
17451 void ZopfliInitOptions(ZopfliOptions* options);
17452  
17453 /* Output format */
17454 typedef enum {
17455 ZOPFLI_FORMAT_GZIP,
17456 ZOPFLI_FORMAT_ZLIB,
17457 ZOPFLI_FORMAT_DEFLATE
17458 } ZopfliFormat;
17459  
17460 /*
17461 Compresses according to the given output format and appends the result to the
17462 output.
17463  
17464 options: global program options
17465 output_type: the output format to use
17466 out: pointer to the dynamic output array to which the result is appended. Must
17467 be freed after use
17468 outsize: pointer to the dynamic output array size
17469 */
17470 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
17471 const unsigned char* in, size_t insize,
17472 unsigned char** out, size_t* outsize);
17473  
17474 #ifdef __cplusplus
17475 } // extern "C"
17476 #endif
17477  
17478 #endif /* ZOPFLI_ZOPFLI_H_ */
17479  
17480  
17481 #ifdef __cplusplus
17482 extern "C" {
17483 #endif
17484  
17485 /*
17486 Compresses according to the zlib specification and append the compressed
17487 result to the output.
17488  
17489 options: global program options
17490 out: pointer to the dynamic output array to which the result is appended. Must
17491 be freed after use.
17492 outsize: pointer to the dynamic output array size.
17493 */
17494 void ZopfliZlibCompress(const ZopfliOptions* options,
17495 const unsigned char* in, size_t insize,
17496 unsigned char** out, size_t* outsize);
17497  
17498 #ifdef __cplusplus
17499 } // extern "C"
17500 #endif
17501  
17502 #endif /* ZOPFLI_ZLIB_H_ */
17503  
17504  
17505 /* Windows workaround for stdout output. */
17506 #if _WIN32
17507 #include <fcntl.h>
17508 #endif
17509  
17510 /*
17511 Loads a file into a memory array. Returns 1 on success, 0 if file doesn't exist
17512 or couldn't be opened.
17513 */
17514 0 static int LoadFile(const char* filename,
17515 unsigned char** out, size_t* outsize) {
17516 FILE* file;
17517  
17518 0 *out = 0;
17519 0 *outsize = 0;
17520 0 file = fopen(filename, "rb");
17521 0 ▶ if (!file) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!file) return 0; 0 0 0
17522  
17523 0 fseek(file , 0 , SEEK_END);
17524 0 *outsize = ftell(file);
17525 0 ▶ if(*outsize > 2147483647) {
Branch: 1 branch
BranchTrueFalseTotal
if(*outsize > 2147483647) { 0 0 0
17526 0 fprintf(stderr,"Files larger than 2GB are not supported.\n");
17527 0 exit(EXIT_FAILURE);
17528 }
17529 0 rewind(file);
17530  
17531 0 *out = (unsigned char*)malloc(*outsize);
17532  
17533 0 ▶ if (*outsize && (*out)) {
Branch: 2 branches
BranchTrueFalseTotal
if (*outsize && (*out)) { 0 0 0
if (*outsize && (*out)) { 0 0 0
17534 0 size_t testsize = fread(*out, 1, *outsize, file);
17535 0 ▶ if (testsize != *outsize) {
Branch: 1 branch
BranchTrueFalseTotal
if (testsize != *outsize) { 0 0 0
17536 /* It could be a directory */
17537 0 free(*out);
17538 0 *out = 0;
17539 0 *outsize = 0;
17540 0 fclose(file);
17541 0 return 0;
17542 }
17543 }
17544  
17545 assert(!(*outsize) || out); /* If size is not zero, out must be allocated. */
17546 0 fclose(file);
17547 0 return 1;
17548 }
17549  
17550 /*
17551 Saves a file from a memory array, overwriting the file if it existed.
17552 */
17553 0 static void SaveFile(const char* filename,
17554 const unsigned char* in, size_t insize) {
17555 0 FILE* file = fopen(filename, "wb" );
17556 0 ▶ if (file == NULL) {
Branch: 1 branch
BranchTrueFalseTotal
if (file == NULL) { 0 0 0
17557 0 fprintf(stderr,"Error: Cannot write to output file, terminating.\n");
17558 0 exit (EXIT_FAILURE);
17559 }
17560 assert(file);
17561 0 fwrite((char*)in, 1, insize, file);
17562 0 fclose(file);
17563 0 }
17564  
17565 /*
17566 outfilename: filename to write output to, or 0 to write to stdout instead
17567 */
17568 0 static void CompressFile(const ZopfliOptions* options,
17569 ZopfliFormat output_type,
17570 const char* infilename,
17571 const char* outfilename) {
17572 unsigned char* in;
17573 size_t insize;
17574 0 unsigned char* out = 0;
17575 0 size_t outsize = 0;
17576 0 ▶ if (!LoadFile(infilename, &in, &insize)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!LoadFile(infilename, &in, &insize)) { 0 0 0
17577 0 fprintf(stderr, "Invalid filename: %s\n", infilename);
17578 0 return;
17579 }
17580  
17581 0 ZopfliCompress(options, output_type, in, insize, &out, &outsize);
17582  
17583 0 ▶ if (outfilename) {
Branch: 1 branch
BranchTrueFalseTotal
if (outfilename) { 0 0 0
17584 0 SaveFile(outfilename, out, outsize);
17585 } else {
17586 #if _WIN32
17587 /* Windows workaround for stdout output. */
17588 _setmode(_fileno(stdout), _O_BINARY);
17589 #endif
17590 0 fwrite(out, 1, outsize, stdout);
17591 }
17592  
17593 0 free(out);
17594 0 free(in);
17595 }
17596  
17597 /*
17598 Add two strings together. Size does not matter. Result must be freed.
17599 */
17600 0 static char* AddStrings(const char* str1, const char* str2) {
17601 0 size_t len = strlen(str1) + strlen(str2);
17602 0 char* result = (char*)malloc(len + 1);
17603 0 ▶ if (!result) exit(-1); /* Allocation failed. */
Branch: 1 branch
BranchTrueFalseTotal
if (!result) exit(-1); /* Allocation failed. */ 0 0 0
17604 0 strcpy(result, str1);
17605 0 strcat(result, str2);
17606 0 return result;
17607 }
17608  
17609 0 static char StringsEqual(const char* str1, const char* str2) {
17610 0 return strcmp(str1, str2) == 0;
17611 }
17612  
17613 0 int main(int argc, char* argv[]) {
17614 ZopfliOptions options;
17615 0 ZopfliFormat output_type = ZOPFLI_FORMAT_GZIP;
17616 0 const char* filename = 0;
17617 0 int output_to_stdout = 0;
17618 int i;
17619  
17620 0 ZopfliInitOptions(&options);
17621  
17622 0 ▶ for (i = 1; i < argc; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 1; i < argc; i++) { 0 0 0
17623 0 const char* arg = argv[i];
17624 0 ▶ if (StringsEqual(arg, "-v")) options.verbose = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (StringsEqual(arg, "-v")) options.verbose = 1; 0 0 0
17625 0 ▶ else if (StringsEqual(arg, "-c")) output_to_stdout = 1;
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "-c")) output_to_stdout = 1; 0 0 0
17626 0 ▶ else if (StringsEqual(arg, "--deflate")) {
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "--deflate")) { 0 0 0
17627 0 output_type = ZOPFLI_FORMAT_DEFLATE;
17628 }
17629 0 ▶ else if (StringsEqual(arg, "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB;
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB; 0 0 0
17630 0 ▶ else if (StringsEqual(arg, "--gzip")) output_type = ZOPFLI_FORMAT_GZIP;
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "--gzip")) output_type = ZOPFLI_FORMAT_GZIP; 0 0 0
17631 0 ▶ else if (StringsEqual(arg, "--splitlast")) /* Ignore */;
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "--splitlast")) /* Ignore */; 0 0 0
17632 0 ▶ else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i'
Branch: 3 branches
BranchTrueFalseTotal
else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' 0 0 0
else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' 0 0 0
else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' 0 0 0
17633 0 ▶ && arg[3] >= '0' && arg[3] <= '9') {
Branch: 2 branches
BranchTrueFalseTotal
&& arg[3] >= '0' && arg[3] <= '9') { 0 0 0
&& arg[3] >= '0' && arg[3] <= '9') { 0 0 0
17634 0 options.numiterations = atoi(arg + 3);
17635 }
17636 0 ▶ else if (StringsEqual(arg, "-h")) {
Branch: 1 branch
BranchTrueFalseTotal
else if (StringsEqual(arg, "-h")) { 0 0 0
17637 0 fprintf(stderr,
17638 "Usage: zopfli [OPTION]... FILE...\n"
17639 " -h gives this help\n"
17640 " -c write the result on standard output, instead of disk"
17641 " filename + '.gz'\n"
17642 " -v verbose mode\n"
17643 " --i# perform # iterations (default 15). More gives"
17644 " more compression but is slower."
17645 " Examples: --i10, --i50, --i1000\n");
17646 0 fprintf(stderr,
17647 " --gzip output to gzip format (default)\n"
17648 " --zlib output to zlib format instead of gzip\n"
17649 " --deflate output to deflate format instead of gzip\n"
17650 " --splitlast ignored, left for backwards compatibility\n");
17651 0 return 0;
17652 }
17653 }
17654  
17655 0 ▶ if (options.numiterations < 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (options.numiterations < 1) { 0 0 0
17656 0 fprintf(stderr, "Error: must have 1 or more iterations\n");
17657 0 return 0;
17658 }
17659  
17660 0 ▶ for (i = 1; i < argc; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 1; i < argc; i++) { 0 0 0
17661 0 ▶ if (argv[i][0] != '-') {
Branch: 1 branch
BranchTrueFalseTotal
if (argv[i][0] != '-') { 0 0 0
17662 char* outfilename;
17663 0 filename = argv[i];
17664 0 ▶ if (output_to_stdout) {
Branch: 1 branch
BranchTrueFalseTotal
if (output_to_stdout) { 0 0 0
17665 0 outfilename = 0;
17666 0 ▶ } else if (output_type == ZOPFLI_FORMAT_GZIP) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (output_type == ZOPFLI_FORMAT_GZIP) { 0 0 0
17667 0 outfilename = AddStrings(filename, ".gz");
17668 0 ▶ } else if (output_type == ZOPFLI_FORMAT_ZLIB) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (output_type == ZOPFLI_FORMAT_ZLIB) { 0 0 0
17669 0 outfilename = AddStrings(filename, ".zlib");
17670 } else {
17671 assert(output_type == ZOPFLI_FORMAT_DEFLATE);
17672 0 outfilename = AddStrings(filename, ".deflate");
17673 }
17674 0 ▶ if (options.verbose && outfilename) {
Branch: 2 branches
BranchTrueFalseTotal
if (options.verbose && outfilename) { 0 0 0
if (options.verbose && outfilename) { 0 0 0
17675 0 fprintf(stderr, "Saving to: %s\n", outfilename);
17676 }
17677 0 CompressFile(&options, output_type, filename, outfilename);
17678 0 free(outfilename);
17679 }
17680 }
17681  
17682 0 ▶ if (!filename) {
Branch: 1 branch
BranchTrueFalseTotal
if (!filename) { 0 0 0
17683 0 fprintf(stderr,
17684 "Please provide filename\nFor help, type: %s -h\n", argv[0]);
17685 }
17686  
17687 0 return 0;
17688 }
17689 /*
17690 Copyright 2011 Google Inc. All Rights Reserved.
17691  
17692 Licensed under the Apache License, Version 2.0 (the "License");
17693 you may not use this file except in compliance with the License.
17694 You may obtain a copy of the License at
17695  
17696 http://www.apache.org/licenses/LICENSE-2.0
17697  
17698 Unless required by applicable law or agreed to in writing, software
17699 distributed under the License is distributed on an "AS IS" BASIS,
17700 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17701 See the License for the specific language governing permissions and
17702 limitations under the License.
17703  
17704 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17705 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17706 */
17707  
17708 /* zopfli.h */
17709 /*
17710 Copyright 2011 Google Inc. All Rights Reserved.
17711  
17712 Licensed under the Apache License, Version 2.0 (the "License");
17713 you may not use this file except in compliance with the License.
17714 You may obtain a copy of the License at
17715  
17716 http://www.apache.org/licenses/LICENSE-2.0
17717  
17718 Unless required by applicable law or agreed to in writing, software
17719 distributed under the License is distributed on an "AS IS" BASIS,
17720 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17721 See the License for the specific language governing permissions and
17722 limitations under the License.
17723  
17724 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17725 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17726 */
17727  
17728 #ifndef ZOPFLI_ZOPFLI_H_
17729 #define ZOPFLI_ZOPFLI_H_
17730  
17731 #include <stddef.h>
17732 #include <stdlib.h> /* for size_t */
17733  
17734 #ifdef __cplusplus
17735 extern "C" {
17736 #endif
17737  
17738 /*
17739 Options used throughout the program.
17740 */
17741 typedef struct ZopfliOptions {
17742 /* Whether to print output */
17743 int verbose;
17744  
17745 /* Whether to print more detailed output */
17746 int verbose_more;
17747  
17748 /*
17749 Maximum amount of times to rerun forward and backward pass to optimize LZ77
17750 compression cost. Good values: 10, 15 for small files, 5 for files over
17751 several MB in size or it will be too slow.
17752 */
17753 int numiterations;
17754  
17755 /*
17756 If true, splits the data in multiple deflate blocks with optimal choice
17757 for the block boundaries. Block splitting gives better compression. Default:
17758 true (1).
17759 */
17760 int blocksplitting;
17761  
17762 /*
17763 No longer used, left for compatibility.
17764 */
17765 int blocksplittinglast;
17766  
17767 /*
17768 Maximum amount of blocks to split into (0 for unlimited, but this can give
17769 extreme results that hurt compression on some files). Default value: 15.
17770 */
17771 int blocksplittingmax;
17772 } ZopfliOptions;
17773  
17774 /* Initializes options with default values. */
17775 void ZopfliInitOptions(ZopfliOptions* options);
17776  
17777 /* Output format */
17778 typedef enum {
17779 ZOPFLI_FORMAT_GZIP,
17780 ZOPFLI_FORMAT_ZLIB,
17781 ZOPFLI_FORMAT_DEFLATE
17782 } ZopfliFormat;
17783  
17784 /*
17785 Compresses according to the given output format and appends the result to the
17786 output.
17787  
17788 options: global program options
17789 output_type: the output format to use
17790 out: pointer to the dynamic output array to which the result is appended. Must
17791 be freed after use
17792 outsize: pointer to the dynamic output array size
17793 */
17794 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
17795 const unsigned char* in, size_t insize,
17796 unsigned char** out, size_t* outsize);
17797  
17798 #ifdef __cplusplus
17799 } // extern "C"
17800 #endif
17801  
17802 #endif /* ZOPFLI_ZOPFLI_H_ */
17803  
17804  
17805 /* deflate.h */
17806 /*
17807 Copyright 2011 Google Inc. All Rights Reserved.
17808  
17809 Licensed under the Apache License, Version 2.0 (the "License");
17810 you may not use this file except in compliance with the License.
17811 You may obtain a copy of the License at
17812  
17813 http://www.apache.org/licenses/LICENSE-2.0
17814  
17815 Unless required by applicable law or agreed to in writing, software
17816 distributed under the License is distributed on an "AS IS" BASIS,
17817 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17818 See the License for the specific language governing permissions and
17819 limitations under the License.
17820  
17821 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17822 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17823 */
17824  
17825 #ifndef ZOPFLI_DEFLATE_H_
17826 #define ZOPFLI_DEFLATE_H_
17827  
17828 /*
17829 Functions to compress according to the DEFLATE specification, using the
17830 "squeeze" LZ77 compression backend.
17831 */
17832  
17833 /* lz77.h */
17834 /*
17835 Copyright 2011 Google Inc. All Rights Reserved.
17836  
17837 Licensed under the Apache License, Version 2.0 (the "License");
17838 you may not use this file except in compliance with the License.
17839 You may obtain a copy of the License at
17840  
17841 http://www.apache.org/licenses/LICENSE-2.0
17842  
17843 Unless required by applicable law or agreed to in writing, software
17844 distributed under the License is distributed on an "AS IS" BASIS,
17845 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17846 See the License for the specific language governing permissions and
17847 limitations under the License.
17848  
17849 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17850 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17851 */
17852  
17853 /*
17854 Functions for basic LZ77 compression and utilities for the "squeeze" LZ77
17855 compression.
17856 */
17857  
17858 #ifndef ZOPFLI_LZ77_H_
17859 #define ZOPFLI_LZ77_H_
17860  
17861 #include <stdlib.h>
17862  
17863 /* cache.h */
17864 /*
17865 Copyright 2011 Google Inc. All Rights Reserved.
17866  
17867 Licensed under the Apache License, Version 2.0 (the "License");
17868 you may not use this file except in compliance with the License.
17869 You may obtain a copy of the License at
17870  
17871 http://www.apache.org/licenses/LICENSE-2.0
17872  
17873 Unless required by applicable law or agreed to in writing, software
17874 distributed under the License is distributed on an "AS IS" BASIS,
17875 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17876 See the License for the specific language governing permissions and
17877 limitations under the License.
17878  
17879 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17880 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17881 */
17882  
17883 /*
17884 The cache that speeds up ZopfliFindLongestMatch of lz77.c.
17885 */
17886  
17887 #ifndef ZOPFLI_CACHE_H_
17888 #define ZOPFLI_CACHE_H_
17889  
17890 /* util.h */
17891 /*
17892 Copyright 2011 Google Inc. All Rights Reserved.
17893  
17894 Licensed under the Apache License, Version 2.0 (the "License");
17895 you may not use this file except in compliance with the License.
17896 You may obtain a copy of the License at
17897  
17898 http://www.apache.org/licenses/LICENSE-2.0
17899  
17900 Unless required by applicable law or agreed to in writing, software
17901 distributed under the License is distributed on an "AS IS" BASIS,
17902 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17903 See the License for the specific language governing permissions and
17904 limitations under the License.
17905  
17906 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17907 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
17908 */
17909  
17910 /*
17911 Several utilities, including: #defines to try different compression results,
17912 basic deflate specification values and generic program options.
17913 */
17914  
17915 #ifndef ZOPFLI_UTIL_H_
17916 #define ZOPFLI_UTIL_H_
17917  
17918 #include <string.h>
17919 #include <stdlib.h>
17920  
17921 /* Minimum and maximum length that can be encoded in deflate. */
17922 #define ZOPFLI_MAX_MATCH 258
17923 #define ZOPFLI_MIN_MATCH 3
17924  
17925 /* Number of distinct literal/length and distance symbols in DEFLATE */
17926 #define ZOPFLI_NUM_LL 288
17927 #define ZOPFLI_NUM_D 32
17928  
17929 /*
17930 The window size for deflate. Must be a power of two. This should be 32768, the
17931 maximum possible by the deflate spec. Anything less hurts compression more than
17932 speed.
17933 */
17934 #define ZOPFLI_WINDOW_SIZE 32768
17935  
17936 /*
17937 The window mask used to wrap indices into the window. This is why the
17938 window size must be a power of two.
17939 */
17940 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
17941  
17942 /*
17943 A block structure of huge, non-smart, blocks to divide the input into, to allow
17944 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
17945 The whole compression algorithm, including the smarter block splitting, will
17946 be executed independently on each huge block.
17947 Dividing into huge blocks hurts compression, but not much relative to the size.
17948 Set it to 0 to disable master blocks.
17949 */
17950 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
17951  
17952 /*
17953 Used to initialize costs for example
17954 */
17955 #define ZOPFLI_LARGE_FLOAT 1e30
17956  
17957 /*
17958 For longest match cache. max 256. Uses huge amounts of memory but makes it
17959 faster. Uses this many times three bytes per single byte of the input data.
17960 This is so because longest match finding has to find the exact distance
17961 that belongs to each length for the best lz77 strategy.
17962 Good values: e.g. 5, 8.
17963 */
17964 #define ZOPFLI_CACHE_LENGTH 8
17965  
17966 /*
17967 limit the max hash chain hits for this hash value. This has an effect only
17968 on files where the hash value is the same very often. On these files, this
17969 gives worse compression (the value should ideally be 32768, which is the
17970 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
17971 faster on some specific files.
17972 Good value: e.g. 8192.
17973 */
17974 #define ZOPFLI_MAX_CHAIN_HITS 8192
17975  
17976 /*
17977 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
17978 consumes a lot of memory but speeds it up. No effect on compression size.
17979 */
17980 #define ZOPFLI_LONGEST_MATCH_CACHE
17981  
17982 /*
17983 Enable to remember amount of successive identical bytes in the hash chain for
17984 finding longest match
17985 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
17986 This has no effect on the compression result, and enabling it increases speed.
17987 */
17988 #define ZOPFLI_HASH_SAME
17989  
17990 /*
17991 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
17992 best length so far is long enough. This is way faster for files with lots of
17993 identical bytes, on which the compressor is otherwise too slow. Regular files
17994 are unaffected or maybe a tiny bit slower.
17995 This has no effect on the compression result, only on speed.
17996 */
17997 #define ZOPFLI_HASH_SAME_HASH
17998  
17999 /*
18000 Enable this, to avoid slowness for files which are a repetition of the same
18001 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
18002 the compression result.
18003 */
18004 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
18005  
18006 /*
18007 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
18008 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
18009 varies from file to file.
18010 */
18011 #define ZOPFLI_LAZY_MATCHING
18012  
18013 /*
18014 Appends value to dynamically allocated memory, doubling its allocation size
18015 whenever needed.
18016  
18017 value: the value to append, type T
18018 data: pointer to the dynamic array to append to, type T**
18019 size: pointer to the size of the array to append to, type size_t*. This is the
18020 size that you consider the array to be, not the internal allocation size.
18021 Precondition: allocated size of data is at least a power of two greater than or
18022 equal than *size.
18023 */
18024 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
18025 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
18026 if (!((*size) & ((*size) - 1))) {\
18027 /*double alloc size if it's a power of two*/\
18028 void** data_void = reinterpret_cast<void**>(data);\
18029 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
18030 : realloc((*data), (*size) * 2 * sizeof(**data));\
18031 }\
18032 (*data)[(*size)] = (value);\
18033 (*size)++;\
18034 }
18035 #else /* C gives problems with strict-aliasing rules for (void**) cast */
18036 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
18037 if (!((*size) & ((*size) - 1))) {\
18038 /*double alloc size if it's a power of two*/\
18039 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
18040 : realloc((*data), (*size) * 2 * sizeof(**data));\
18041 }\
18042 (*data)[(*size)] = (value);\
18043 (*size)++;\
18044 }
18045 #endif
18046  
18047  
18048 #endif /* ZOPFLI_UTIL_H_ */
18049  
18050  
18051 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
18052  
18053 /*
18054 Cache used by ZopfliFindLongestMatch to remember previously found length/dist
18055 values.
18056 This is needed because the squeeze runs will ask these values multiple times for
18057 the same position.
18058 Uses large amounts of memory, since it has to remember the distance belonging
18059 to every possible shorter-than-the-best length (the so called "sublen" array).
18060 */
18061 typedef struct ZopfliLongestMatchCache {
18062 unsigned short* length;
18063 unsigned short* dist;
18064 unsigned char* sublen;
18065 } ZopfliLongestMatchCache;
18066  
18067 /* Initializes the ZopfliLongestMatchCache. */
18068 void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc);
18069  
18070 /* Frees up the memory of the ZopfliLongestMatchCache. */
18071 void ZopfliCleanCache(ZopfliLongestMatchCache* lmc);
18072  
18073 /* Stores sublen array in the cache. */
18074 void ZopfliSublenToCache(const unsigned short* sublen,
18075 size_t pos, size_t length,
18076 ZopfliLongestMatchCache* lmc);
18077  
18078 /* Extracts sublen array from the cache. */
18079 void ZopfliCacheToSublen(const ZopfliLongestMatchCache* lmc,
18080 size_t pos, size_t length,
18081 unsigned short* sublen);
18082 /* Returns the length up to which could be stored in the cache. */
18083 unsigned ZopfliMaxCachedSublen(const ZopfliLongestMatchCache* lmc,
18084 size_t pos, size_t length);
18085  
18086 #endif /* ZOPFLI_LONGEST_MATCH_CACHE */
18087  
18088 #endif /* ZOPFLI_CACHE_H_ */
18089  
18090 /* hash.h */
18091 /*
18092 Copyright 2011 Google Inc. All Rights Reserved.
18093  
18094 Licensed under the Apache License, Version 2.0 (the "License");
18095 you may not use this file except in compliance with the License.
18096 You may obtain a copy of the License at
18097  
18098 http://www.apache.org/licenses/LICENSE-2.0
18099  
18100 Unless required by applicable law or agreed to in writing, software
18101 distributed under the License is distributed on an "AS IS" BASIS,
18102 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18103 See the License for the specific language governing permissions and
18104 limitations under the License.
18105  
18106 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18107 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18108 */
18109  
18110 /*
18111 The hash for ZopfliFindLongestMatch of lz77.c.
18112 */
18113  
18114 #ifndef ZOPFLI_HASH_H_
18115 #define ZOPFLI_HASH_H_
18116  
18117 /* util.h */
18118 /*
18119 Copyright 2011 Google Inc. All Rights Reserved.
18120  
18121 Licensed under the Apache License, Version 2.0 (the "License");
18122 you may not use this file except in compliance with the License.
18123 You may obtain a copy of the License at
18124  
18125 http://www.apache.org/licenses/LICENSE-2.0
18126  
18127 Unless required by applicable law or agreed to in writing, software
18128 distributed under the License is distributed on an "AS IS" BASIS,
18129 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18130 See the License for the specific language governing permissions and
18131 limitations under the License.
18132  
18133 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18134 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18135 */
18136  
18137 /*
18138 Several utilities, including: #defines to try different compression results,
18139 basic deflate specification values and generic program options.
18140 */
18141  
18142 #ifndef ZOPFLI_UTIL_H_
18143 #define ZOPFLI_UTIL_H_
18144  
18145 #include <string.h>
18146 #include <stdlib.h>
18147  
18148 /* Minimum and maximum length that can be encoded in deflate. */
18149 #define ZOPFLI_MAX_MATCH 258
18150 #define ZOPFLI_MIN_MATCH 3
18151  
18152 /* Number of distinct literal/length and distance symbols in DEFLATE */
18153 #define ZOPFLI_NUM_LL 288
18154 #define ZOPFLI_NUM_D 32
18155  
18156 /*
18157 The window size for deflate. Must be a power of two. This should be 32768, the
18158 maximum possible by the deflate spec. Anything less hurts compression more than
18159 speed.
18160 */
18161 #define ZOPFLI_WINDOW_SIZE 32768
18162  
18163 /*
18164 The window mask used to wrap indices into the window. This is why the
18165 window size must be a power of two.
18166 */
18167 #define ZOPFLI_WINDOW_MASK (ZOPFLI_WINDOW_SIZE - 1)
18168  
18169 /*
18170 A block structure of huge, non-smart, blocks to divide the input into, to allow
18171 operating on huge files without exceeding memory, such as the 1GB wiki9 corpus.
18172 The whole compression algorithm, including the smarter block splitting, will
18173 be executed independently on each huge block.
18174 Dividing into huge blocks hurts compression, but not much relative to the size.
18175 Set it to 0 to disable master blocks.
18176 */
18177 #define ZOPFLI_MASTER_BLOCK_SIZE 1000000
18178  
18179 /*
18180 Used to initialize costs for example
18181 */
18182 #define ZOPFLI_LARGE_FLOAT 1e30
18183  
18184 /*
18185 For longest match cache. max 256. Uses huge amounts of memory but makes it
18186 faster. Uses this many times three bytes per single byte of the input data.
18187 This is so because longest match finding has to find the exact distance
18188 that belongs to each length for the best lz77 strategy.
18189 Good values: e.g. 5, 8.
18190 */
18191 #define ZOPFLI_CACHE_LENGTH 8
18192  
18193 /*
18194 limit the max hash chain hits for this hash value. This has an effect only
18195 on files where the hash value is the same very often. On these files, this
18196 gives worse compression (the value should ideally be 32768, which is the
18197 ZOPFLI_WINDOW_SIZE, while zlib uses 4096 even for best level), but makes it
18198 faster on some specific files.
18199 Good value: e.g. 8192.
18200 */
18201 #define ZOPFLI_MAX_CHAIN_HITS 8192
18202  
18203 /*
18204 Whether to use the longest match cache for ZopfliFindLongestMatch. This cache
18205 consumes a lot of memory but speeds it up. No effect on compression size.
18206 */
18207 #define ZOPFLI_LONGEST_MATCH_CACHE
18208  
18209 /*
18210 Enable to remember amount of successive identical bytes in the hash chain for
18211 finding longest match
18212 required for ZOPFLI_HASH_SAME_HASH and ZOPFLI_SHORTCUT_LONG_REPETITIONS
18213 This has no effect on the compression result, and enabling it increases speed.
18214 */
18215 #define ZOPFLI_HASH_SAME
18216  
18217 /*
18218 Switch to a faster hash based on the info from ZOPFLI_HASH_SAME once the
18219 best length so far is long enough. This is way faster for files with lots of
18220 identical bytes, on which the compressor is otherwise too slow. Regular files
18221 are unaffected or maybe a tiny bit slower.
18222 This has no effect on the compression result, only on speed.
18223 */
18224 #define ZOPFLI_HASH_SAME_HASH
18225  
18226 /*
18227 Enable this, to avoid slowness for files which are a repetition of the same
18228 character more than a multiple of ZOPFLI_MAX_MATCH times. This should not affect
18229 the compression result.
18230 */
18231 #define ZOPFLI_SHORTCUT_LONG_REPETITIONS
18232  
18233 /*
18234 Whether to use lazy matching in the greedy LZ77 implementation. This gives a
18235 better result of ZopfliLZ77Greedy, but the effect this has on the optimal LZ77
18236 varies from file to file.
18237 */
18238 #define ZOPFLI_LAZY_MATCHING
18239  
18240 /*
18241 Appends value to dynamically allocated memory, doubling its allocation size
18242 whenever needed.
18243  
18244 value: the value to append, type T
18245 data: pointer to the dynamic array to append to, type T**
18246 size: pointer to the size of the array to append to, type size_t*. This is the
18247 size that you consider the array to be, not the internal allocation size.
18248 Precondition: allocated size of data is at least a power of two greater than or
18249 equal than *size.
18250 */
18251 #ifdef __cplusplus /* C++ cannot assign void* from malloc to *data */
18252 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
18253 if (!((*size) & ((*size) - 1))) {\
18254 /*double alloc size if it's a power of two*/\
18255 void** data_void = reinterpret_cast<void**>(data);\
18256 *data_void = (*size) == 0 ? malloc(sizeof(**data))\
18257 : realloc((*data), (*size) * 2 * sizeof(**data));\
18258 }\
18259 (*data)[(*size)] = (value);\
18260 (*size)++;\
18261 }
18262 #else /* C gives problems with strict-aliasing rules for (void**) cast */
18263 #define ZOPFLI_APPEND_DATA(/* T */ value, /* T** */ data, /* size_t* */ size) {\
18264 if (!((*size) & ((*size) - 1))) {\
18265 /*double alloc size if it's a power of two*/\
18266 (*data) = (*size) == 0 ? malloc(sizeof(**data))\
18267 : realloc((*data), (*size) * 2 * sizeof(**data));\
18268 }\
18269 (*data)[(*size)] = (value);\
18270 (*size)++;\
18271 }
18272 #endif
18273  
18274  
18275 #endif /* ZOPFLI_UTIL_H_ */
18276  
18277  
18278 typedef struct ZopfliHash {
18279 int* head; /* Hash value to index of its most recent occurrence. */
18280 unsigned short* prev; /* Index to index of prev. occurrence of same hash. */
18281 int* hashval; /* Index to hash value at this index. */
18282 int val; /* Current hash value. */
18283  
18284 #ifdef ZOPFLI_HASH_SAME_HASH
18285 /* Fields with similar purpose as the above hash, but for the second hash with
18286 a value that is calculated differently. */
18287 int* head2; /* Hash value to index of its most recent occurrence. */
18288 unsigned short* prev2; /* Index to index of prev. occurrence of same hash. */
18289 int* hashval2; /* Index to hash value at this index. */
18290 int val2; /* Current hash value. */
18291 #endif
18292  
18293 #ifdef ZOPFLI_HASH_SAME
18294 unsigned short* same; /* Amount of repetitions of same byte after this .*/
18295 #endif
18296 } ZopfliHash;
18297  
18298 /* Allocates ZopfliHash memory. */
18299 void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
18300  
18301 /* Resets all fields of ZopfliHash. */
18302 void ZopfliResetHash(size_t window_size, ZopfliHash* h);
18303  
18304 /* Frees ZopfliHash memory. */
18305 void ZopfliCleanHash(ZopfliHash* h);
18306  
18307 /*
18308 Updates the hash values based on the current position in the array. All calls
18309 to this must be made for consecutive bytes.
18310 */
18311 void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
18312 ZopfliHash* h);
18313  
18314 /*
18315 Prepopulates hash:
18316 Fills in the initial values in the hash, before ZopfliUpdateHash can be used
18317 correctly.
18318 */
18319 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
18320 ZopfliHash* h);
18321  
18322 #endif /* ZOPFLI_HASH_H_ */
18323  
18324 /* zopfli.h */
18325 /*
18326 Copyright 2011 Google Inc. All Rights Reserved.
18327  
18328 Licensed under the Apache License, Version 2.0 (the "License");
18329 you may not use this file except in compliance with the License.
18330 You may obtain a copy of the License at
18331  
18332 http://www.apache.org/licenses/LICENSE-2.0
18333  
18334 Unless required by applicable law or agreed to in writing, software
18335 distributed under the License is distributed on an "AS IS" BASIS,
18336 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18337 See the License for the specific language governing permissions and
18338 limitations under the License.
18339  
18340 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18341 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18342 */
18343  
18344 #ifndef ZOPFLI_ZOPFLI_H_
18345 #define ZOPFLI_ZOPFLI_H_
18346  
18347 #include <stddef.h>
18348 #include <stdlib.h> /* for size_t */
18349  
18350 #ifdef __cplusplus
18351 extern "C" {
18352 #endif
18353  
18354 /*
18355 Options used throughout the program.
18356 */
18357 typedef struct ZopfliOptions {
18358 /* Whether to print output */
18359 int verbose;
18360  
18361 /* Whether to print more detailed output */
18362 int verbose_more;
18363  
18364 /*
18365 Maximum amount of times to rerun forward and backward pass to optimize LZ77
18366 compression cost. Good values: 10, 15 for small files, 5 for files over
18367 several MB in size or it will be too slow.
18368 */
18369 int numiterations;
18370  
18371 /*
18372 If true, splits the data in multiple deflate blocks with optimal choice
18373 for the block boundaries. Block splitting gives better compression. Default:
18374 true (1).
18375 */
18376 int blocksplitting;
18377  
18378 /*
18379 No longer used, left for compatibility.
18380 */
18381 int blocksplittinglast;
18382  
18383 /*
18384 Maximum amount of blocks to split into (0 for unlimited, but this can give
18385 extreme results that hurt compression on some files). Default value: 15.
18386 */
18387 int blocksplittingmax;
18388 } ZopfliOptions;
18389  
18390 /* Initializes options with default values. */
18391 void ZopfliInitOptions(ZopfliOptions* options);
18392  
18393 /* Output format */
18394 typedef enum {
18395 ZOPFLI_FORMAT_GZIP,
18396 ZOPFLI_FORMAT_ZLIB,
18397 ZOPFLI_FORMAT_DEFLATE
18398 } ZopfliFormat;
18399  
18400 /*
18401 Compresses according to the given output format and appends the result to the
18402 output.
18403  
18404 options: global program options
18405 output_type: the output format to use
18406 out: pointer to the dynamic output array to which the result is appended. Must
18407 be freed after use
18408 outsize: pointer to the dynamic output array size
18409 */
18410 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
18411 const unsigned char* in, size_t insize,
18412 unsigned char** out, size_t* outsize);
18413  
18414 #ifdef __cplusplus
18415 } // extern "C"
18416 #endif
18417  
18418 #endif /* ZOPFLI_ZOPFLI_H_ */
18419  
18420  
18421 /*
18422 Stores lit/length and dist pairs for LZ77.
18423 Parameter litlens: Contains the literal symbols or length values.
18424 Parameter dists: Contains the distances. A value is 0 to indicate that there is
18425 no dist and the corresponding litlens value is a literal instead of a length.
18426 Parameter size: The size of both the litlens and dists arrays.
18427 The memory can best be managed by using ZopfliInitLZ77Store to initialize it,
18428 ZopfliCleanLZ77Store to destroy it, and ZopfliStoreLitLenDist to append values.
18429  
18430 */
18431 typedef struct ZopfliLZ77Store {
18432 unsigned short* litlens; /* Lit or len. */
18433 unsigned short* dists; /* If 0: indicates literal in corresponding litlens,
18434 if > 0: length in corresponding litlens, this is the distance. */
18435 size_t size;
18436  
18437 const unsigned char* data; /* original data */
18438 size_t* pos; /* position in data where this LZ77 command begins */
18439  
18440 unsigned short* ll_symbol;
18441 unsigned short* d_symbol;
18442  
18443 /* Cumulative histograms wrapping around per chunk. Each chunk has the amount
18444 of distinct symbols as length, so using 1 value per LZ77 symbol, we have a
18445 precise histogram at every N symbols, and the rest can be calculated by
18446 looping through the actual symbols of this chunk. */
18447 size_t* ll_counts;
18448 size_t* d_counts;
18449 } ZopfliLZ77Store;
18450  
18451 void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store);
18452 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store);
18453 void ZopfliCopyLZ77Store(const ZopfliLZ77Store* source, ZopfliLZ77Store* dest);
18454 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
18455 size_t pos, ZopfliLZ77Store* store);
18456 void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
18457 ZopfliLZ77Store* target);
18458 /* Gets the amount of raw bytes that this range of LZ77 symbols spans. */
18459 size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
18460 size_t lstart, size_t lend);
18461 /* Gets the histogram of lit/len and dist symbols in the given range, using the
18462 cumulative histograms, so faster than adding one by one for large range. Does
18463 not add the one end symbol of value 256. */
18464 void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
18465 size_t lstart, size_t lend,
18466 size_t* ll_counts, size_t* d_counts);
18467  
18468 /*
18469 Some state information for compressing a block.
18470 This is currently a bit under-used (with mainly only the longest match cache),
18471 but is kept for easy future expansion.
18472 */
18473 typedef struct ZopfliBlockState {
18474 const ZopfliOptions* options;
18475  
18476 #ifdef ZOPFLI_LONGEST_MATCH_CACHE
18477 /* Cache for length/distance pairs found so far. */
18478 ZopfliLongestMatchCache* lmc;
18479 #endif
18480  
18481 /* The start (inclusive) and end (not inclusive) of the current block. */
18482 size_t blockstart;
18483 size_t blockend;
18484 } ZopfliBlockState;
18485  
18486 void ZopfliInitBlockState(const ZopfliOptions* options,
18487 size_t blockstart, size_t blockend, int add_lmc,
18488 ZopfliBlockState* s);
18489 void ZopfliCleanBlockState(ZopfliBlockState* s);
18490  
18491 /*
18492 Finds the longest match (length and corresponding distance) for LZ77
18493 compression.
18494 Even when not using "sublen", it can be more efficient to provide an array,
18495 because only then the caching is used.
18496 array: the data
18497 pos: position in the data to find the match for
18498 size: size of the data
18499 limit: limit length to maximum this value (default should be 258). This allows
18500 finding a shorter dist for that length (= less extra bits). Must be
18501 in the range [ZOPFLI_MIN_MATCH, ZOPFLI_MAX_MATCH].
18502 sublen: output array of 259 elements, or null. Has, for each length, the
18503 smallest distance required to reach this length. Only 256 of its 259 values
18504 are used, the first 3 are ignored (the shortest length is 3. It is purely
18505 for convenience that the array is made 3 longer).
18506 */
18507 void ZopfliFindLongestMatch(
18508 ZopfliBlockState *s, const ZopfliHash* h, const unsigned char* array,
18509 size_t pos, size_t size, size_t limit,
18510 unsigned short* sublen, unsigned short* distance, unsigned short* length);
18511  
18512 /*
18513 Verifies if length and dist are indeed valid, only used for assertion.
18514 */
18515 void ZopfliVerifyLenDist(const unsigned char* data, size_t datasize, size_t pos,
18516 unsigned short dist, unsigned short length);
18517  
18518 /*
18519 Does LZ77 using an algorithm similar to gzip, with lazy matching, rather than
18520 with the slow but better "squeeze" implementation.
18521 The result is placed in the ZopfliLZ77Store.
18522 If instart is larger than 0, it uses values before instart as starting
18523 dictionary.
18524 */
18525 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
18526 size_t instart, size_t inend,
18527 ZopfliLZ77Store* store, ZopfliHash* h);
18528  
18529 #endif /* ZOPFLI_LZ77_H_ */
18530  
18531 /* zopfli.h */
18532 /*
18533 Copyright 2011 Google Inc. All Rights Reserved.
18534  
18535 Licensed under the Apache License, Version 2.0 (the "License");
18536 you may not use this file except in compliance with the License.
18537 You may obtain a copy of the License at
18538  
18539 http://www.apache.org/licenses/LICENSE-2.0
18540  
18541 Unless required by applicable law or agreed to in writing, software
18542 distributed under the License is distributed on an "AS IS" BASIS,
18543 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18544 See the License for the specific language governing permissions and
18545 limitations under the License.
18546  
18547 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18548 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18549 */
18550  
18551 #ifndef ZOPFLI_ZOPFLI_H_
18552 #define ZOPFLI_ZOPFLI_H_
18553  
18554 #include <stddef.h>
18555 #include <stdlib.h> /* for size_t */
18556  
18557 #ifdef __cplusplus
18558 extern "C" {
18559 #endif
18560  
18561 /*
18562 Options used throughout the program.
18563 */
18564 typedef struct ZopfliOptions {
18565 /* Whether to print output */
18566 int verbose;
18567  
18568 /* Whether to print more detailed output */
18569 int verbose_more;
18570  
18571 /*
18572 Maximum amount of times to rerun forward and backward pass to optimize LZ77
18573 compression cost. Good values: 10, 15 for small files, 5 for files over
18574 several MB in size or it will be too slow.
18575 */
18576 int numiterations;
18577  
18578 /*
18579 If true, splits the data in multiple deflate blocks with optimal choice
18580 for the block boundaries. Block splitting gives better compression. Default:
18581 true (1).
18582 */
18583 int blocksplitting;
18584  
18585 /*
18586 No longer used, left for compatibility.
18587 */
18588 int blocksplittinglast;
18589  
18590 /*
18591 Maximum amount of blocks to split into (0 for unlimited, but this can give
18592 extreme results that hurt compression on some files). Default value: 15.
18593 */
18594 int blocksplittingmax;
18595 } ZopfliOptions;
18596  
18597 /* Initializes options with default values. */
18598 void ZopfliInitOptions(ZopfliOptions* options);
18599  
18600 /* Output format */
18601 typedef enum {
18602 ZOPFLI_FORMAT_GZIP,
18603 ZOPFLI_FORMAT_ZLIB,
18604 ZOPFLI_FORMAT_DEFLATE
18605 } ZopfliFormat;
18606  
18607 /*
18608 Compresses according to the given output format and appends the result to the
18609 output.
18610  
18611 options: global program options
18612 output_type: the output format to use
18613 out: pointer to the dynamic output array to which the result is appended. Must
18614 be freed after use
18615 outsize: pointer to the dynamic output array size
18616 */
18617 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
18618 const unsigned char* in, size_t insize,
18619 unsigned char** out, size_t* outsize);
18620  
18621 #ifdef __cplusplus
18622 } // extern "C"
18623 #endif
18624  
18625 #endif /* ZOPFLI_ZOPFLI_H_ */
18626  
18627  
18628 #ifdef __cplusplus
18629 extern "C" {
18630 #endif
18631  
18632 /*
18633 Compresses according to the deflate specification and append the compressed
18634 result to the output.
18635 This function will usually output multiple deflate blocks. If final is 1, then
18636 the final bit will be set on the last block.
18637  
18638 options: global program options
18639 btype: the deflate block type. Use 2 for best compression.
18640 -0: non compressed blocks (00)
18641 -1: blocks with fixed tree (01)
18642 -2: blocks with dynamic tree (10)
18643 final: whether this is the last section of the input, sets the final bit to the
18644 last deflate block.
18645 in: the input bytes
18646 insize: number of input bytes
18647 bp: bit pointer for the output array. This must initially be 0, and for
18648 consecutive calls must be reused (it can have values from 0-7). This is
18649 because deflate appends blocks as bit-based data, rather than on byte
18650 boundaries.
18651 out: pointer to the dynamic output array to which the result is appended. Must
18652 be freed after use.
18653 outsize: pointer to the dynamic output array size.
18654 */
18655 void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
18656 const unsigned char* in, size_t insize,
18657 unsigned char* bp, unsigned char** out, size_t* outsize);
18658  
18659 /*
18660 Like ZopfliDeflate, but allows to specify start and end byte with instart and
18661 inend. Only that part is compressed, but earlier bytes are still used for the
18662 back window.
18663 */
18664 void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
18665 const unsigned char* in, size_t instart, size_t inend,
18666 unsigned char* bp, unsigned char** out,
18667 size_t* outsize);
18668  
18669 /*
18670 Calculates block size in bits.
18671 litlens: lz77 lit/lengths
18672 dists: ll77 distances
18673 lstart: start of block
18674 lend: end of block (not inclusive)
18675 */
18676 double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
18677 size_t lstart, size_t lend, int btype);
18678  
18679 /*
18680 Calculates block size in bits, automatically using the best btype.
18681 */
18682 double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
18683 size_t lstart, size_t lend);
18684  
18685 #ifdef __cplusplus
18686 } // extern "C"
18687 #endif
18688  
18689 #endif /* ZOPFLI_DEFLATE_H_ */
18690  
18691 /* gzip_container.h */
18692 /*
18693 Copyright 2013 Google Inc. All Rights Reserved.
18694  
18695 Licensed under the Apache License, Version 2.0 (the "License");
18696 you may not use this file except in compliance with the License.
18697 You may obtain a copy of the License at
18698  
18699 http://www.apache.org/licenses/LICENSE-2.0
18700  
18701 Unless required by applicable law or agreed to in writing, software
18702 distributed under the License is distributed on an "AS IS" BASIS,
18703 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18704 See the License for the specific language governing permissions and
18705 limitations under the License.
18706  
18707 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18708 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18709 */
18710  
18711 #ifndef ZOPFLI_GZIP_H_
18712 #define ZOPFLI_GZIP_H_
18713  
18714 /*
18715 Functions to compress according to the Gzip specification.
18716 */
18717  
18718 /* zopfli.h */
18719 /*
18720 Copyright 2011 Google Inc. All Rights Reserved.
18721  
18722 Licensed under the Apache License, Version 2.0 (the "License");
18723 you may not use this file except in compliance with the License.
18724 You may obtain a copy of the License at
18725  
18726 http://www.apache.org/licenses/LICENSE-2.0
18727  
18728 Unless required by applicable law or agreed to in writing, software
18729 distributed under the License is distributed on an "AS IS" BASIS,
18730 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18731 See the License for the specific language governing permissions and
18732 limitations under the License.
18733  
18734 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18735 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18736 */
18737  
18738 #ifndef ZOPFLI_ZOPFLI_H_
18739 #define ZOPFLI_ZOPFLI_H_
18740  
18741 #include <stddef.h>
18742 #include <stdlib.h> /* for size_t */
18743  
18744 #ifdef __cplusplus
18745 extern "C" {
18746 #endif
18747  
18748 /*
18749 Options used throughout the program.
18750 */
18751 typedef struct ZopfliOptions {
18752 /* Whether to print output */
18753 int verbose;
18754  
18755 /* Whether to print more detailed output */
18756 int verbose_more;
18757  
18758 /*
18759 Maximum amount of times to rerun forward and backward pass to optimize LZ77
18760 compression cost. Good values: 10, 15 for small files, 5 for files over
18761 several MB in size or it will be too slow.
18762 */
18763 int numiterations;
18764  
18765 /*
18766 If true, splits the data in multiple deflate blocks with optimal choice
18767 for the block boundaries. Block splitting gives better compression. Default:
18768 true (1).
18769 */
18770 int blocksplitting;
18771  
18772 /*
18773 No longer used, left for compatibility.
18774 */
18775 int blocksplittinglast;
18776  
18777 /*
18778 Maximum amount of blocks to split into (0 for unlimited, but this can give
18779 extreme results that hurt compression on some files). Default value: 15.
18780 */
18781 int blocksplittingmax;
18782 } ZopfliOptions;
18783  
18784 /* Initializes options with default values. */
18785 void ZopfliInitOptions(ZopfliOptions* options);
18786  
18787 /* Output format */
18788 typedef enum {
18789 ZOPFLI_FORMAT_GZIP,
18790 ZOPFLI_FORMAT_ZLIB,
18791 ZOPFLI_FORMAT_DEFLATE
18792 } ZopfliFormat;
18793  
18794 /*
18795 Compresses according to the given output format and appends the result to the
18796 output.
18797  
18798 options: global program options
18799 output_type: the output format to use
18800 out: pointer to the dynamic output array to which the result is appended. Must
18801 be freed after use
18802 outsize: pointer to the dynamic output array size
18803 */
18804 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
18805 const unsigned char* in, size_t insize,
18806 unsigned char** out, size_t* outsize);
18807  
18808 #ifdef __cplusplus
18809 } // extern "C"
18810 #endif
18811  
18812 #endif /* ZOPFLI_ZOPFLI_H_ */
18813  
18814  
18815 #ifdef __cplusplus
18816 extern "C" {
18817 #endif
18818  
18819 /*
18820 Compresses according to the gzip specification and append the compressed
18821 result to the output.
18822  
18823 options: global program options
18824 out: pointer to the dynamic output array to which the result is appended. Must
18825 be freed after use.
18826 outsize: pointer to the dynamic output array size.
18827 */
18828 void ZopfliGzipCompress(const ZopfliOptions* options,
18829 const unsigned char* in, size_t insize,
18830 unsigned char** out, size_t* outsize);
18831  
18832 #ifdef __cplusplus
18833 } // extern "C"
18834 #endif
18835  
18836 #endif /* ZOPFLI_GZIP_H_ */
18837  
18838 /* zlib_container.h */
18839 /*
18840 Copyright 2013 Google Inc. All Rights Reserved.
18841  
18842 Licensed under the Apache License, Version 2.0 (the "License");
18843 you may not use this file except in compliance with the License.
18844 You may obtain a copy of the License at
18845  
18846 http://www.apache.org/licenses/LICENSE-2.0
18847  
18848 Unless required by applicable law or agreed to in writing, software
18849 distributed under the License is distributed on an "AS IS" BASIS,
18850 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18851 See the License for the specific language governing permissions and
18852 limitations under the License.
18853  
18854 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18855 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18856 */
18857  
18858 #ifndef ZOPFLI_ZLIB_H_
18859 #define ZOPFLI_ZLIB_H_
18860  
18861 /*
18862 Functions to compress according to the Zlib specification.
18863 */
18864  
18865 /* zopfli.h */
18866 /*
18867 Copyright 2011 Google Inc. All Rights Reserved.
18868  
18869 Licensed under the Apache License, Version 2.0 (the "License");
18870 you may not use this file except in compliance with the License.
18871 You may obtain a copy of the License at
18872  
18873 http://www.apache.org/licenses/LICENSE-2.0
18874  
18875 Unless required by applicable law or agreed to in writing, software
18876 distributed under the License is distributed on an "AS IS" BASIS,
18877 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18878 See the License for the specific language governing permissions and
18879 limitations under the License.
18880  
18881 Author: lode.vandevenne@gmail.com (Lode Vandevenne)
18882 Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18883 */
18884  
18885 #ifndef ZOPFLI_ZOPFLI_H_
18886 #define ZOPFLI_ZOPFLI_H_
18887  
18888 #include <stddef.h>
18889 #include <stdlib.h> /* for size_t */
18890  
18891 #ifdef __cplusplus
18892 extern "C" {
18893 #endif
18894  
18895 /*
18896 Options used throughout the program.
18897 */
18898 typedef struct ZopfliOptions {
18899 /* Whether to print output */
18900 int verbose;
18901  
18902 /* Whether to print more detailed output */
18903 int verbose_more;
18904  
18905 /*
18906 Maximum amount of times to rerun forward and backward pass to optimize LZ77
18907 compression cost. Good values: 10, 15 for small files, 5 for files over
18908 several MB in size or it will be too slow.
18909 */
18910 int numiterations;
18911  
18912 /*
18913 If true, splits the data in multiple deflate blocks with optimal choice
18914 for the block boundaries. Block splitting gives better compression. Default:
18915 true (1).
18916 */
18917 int blocksplitting;
18918  
18919 /*
18920 No longer used, left for compatibility.
18921 */
18922 int blocksplittinglast;
18923  
18924 /*
18925 Maximum amount of blocks to split into (0 for unlimited, but this can give
18926 extreme results that hurt compression on some files). Default value: 15.
18927 */
18928 int blocksplittingmax;
18929 } ZopfliOptions;
18930  
18931 /* Initializes options with default values. */
18932 void ZopfliInitOptions(ZopfliOptions* options);
18933  
18934 /* Output format */
18935 typedef enum {
18936 ZOPFLI_FORMAT_GZIP,
18937 ZOPFLI_FORMAT_ZLIB,
18938 ZOPFLI_FORMAT_DEFLATE
18939 } ZopfliFormat;
18940  
18941 /*
18942 Compresses according to the given output format and appends the result to the
18943 output.
18944  
18945 options: global program options
18946 output_type: the output format to use
18947 out: pointer to the dynamic output array to which the result is appended. Must
18948 be freed after use
18949 outsize: pointer to the dynamic output array size
18950 */
18951 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
18952 const unsigned char* in, size_t insize,
18953 unsigned char** out, size_t* outsize);
18954  
18955 #ifdef __cplusplus
18956 } // extern "C"
18957 #endif
18958  
18959 #endif /* ZOPFLI_ZOPFLI_H_ */
18960  
18961  
18962 #ifdef __cplusplus
18963 extern "C" {
18964 #endif
18965  
18966 /*
18967 Compresses according to the zlib specification and append the compressed
18968 result to the output.
18969  
18970 options: global program options
18971 out: pointer to the dynamic output array to which the result is appended. Must
18972 be freed after use.
18973 outsize: pointer to the dynamic output array size.
18974 */
18975 void ZopfliZlibCompress(const ZopfliOptions* options,
18976 const unsigned char* in, size_t insize,
18977 unsigned char** out, size_t* outsize);
18978  
18979 #ifdef __cplusplus
18980 } // extern "C"
18981 #endif
18982  
18983 #endif /* ZOPFLI_ZLIB_H_ */
18984  
18985  
18986 #include <assert.h>
18987  
18988 3 void ZopfliCompress(const ZopfliOptions* options, ZopfliFormat output_type,
18989 const unsigned char* in, size_t insize,
18990 unsigned char** out, size_t* outsize) {
18991 3 ▶ if (output_type == ZOPFLI_FORMAT_GZIP) {
Branch: 1 branch
BranchTrueFalseTotal
if (output_type == ZOPFLI_FORMAT_GZIP) { 3 0 3
18992 3 ZopfliGzipCompress(options, in, insize, out, outsize);
18993 0 ▶ } else if (output_type == ZOPFLI_FORMAT_ZLIB) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (output_type == ZOPFLI_FORMAT_ZLIB) { 0 0 0
18994 0 ZopfliZlibCompress(options, in, insize, out, outsize);
18995 0 ▶ } else if (output_type == ZOPFLI_FORMAT_DEFLATE) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (output_type == ZOPFLI_FORMAT_DEFLATE) { 0 0 0
18996 0 unsigned char bp = 0;
18997 0 ZopfliDeflate(options, 2 /* Dynamic block */, 1,
18998 in, insize, &bp, out, outsize);
18999 } else {
19000 assert(0);
19001 }
19002 3 }
« gzip-zopfli-perl.c ↑ Summary Zopfli.xs »