libdeflate-one.c

filter: Statementstmt 54.1% 1880 / 3474 Branchbran 36.3% 872 / 2402 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 46.8% 2752 / 5876 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« blib/lib/Gzip/Libdeflate.pm ↑ Summary gzip-libdeflate-perl.c »
1 /*
2 Copyright 2016 Eric Biggers
3 Copyright 2024 Google LLC
4  
5 Permission is hereby granted, free of charge, to any person
6 obtaining a copy of this software and associated documentation files
7 (the "Software"), to deal in the Software without restriction,
8 including without limitation the rights to use, copy, modify, merge,
9 publish, distribute, sublicense, and/or sell copies of the Software,
10 and to permit persons to whom the Software is furnished to do so,
11 subject to the following conditions:
12  
13 The above copyright notice and this permission notice shall be
14 included in all copies or substantial portions of the Software.
15  
16 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
20 BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
21 ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 SOFTWARE.
24  
25 */
26 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/adler32.c */
27  
28  
29 /* #include "lib_common.h" */
30  
31  
32 #ifndef LIB_LIB_COMMON_H
33 #define LIB_LIB_COMMON_H
34  
35 #ifdef LIBDEFLATE_H
36
37 # error "lib_common.h must always be included before libdeflate.h"
38 #endif
39  
40 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
41 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
42 #elif defined(__GNUC__)
43 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
44 #else
45 # define LIBDEFLATE_EXPORT_SYM
46 #endif
47  
48  
49 #if defined(__GNUC__) && defined(__i386__)
50 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
51 #else
52 # define LIBDEFLATE_ALIGN_STACK
53 #endif
54  
55 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
56  
57 /* #include "../common_defs.h" */
58  
59  
60 #ifndef COMMON_DEFS_H
61 #define COMMON_DEFS_H
62  
63 /* #include "libdeflate.h" */
64  
65  
66 #ifndef LIBDEFLATE_H
67 #define LIBDEFLATE_H
68  
69 #include <stddef.h>
70 #include <stdint.h>
71  
72 #ifdef __cplusplus
73 extern "C" {
74 #endif
75  
76 #define LIBDEFLATE_VERSION_MAJOR 1
77 #define LIBDEFLATE_VERSION_MINOR 26
78 #define LIBDEFLATE_VERSION_STRING "1.26"
79  
80  
81 #ifndef LIBDEFLATEAPI
82 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
83 # define LIBDEFLATEAPI __declspec(dllimport)
84 # else
85 # define LIBDEFLATEAPI
86 # endif
87 #endif
88  
89  
90  
91  
92  
93 struct libdeflate_compressor;
94 struct libdeflate_options;
95  
96  
97 LIBDEFLATEAPI struct libdeflate_compressor *
98 libdeflate_alloc_compressor(int compression_level);
99  
100  
101 LIBDEFLATEAPI struct libdeflate_compressor *
102 libdeflate_alloc_compressor_ex(int compression_level,
103 const struct libdeflate_options *options);
104  
105  
106 LIBDEFLATEAPI size_t
107 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
108 const void *in, size_t in_nbytes,
109 void *out, size_t out_nbytes_avail);
110  
111  
112 LIBDEFLATEAPI size_t
113 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
114 size_t in_nbytes);
115  
116  
117 LIBDEFLATEAPI size_t
118 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
119 const void *in, size_t in_nbytes,
120 void *out, size_t out_nbytes_avail);
121  
122  
123 LIBDEFLATEAPI size_t
124 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
125 size_t in_nbytes);
126  
127  
128 LIBDEFLATEAPI size_t
129 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
130 const void *in, size_t in_nbytes,
131 void *out, size_t out_nbytes_avail);
132  
133  
134 LIBDEFLATEAPI size_t
135 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
136 size_t in_nbytes);
137  
138  
139 LIBDEFLATEAPI void
140 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
141  
142  
143  
144  
145  
146 struct libdeflate_decompressor;
147 struct libdeflate_options;
148  
149  
150 LIBDEFLATEAPI struct libdeflate_decompressor *
151 libdeflate_alloc_decompressor(void);
152  
153  
154 LIBDEFLATEAPI struct libdeflate_decompressor *
155 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
156  
157  
158 enum libdeflate_result {
159
160 LIBDEFLATE_SUCCESS = 0,
161  
162
163 LIBDEFLATE_BAD_DATA = 1,
164  
165
166 LIBDEFLATE_SHORT_OUTPUT = 2,
167  
168
169 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
170 };
171  
172  
173 LIBDEFLATEAPI enum libdeflate_result
174 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
175 const void *in, size_t in_nbytes,
176 void *out, size_t out_nbytes_avail,
177 size_t *actual_out_nbytes_ret);
178  
179  
180 LIBDEFLATEAPI enum libdeflate_result
181 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
182 const void *in, size_t in_nbytes,
183 void *out, size_t out_nbytes_avail,
184 size_t *actual_in_nbytes_ret,
185 size_t *actual_out_nbytes_ret);
186  
187  
188 LIBDEFLATEAPI enum libdeflate_result
189 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
190 const void *in, size_t in_nbytes,
191 void *out, size_t out_nbytes_avail,
192 size_t *actual_out_nbytes_ret);
193  
194  
195 LIBDEFLATEAPI enum libdeflate_result
196 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
197 const void *in, size_t in_nbytes,
198 void *out, size_t out_nbytes_avail,
199 size_t *actual_in_nbytes_ret,
200 size_t *actual_out_nbytes_ret);
201  
202  
203 LIBDEFLATEAPI enum libdeflate_result
204 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
205 const void *in, size_t in_nbytes,
206 void *out, size_t out_nbytes_avail,
207 size_t *actual_out_nbytes_ret);
208  
209  
210 LIBDEFLATEAPI enum libdeflate_result
211 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
212 const void *in, size_t in_nbytes,
213 void *out, size_t out_nbytes_avail,
214 size_t *actual_in_nbytes_ret,
215 size_t *actual_out_nbytes_ret);
216  
217  
218 LIBDEFLATEAPI void
219 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
220  
221  
222  
223  
224  
225  
226 LIBDEFLATEAPI uint32_t
227 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
228  
229  
230  
231 LIBDEFLATEAPI uint32_t
232 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
233  
234  
235  
236  
237  
238  
239 LIBDEFLATEAPI void
240 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
241 void (*free_func)(void *));
242  
243  
244 struct libdeflate_options {
245  
246
247 size_t sizeof_options;
248  
249
250 void *(*malloc_func)(size_t);
251 void (*free_func)(void *);
252 };
253  
254 #ifdef __cplusplus
255 }
256 #endif
257  
258 #endif
259  
260  
261 #include <stdbool.h>
262 #include <stddef.h>
263 #include <stdint.h>
264 #ifdef _MSC_VER
265 # include <intrin.h>
266 # include <stdlib.h>
267
268
269 # pragma warning(disable : 4146)
270
271 # pragma warning(disable : 4018)
272 # pragma warning(disable : 4244)
273 # pragma warning(disable : 4267)
274 # pragma warning(disable : 4310)
275
276 # pragma warning(disable : 4100)
277 # pragma warning(disable : 4127)
278 # pragma warning(disable : 4189)
279 # pragma warning(disable : 4232)
280 # pragma warning(disable : 4245)
281 # pragma warning(disable : 4295)
282 #endif
283 #ifndef FREESTANDING
284 # include <string.h>
285 #endif
286  
287  
288  
289  
290  
291  
292 #undef ARCH_X86_64
293 #undef ARCH_X86_32
294 #undef ARCH_ARM64
295 #undef ARCH_ARM32
296 #undef ARCH_RISCV
297 #ifdef _MSC_VER
298
299 # if defined(_M_X64) && !defined(_M_ARM64EC)
300 # define ARCH_X86_64
301 # elif defined(_M_IX86)
302 # define ARCH_X86_32
303 # elif defined(_M_ARM64)
304 # define ARCH_ARM64
305 # elif defined(_M_ARM)
306 # define ARCH_ARM32
307 # endif
308 #else
309 # if defined(__x86_64__)
310 # define ARCH_X86_64
311 # elif defined(__i386__)
312 # define ARCH_X86_32
313 # elif defined(__aarch64__)
314 # define ARCH_ARM64
315 # elif defined(__arm__)
316 # define ARCH_ARM32
317 # elif defined(__riscv)
318 # define ARCH_RISCV
319 # endif
320 #endif
321  
322  
323  
324  
325  
326  
327 typedef uint8_t u8;
328 typedef uint16_t u16;
329 typedef uint32_t u32;
330 typedef uint64_t u64;
331 typedef int8_t s8;
332 typedef int16_t s16;
333 typedef int32_t s32;
334 typedef int64_t s64;
335  
336  
337 #ifdef _MSC_VER
338 # ifdef _WIN64
339 typedef long long ssize_t;
340 # else
341 typedef long ssize_t;
342 # endif
343 #endif
344  
345  
346 typedef size_t machine_word_t;
347  
348  
349 #define WORDBYTES ((int)sizeof(machine_word_t))
350  
351  
352 #define WORDBITS (8 * WORDBYTES)
353  
354  
355  
356  
357  
358  
359 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
360 # define GCC_PREREQ(major, minor) \
361 (__GNUC__ > (major) || \
362 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
363 # if !GCC_PREREQ(4, 9)
364 # error "gcc versions older than 4.9 are no longer supported"
365 # endif
366 #else
367 # define GCC_PREREQ(major, minor) 0
368 #endif
369 #ifdef __clang__
370 # ifdef __apple_build_version__
371 # define CLANG_PREREQ(major, minor, apple_version) \
372 (__apple_build_version__ >= (apple_version))
373 # else
374 # define CLANG_PREREQ(major, minor, apple_version) \
375 (__clang_major__ > (major) || \
376 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
377 # endif
378 # if !CLANG_PREREQ(3, 9, 8000000)
379 # error "clang versions older than 3.9 are no longer supported"
380 # endif
381 #else
382 # define CLANG_PREREQ(major, minor, apple_version) 0
383 #endif
384 #ifdef _MSC_VER
385 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
386 # if !MSVC_PREREQ(1928)
387 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
388 # endif
389 #else
390 # define MSVC_PREREQ(version) 0
391 #endif
392  
393  
394 #ifndef __has_attribute
395 # define __has_attribute(attribute) 0
396 #endif
397  
398  
399 #ifndef __has_builtin
400 # define __has_builtin(builtin) 0
401 #endif
402  
403  
404 #ifdef _MSC_VER
405 # define inline __inline
406 #endif
407  
408  
409 #if defined(__GNUC__) || __has_attribute(always_inline)
410 # define forceinline inline __attribute__((always_inline))
411 #elif defined(_MSC_VER)
412 # define forceinline __forceinline
413 #else
414 # define forceinline inline
415 #endif
416  
417  
418 #if defined(__GNUC__) || __has_attribute(unused)
419 # define MAYBE_UNUSED __attribute__((unused))
420 #else
421 # define MAYBE_UNUSED
422 #endif
423  
424  
425 #if defined(__GNUC__) || __has_attribute(noreturn)
426 # define NORETURN __attribute__((noreturn))
427 #else
428 # define NORETURN
429 #endif
430  
431  
432 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
433 # define likely(expr) __builtin_expect(!!(expr), 1)
434 #else
435 # define likely(expr) (expr)
436 #endif
437  
438  
439 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
440 # define unlikely(expr) __builtin_expect(!!(expr), 0)
441 #else
442 # define unlikely(expr) (expr)
443 #endif
444  
445  
446 #undef prefetchr
447 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
448 # define prefetchr(addr) __builtin_prefetch((addr), 0)
449 #elif defined(_MSC_VER)
450 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
451 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
452 # elif defined(ARCH_ARM64)
453 # define prefetchr(addr) __prefetch2((addr), 0x00 )
454 # elif defined(ARCH_ARM32)
455 # define prefetchr(addr) __prefetch(addr)
456 # endif
457 #endif
458 #ifndef prefetchr
459 # define prefetchr(addr)
460 #endif
461  
462  
463 #undef prefetchw
464 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
465 # define prefetchw(addr) __builtin_prefetch((addr), 1)
466 #elif defined(_MSC_VER)
467 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
468 # define prefetchw(addr) _m_prefetchw(addr)
469 # elif defined(ARCH_ARM64)
470 # define prefetchw(addr) __prefetch2((addr), 0x10 )
471 # elif defined(ARCH_ARM32)
472 # define prefetchw(addr) __prefetchw(addr)
473 # endif
474 #endif
475 #ifndef prefetchw
476 # define prefetchw(addr)
477 #endif
478  
479  
480 #undef _aligned_attribute
481 #if defined(__GNUC__) || __has_attribute(aligned)
482 # define _aligned_attribute(n) __attribute__((aligned(n)))
483 #elif defined(_MSC_VER)
484 # define _aligned_attribute(n) __declspec(align(n))
485 #endif
486  
487  
488 #if defined(__GNUC__) || __has_attribute(target)
489 # define _target_attribute(attrs) __attribute__((target(attrs)))
490 #else
491 # define _target_attribute(attrs)
492 #endif
493  
494  
495  
496  
497  
498 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
499 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
500 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
501 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
502 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
503 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
504 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
505  
506  
507  
508  
509  
510  
511 #if defined(__BYTE_ORDER__)
512 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
513 #elif defined(_MSC_VER)
514 # define CPU_IS_LITTLE_ENDIAN() true
515 #else
516 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
517 {
518 union {
519 u32 w;
520 u8 b;
521 } u;
522  
523 u.w = 1;
524 return u.b;
525 }
526 #endif
527  
528  
529 static forceinline u16 bswap16(u16 v)
530 {
531 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
532 24 return __builtin_bswap16(v);
533 #elif defined(_MSC_VER)
534 return _byteswap_ushort(v);
535 #else
536 return (v << 8) | (v >> 8);
537 #endif
538 }
539  
540  
541 static forceinline u32 bswap32(u32 v)
542 {
543 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
544 12 return __builtin_bswap32(v);
545 #elif defined(_MSC_VER)
546 return _byteswap_ulong(v);
547 #else
548 return ((v & 0x000000FF) << 24) |
549 ((v & 0x0000FF00) << 8) |
550 ((v & 0x00FF0000) >> 8) |
551 ((v & 0xFF000000) >> 24);
552 #endif
553 }
554  
555  
556 static forceinline u64 bswap64(u64 v)
557 {
558 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
559 return __builtin_bswap64(v);
560 #elif defined(_MSC_VER)
561 return _byteswap_uint64(v);
562 #else
563 return ((v & 0x00000000000000FF) << 56) |
564 ((v & 0x000000000000FF00) << 40) |
565 ((v & 0x0000000000FF0000) << 24) |
566 ((v & 0x00000000FF000000) << 8) |
567 ((v & 0x000000FF00000000) >> 8) |
568 ((v & 0x0000FF0000000000) >> 24) |
569 ((v & 0x00FF000000000000) >> 40) |
570 ((v & 0xFF00000000000000) >> 56);
571 #endif
572 }
573  
574 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
575 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
576 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
577 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
578 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
579 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
580  
581  
582  
583  
584  
585  
586 #if (defined(__GNUC__) || defined(__clang__)) && \
587 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
588 defined(__ARM_FEATURE_UNALIGNED) || \
589 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
590 defined(__riscv_misaligned_fast) || \
591 defined(__wasm__))
592 # define UNALIGNED_ACCESS_IS_FAST 1
593 #elif defined(_MSC_VER)
594 # define UNALIGNED_ACCESS_IS_FAST 1
595 #else
596 # define UNALIGNED_ACCESS_IS_FAST 0
597 #endif
598  
599  
600  
601 #ifdef FREESTANDING
602 # define MEMCOPY __builtin_memcpy
603 #else
604 # define MEMCOPY memcpy
605 #endif
606  
607  
608  
609 #define DEFINE_UNALIGNED_TYPE(type) \
610 static forceinline type \
611 load_##type##_unaligned(const void *p) \
612 { \
613 type v; \
614 \
615 MEMCOPY(&v, p, sizeof(v)); \
616 return v; \
617 } \
618 \
619 static forceinline void \
620 store_##type##_unaligned(type v, void *p) \
621 { \
622 MEMCOPY(p, &v, sizeof(v)); \
623 }
624  
625 12 DEFINE_UNALIGNED_TYPE(u16)
626 90016 DEFINE_UNALIGNED_TYPE(u32)
627 14708 DEFINE_UNALIGNED_TYPE(u64)
628 8448 DEFINE_UNALIGNED_TYPE(machine_word_t)
629  
630 #undef MEMCOPY
631  
632 #define load_word_unaligned load_machine_word_t_unaligned
633 #define store_word_unaligned store_machine_word_t_unaligned
634  
635  
636  
637 static forceinline u16
638 get_unaligned_le16(const u8 *p)
639 {
640 if (UNALIGNED_ACCESS_IS_FAST)
641 0 return le16_bswap(load_u16_unaligned(p));
642 else
643 return ((u16)p[1] << 8) | p[0];
644 }
645  
646 static forceinline u16
647 get_unaligned_be16(const u8 *p)
648 {
649 if (UNALIGNED_ACCESS_IS_FAST)
650 24 return be16_bswap(load_u16_unaligned(p));
651 else
652 return ((u16)p[0] << 8) | p[1];
653 }
654  
655 static forceinline u32
656 get_unaligned_le32(const u8 *p)
657 {
658 if (UNALIGNED_ACCESS_IS_FAST)
659 86219 return le32_bswap(load_u32_unaligned(p));
660 else
661 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
662 ((u32)p[1] << 8) | p[0];
663 }
664  
665 static forceinline u32
666 get_unaligned_be32(const u8 *p)
667 {
668 if (UNALIGNED_ACCESS_IS_FAST)
669 24 return be32_bswap(load_u32_unaligned(p));
670 else
671 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
672 ((u32)p[2] << 8) | p[3];
673 }
674  
675 static forceinline u64
676 get_unaligned_le64(const u8 *p)
677 {
678 if (UNALIGNED_ACCESS_IS_FAST)
679 11015 return le64_bswap(load_u64_unaligned(p));
680 else
681 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
682 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
683 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
684 ((u64)p[1] << 8) | p[0];
685 }
686  
687 static forceinline machine_word_t
688 get_unaligned_leword(const u8 *p)
689 {
690 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
691 if (WORDBITS == 32)
692 return get_unaligned_le32(p);
693 else
694 11015 return get_unaligned_le64(p);
695 }
696  
697  
698  
699 static forceinline void
700 put_unaligned_le16(u16 v, u8 *p)
701 {
702 if (UNALIGNED_ACCESS_IS_FAST) {
703 0 store_u16_unaligned(le16_bswap(v), p);
704 } else {
705 p[0] = (u8)(v >> 0);
706 p[1] = (u8)(v >> 8);
707 }
708 0 }
709  
710 static forceinline void
711 put_unaligned_be16(u16 v, u8 *p)
712 {
713 if (UNALIGNED_ACCESS_IS_FAST) {
714 24 store_u16_unaligned(be16_bswap(v), p);
715 } else {
716 p[0] = (u8)(v >> 8);
717 p[1] = (u8)(v >> 0);
718 }
719 12 }
720  
721 static forceinline void
722 put_unaligned_le32(u32 v, u8 *p)
723 {
724 if (UNALIGNED_ACCESS_IS_FAST) {
725 39 store_u32_unaligned(le32_bswap(v), p);
726 } else {
727 p[0] = (u8)(v >> 0);
728 p[1] = (u8)(v >> 8);
729 p[2] = (u8)(v >> 16);
730 p[3] = (u8)(v >> 24);
731 }
732 39 }
733  
734 static forceinline void
735 put_unaligned_be32(u32 v, u8 *p)
736 {
737 if (UNALIGNED_ACCESS_IS_FAST) {
738 12 store_u32_unaligned(be32_bswap(v), p);
739 } else {
740 p[0] = (u8)(v >> 24);
741 p[1] = (u8)(v >> 16);
742 p[2] = (u8)(v >> 8);
743 p[3] = (u8)(v >> 0);
744 }
745 12 }
746  
747 static forceinline void
748 put_unaligned_le64(u64 v, u8 *p)
749 {
750 if (UNALIGNED_ACCESS_IS_FAST) {
751 3693 store_u64_unaligned(le64_bswap(v), p);
752 } else {
753 p[0] = (u8)(v >> 0);
754 p[1] = (u8)(v >> 8);
755 p[2] = (u8)(v >> 16);
756 p[3] = (u8)(v >> 24);
757 p[4] = (u8)(v >> 32);
758 p[5] = (u8)(v >> 40);
759 p[6] = (u8)(v >> 48);
760 p[7] = (u8)(v >> 56);
761 }
762 3693 }
763  
764 static forceinline void
765 put_unaligned_leword(machine_word_t v, u8 *p)
766 {
767 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
768 if (WORDBITS == 32)
769 put_unaligned_le32(v, p);
770 else
771 put_unaligned_le64(v, p);
772 3693 }
773  
774  
775  
776  
777  
778  
779  
780 static forceinline unsigned
781 bsr32(u32 v)
782 {
783 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
784 12387 return 31 - __builtin_clz(v);
785 #elif defined(_MSC_VER)
786 unsigned long i;
787  
788 _BitScanReverse(&i, v);
789 return i;
790 #else
791 unsigned i = 0;
792  
793 while ((v >>= 1) != 0)
794 i++;
795 return i;
796 #endif
797 }
798  
799 static forceinline unsigned
800 bsr64(u64 v)
801 {
802 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
803 return 63 - __builtin_clzll(v);
804 #elif defined(_MSC_VER) && defined(_WIN64)
805 unsigned long i;
806  
807 _BitScanReverse64(&i, v);
808 return i;
809 #else
810 unsigned i = 0;
811  
812 while ((v >>= 1) != 0)
813 i++;
814 return i;
815 #endif
816 }
817  
818 static forceinline unsigned
819 bsrw(machine_word_t v)
820 {
821 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
822 if (WORDBITS == 32)
823 return bsr32(v);
824 else
825 return bsr64(v);
826 }
827  
828  
829  
830 static forceinline unsigned
831 bsf32(u32 v)
832 {
833 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
834 return __builtin_ctz(v);
835 #elif defined(_MSC_VER)
836 unsigned long i;
837  
838 _BitScanForward(&i, v);
839 return i;
840 #else
841 unsigned i = 0;
842  
843 for (; (v & 1) == 0; v >>= 1)
844 i++;
845 return i;
846 #endif
847 }
848  
849 static forceinline unsigned
850 bsf64(u64 v)
851 {
852 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
853 2632 return __builtin_ctzll(v);
854 #elif defined(_MSC_VER) && defined(_WIN64)
855 unsigned long i;
856  
857 _BitScanForward64(&i, v);
858 return i;
859 #else
860 unsigned i = 0;
861  
862 for (; (v & 1) == 0; v >>= 1)
863 i++;
864 return i;
865 #endif
866 }
867  
868 static forceinline unsigned
869 bsfw(machine_word_t v)
870 {
871 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
872 if (WORDBITS == 32)
873 return bsf32(v);
874 else
875 2632 return bsf64(v);
876 }
877  
878  
879 #undef rbit32
880 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
881 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
882 static forceinline u32
883 rbit32(u32 v)
884 {
885 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
886 return v;
887 }
888 #define rbit32 rbit32
889 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
890 static forceinline u32
891 rbit32(u32 v)
892 {
893 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
894 return v;
895 }
896 #define rbit32 rbit32
897 #endif
898  
899 #endif
900  
901  
902 typedef void *(*malloc_func_t)(size_t);
903 typedef void (*free_func_t)(void *);
904  
905 extern malloc_func_t libdeflate_default_malloc_func;
906 extern free_func_t libdeflate_default_free_func;
907  
908 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
909 size_t alignment, size_t size);
910 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
911  
912 #ifdef FREESTANDING
913  
914 void *memset(void *s, int c, size_t n);
915 #define memset(s, c, n) __builtin_memset((s), (c), (n))
916  
917 void *memcpy(void *dest, const void *src, size_t n);
918 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
919  
920 void *memmove(void *dest, const void *src, size_t n);
921 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
922  
923 int memcmp(const void *s1, const void *s2, size_t n);
924 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
925  
926 #undef LIBDEFLATE_ENABLE_ASSERTIONS
927 #else
928 # include <string.h>
929
930 # ifdef __clang_analyzer__
931 # define LIBDEFLATE_ENABLE_ASSERTIONS
932 # endif
933 #endif
934  
935  
936 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
937 NORETURN void
938 libdeflate_assertion_failed(const char *expr, const char *file, int line);
939 #define ASSERT(expr) { if (unlikely(!(expr))) \
940 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
941 #else
942 #define ASSERT(expr) (void)(expr)
943 #endif
944  
945 #define CONCAT_IMPL(a, b) a##b
946 #define CONCAT(a, b) CONCAT_IMPL(a, b)
947 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
948  
949 #endif
950  
951  
952  
953 #define DIVISOR 65521
954  
955  
956 #define MAX_CHUNK_LEN 5552
957  
958  
959 #define ADLER32_CHUNK(s1, s2, p, n) \
960 do { \
961 if (n >= 4) { \
962 u32 s1_sum = 0; \
963 u32 byte_0_sum = 0; \
964 u32 byte_1_sum = 0; \
965 u32 byte_2_sum = 0; \
966 u32 byte_3_sum = 0; \
967 \
968 do { \
969 s1_sum += s1; \
970 s1 += p[0] + p[1] + p[2] + p[3]; \
971 byte_0_sum += p[0]; \
972 byte_1_sum += p[1]; \
973 byte_2_sum += p[2]; \
974 byte_3_sum += p[3]; \
975 p += 4; \
976 n -= 4; \
977 } while (n >= 4); \
978 s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) + \
979 (2 * byte_2_sum) + byte_3_sum; \
980 } \
981 for (; n; n--, p++) { \
982 s1 += *p; \
983 s2 += s1; \
984 } \
985 s1 %= DIVISOR; \
986 s2 %= DIVISOR; \
987 } while (0)
988  
989 static u32 MAYBE_UNUSED
990 0 adler32_generic(u32 adler, const u8 *p, size_t len)
991 {
992 0 u32 s1 = adler & 0xFFFF;
993 0 u32 s2 = adler >> 16;
994  
995 0 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 0 0 0
996 0 size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
997  
998 0 len -= n;
999 0 ▶ ADLER32_CHUNK(s1, s2, p, n);
Branch: 3 branches
BranchTrueFalseTotal
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
1000 }
1001  
1002 0 return (s2 << 16) | s1;
1003 }
1004  
1005  
1006 #undef DEFAULT_IMPL
1007 #undef arch_select_adler32_func
1008 typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
1009 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
1010 /* # include "arm/adler32_impl.h" */
1011  
1012  
1013 #ifndef LIB_ARM_ADLER32_IMPL_H
1014 #define LIB_ARM_ADLER32_IMPL_H
1015  
1016 /* #include "arm-cpu_features.h" */
1017  
1018  
1019 #ifndef LIB_ARM_CPU_FEATURES_H
1020 #define LIB_ARM_CPU_FEATURES_H
1021  
1022 /* #include "lib_common.h" */
1023  
1024  
1025 #ifndef LIB_LIB_COMMON_H
1026 #define LIB_LIB_COMMON_H
1027  
1028 #ifdef LIBDEFLATE_H
1029
1030 # error "lib_common.h must always be included before libdeflate.h"
1031 #endif
1032  
1033 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
1034 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
1035 #elif defined(__GNUC__)
1036 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
1037 #else
1038 # define LIBDEFLATE_EXPORT_SYM
1039 #endif
1040  
1041  
1042 #if defined(__GNUC__) && defined(__i386__)
1043 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
1044 #else
1045 # define LIBDEFLATE_ALIGN_STACK
1046 #endif
1047  
1048 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
1049  
1050 /* #include "../common_defs.h" */
1051  
1052  
1053 #ifndef COMMON_DEFS_H
1054 #define COMMON_DEFS_H
1055  
1056 /* #include "libdeflate.h" */
1057  
1058  
1059 #ifndef LIBDEFLATE_H
1060 #define LIBDEFLATE_H
1061  
1062 #include <stddef.h>
1063 #include <stdint.h>
1064  
1065 #ifdef __cplusplus
1066 extern "C" {
1067 #endif
1068  
1069 #define LIBDEFLATE_VERSION_MAJOR 1
1070 #define LIBDEFLATE_VERSION_MINOR 26
1071 #define LIBDEFLATE_VERSION_STRING "1.26"
1072  
1073  
1074 #ifndef LIBDEFLATEAPI
1075 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
1076 # define LIBDEFLATEAPI __declspec(dllimport)
1077 # else
1078 # define LIBDEFLATEAPI
1079 # endif
1080 #endif
1081  
1082  
1083  
1084  
1085  
1086 struct libdeflate_compressor;
1087 struct libdeflate_options;
1088  
1089  
1090 LIBDEFLATEAPI struct libdeflate_compressor *
1091 libdeflate_alloc_compressor(int compression_level);
1092  
1093  
1094 LIBDEFLATEAPI struct libdeflate_compressor *
1095 libdeflate_alloc_compressor_ex(int compression_level,
1096 const struct libdeflate_options *options);
1097  
1098  
1099 LIBDEFLATEAPI size_t
1100 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
1101 const void *in, size_t in_nbytes,
1102 void *out, size_t out_nbytes_avail);
1103  
1104  
1105 LIBDEFLATEAPI size_t
1106 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
1107 size_t in_nbytes);
1108  
1109  
1110 LIBDEFLATEAPI size_t
1111 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
1112 const void *in, size_t in_nbytes,
1113 void *out, size_t out_nbytes_avail);
1114  
1115  
1116 LIBDEFLATEAPI size_t
1117 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
1118 size_t in_nbytes);
1119  
1120  
1121 LIBDEFLATEAPI size_t
1122 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
1123 const void *in, size_t in_nbytes,
1124 void *out, size_t out_nbytes_avail);
1125  
1126  
1127 LIBDEFLATEAPI size_t
1128 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
1129 size_t in_nbytes);
1130  
1131  
1132 LIBDEFLATEAPI void
1133 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
1134  
1135  
1136  
1137  
1138  
1139 struct libdeflate_decompressor;
1140 struct libdeflate_options;
1141  
1142  
1143 LIBDEFLATEAPI struct libdeflate_decompressor *
1144 libdeflate_alloc_decompressor(void);
1145  
1146  
1147 LIBDEFLATEAPI struct libdeflate_decompressor *
1148 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
1149  
1150  
1151 enum libdeflate_result {
1152
1153 LIBDEFLATE_SUCCESS = 0,
1154  
1155
1156 LIBDEFLATE_BAD_DATA = 1,
1157  
1158
1159 LIBDEFLATE_SHORT_OUTPUT = 2,
1160  
1161
1162 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
1163 };
1164  
1165  
1166 LIBDEFLATEAPI enum libdeflate_result
1167 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
1168 const void *in, size_t in_nbytes,
1169 void *out, size_t out_nbytes_avail,
1170 size_t *actual_out_nbytes_ret);
1171  
1172  
1173 LIBDEFLATEAPI enum libdeflate_result
1174 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
1175 const void *in, size_t in_nbytes,
1176 void *out, size_t out_nbytes_avail,
1177 size_t *actual_in_nbytes_ret,
1178 size_t *actual_out_nbytes_ret);
1179  
1180  
1181 LIBDEFLATEAPI enum libdeflate_result
1182 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
1183 const void *in, size_t in_nbytes,
1184 void *out, size_t out_nbytes_avail,
1185 size_t *actual_out_nbytes_ret);
1186  
1187  
1188 LIBDEFLATEAPI enum libdeflate_result
1189 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
1190 const void *in, size_t in_nbytes,
1191 void *out, size_t out_nbytes_avail,
1192 size_t *actual_in_nbytes_ret,
1193 size_t *actual_out_nbytes_ret);
1194  
1195  
1196 LIBDEFLATEAPI enum libdeflate_result
1197 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
1198 const void *in, size_t in_nbytes,
1199 void *out, size_t out_nbytes_avail,
1200 size_t *actual_out_nbytes_ret);
1201  
1202  
1203 LIBDEFLATEAPI enum libdeflate_result
1204 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
1205 const void *in, size_t in_nbytes,
1206 void *out, size_t out_nbytes_avail,
1207 size_t *actual_in_nbytes_ret,
1208 size_t *actual_out_nbytes_ret);
1209  
1210  
1211 LIBDEFLATEAPI void
1212 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
1213  
1214  
1215  
1216  
1217  
1218  
1219 LIBDEFLATEAPI uint32_t
1220 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
1221  
1222  
1223  
1224 LIBDEFLATEAPI uint32_t
1225 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
1226  
1227  
1228  
1229  
1230  
1231  
1232 LIBDEFLATEAPI void
1233 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
1234 void (*free_func)(void *));
1235  
1236  
1237 struct libdeflate_options {
1238  
1239
1240 size_t sizeof_options;
1241  
1242
1243 void *(*malloc_func)(size_t);
1244 void (*free_func)(void *);
1245 };
1246  
1247 #ifdef __cplusplus
1248 }
1249 #endif
1250  
1251 #endif
1252  
1253  
1254 #include <stdbool.h>
1255 #include <stddef.h>
1256 #include <stdint.h>
1257 #ifdef _MSC_VER
1258 # include <intrin.h>
1259 # include <stdlib.h>
1260
1261
1262 # pragma warning(disable : 4146)
1263
1264 # pragma warning(disable : 4018)
1265 # pragma warning(disable : 4244)
1266 # pragma warning(disable : 4267)
1267 # pragma warning(disable : 4310)
1268
1269 # pragma warning(disable : 4100)
1270 # pragma warning(disable : 4127)
1271 # pragma warning(disable : 4189)
1272 # pragma warning(disable : 4232)
1273 # pragma warning(disable : 4245)
1274 # pragma warning(disable : 4295)
1275 #endif
1276 #ifndef FREESTANDING
1277 # include <string.h>
1278 #endif
1279  
1280  
1281  
1282  
1283  
1284  
1285 #undef ARCH_X86_64
1286 #undef ARCH_X86_32
1287 #undef ARCH_ARM64
1288 #undef ARCH_ARM32
1289 #undef ARCH_RISCV
1290 #ifdef _MSC_VER
1291
1292 # if defined(_M_X64) && !defined(_M_ARM64EC)
1293 # define ARCH_X86_64
1294 # elif defined(_M_IX86)
1295 # define ARCH_X86_32
1296 # elif defined(_M_ARM64)
1297 # define ARCH_ARM64
1298 # elif defined(_M_ARM)
1299 # define ARCH_ARM32
1300 # endif
1301 #else
1302 # if defined(__x86_64__)
1303 # define ARCH_X86_64
1304 # elif defined(__i386__)
1305 # define ARCH_X86_32
1306 # elif defined(__aarch64__)
1307 # define ARCH_ARM64
1308 # elif defined(__arm__)
1309 # define ARCH_ARM32
1310 # elif defined(__riscv)
1311 # define ARCH_RISCV
1312 # endif
1313 #endif
1314  
1315  
1316  
1317  
1318  
1319  
1320 typedef uint8_t u8;
1321 typedef uint16_t u16;
1322 typedef uint32_t u32;
1323 typedef uint64_t u64;
1324 typedef int8_t s8;
1325 typedef int16_t s16;
1326 typedef int32_t s32;
1327 typedef int64_t s64;
1328  
1329  
1330 #ifdef _MSC_VER
1331 # ifdef _WIN64
1332 typedef long long ssize_t;
1333 # else
1334 typedef long ssize_t;
1335 # endif
1336 #endif
1337  
1338  
1339 typedef size_t machine_word_t;
1340  
1341  
1342 #define WORDBYTES ((int)sizeof(machine_word_t))
1343  
1344  
1345 #define WORDBITS (8 * WORDBYTES)
1346  
1347  
1348  
1349  
1350  
1351  
1352 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
1353 # define GCC_PREREQ(major, minor) \
1354 (__GNUC__ > (major) || \
1355 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
1356 # if !GCC_PREREQ(4, 9)
1357 # error "gcc versions older than 4.9 are no longer supported"
1358 # endif
1359 #else
1360 # define GCC_PREREQ(major, minor) 0
1361 #endif
1362 #ifdef __clang__
1363 # ifdef __apple_build_version__
1364 # define CLANG_PREREQ(major, minor, apple_version) \
1365 (__apple_build_version__ >= (apple_version))
1366 # else
1367 # define CLANG_PREREQ(major, minor, apple_version) \
1368 (__clang_major__ > (major) || \
1369 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
1370 # endif
1371 # if !CLANG_PREREQ(3, 9, 8000000)
1372 # error "clang versions older than 3.9 are no longer supported"
1373 # endif
1374 #else
1375 # define CLANG_PREREQ(major, minor, apple_version) 0
1376 #endif
1377 #ifdef _MSC_VER
1378 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
1379 # if !MSVC_PREREQ(1928)
1380 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
1381 # endif
1382 #else
1383 # define MSVC_PREREQ(version) 0
1384 #endif
1385  
1386  
1387 #ifndef __has_attribute
1388 # define __has_attribute(attribute) 0
1389 #endif
1390  
1391  
1392 #ifndef __has_builtin
1393 # define __has_builtin(builtin) 0
1394 #endif
1395  
1396  
1397 #ifdef _MSC_VER
1398 # define inline __inline
1399 #endif
1400  
1401  
1402 #if defined(__GNUC__) || __has_attribute(always_inline)
1403 # define forceinline inline __attribute__((always_inline))
1404 #elif defined(_MSC_VER)
1405 # define forceinline __forceinline
1406 #else
1407 # define forceinline inline
1408 #endif
1409  
1410  
1411 #if defined(__GNUC__) || __has_attribute(unused)
1412 # define MAYBE_UNUSED __attribute__((unused))
1413 #else
1414 # define MAYBE_UNUSED
1415 #endif
1416  
1417  
1418 #if defined(__GNUC__) || __has_attribute(noreturn)
1419 # define NORETURN __attribute__((noreturn))
1420 #else
1421 # define NORETURN
1422 #endif
1423  
1424  
1425 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
1426 # define likely(expr) __builtin_expect(!!(expr), 1)
1427 #else
1428 # define likely(expr) (expr)
1429 #endif
1430  
1431  
1432 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
1433 # define unlikely(expr) __builtin_expect(!!(expr), 0)
1434 #else
1435 # define unlikely(expr) (expr)
1436 #endif
1437  
1438  
1439 #undef prefetchr
1440 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
1441 # define prefetchr(addr) __builtin_prefetch((addr), 0)
1442 #elif defined(_MSC_VER)
1443 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
1444 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
1445 # elif defined(ARCH_ARM64)
1446 # define prefetchr(addr) __prefetch2((addr), 0x00 )
1447 # elif defined(ARCH_ARM32)
1448 # define prefetchr(addr) __prefetch(addr)
1449 # endif
1450 #endif
1451 #ifndef prefetchr
1452 # define prefetchr(addr)
1453 #endif
1454  
1455  
1456 #undef prefetchw
1457 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
1458 # define prefetchw(addr) __builtin_prefetch((addr), 1)
1459 #elif defined(_MSC_VER)
1460 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
1461 # define prefetchw(addr) _m_prefetchw(addr)
1462 # elif defined(ARCH_ARM64)
1463 # define prefetchw(addr) __prefetch2((addr), 0x10 )
1464 # elif defined(ARCH_ARM32)
1465 # define prefetchw(addr) __prefetchw(addr)
1466 # endif
1467 #endif
1468 #ifndef prefetchw
1469 # define prefetchw(addr)
1470 #endif
1471  
1472  
1473 #undef _aligned_attribute
1474 #if defined(__GNUC__) || __has_attribute(aligned)
1475 # define _aligned_attribute(n) __attribute__((aligned(n)))
1476 #elif defined(_MSC_VER)
1477 # define _aligned_attribute(n) __declspec(align(n))
1478 #endif
1479  
1480  
1481 #if defined(__GNUC__) || __has_attribute(target)
1482 # define _target_attribute(attrs) __attribute__((target(attrs)))
1483 #else
1484 # define _target_attribute(attrs)
1485 #endif
1486  
1487  
1488  
1489  
1490  
1491 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
1492 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
1493 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
1494 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
1495 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
1496 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
1497 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
1498  
1499  
1500  
1501  
1502  
1503  
1504 #if defined(__BYTE_ORDER__)
1505 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
1506 #elif defined(_MSC_VER)
1507 # define CPU_IS_LITTLE_ENDIAN() true
1508 #else
1509 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
1510 {
1511 union {
1512 u32 w;
1513 u8 b;
1514 } u;
1515  
1516 u.w = 1;
1517 return u.b;
1518 }
1519 #endif
1520  
1521  
1522 static forceinline u16 bswap16(u16 v)
1523 {
1524 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
1525 return __builtin_bswap16(v);
1526 #elif defined(_MSC_VER)
1527 return _byteswap_ushort(v);
1528 #else
1529 return (v << 8) | (v >> 8);
1530 #endif
1531 }
1532  
1533  
1534 static forceinline u32 bswap32(u32 v)
1535 {
1536 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
1537 return __builtin_bswap32(v);
1538 #elif defined(_MSC_VER)
1539 return _byteswap_ulong(v);
1540 #else
1541 return ((v & 0x000000FF) << 24) |
1542 ((v & 0x0000FF00) << 8) |
1543 ((v & 0x00FF0000) >> 8) |
1544 ((v & 0xFF000000) >> 24);
1545 #endif
1546 }
1547  
1548  
1549 static forceinline u64 bswap64(u64 v)
1550 {
1551 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
1552 return __builtin_bswap64(v);
1553 #elif defined(_MSC_VER)
1554 return _byteswap_uint64(v);
1555 #else
1556 return ((v & 0x00000000000000FF) << 56) |
1557 ((v & 0x000000000000FF00) << 40) |
1558 ((v & 0x0000000000FF0000) << 24) |
1559 ((v & 0x00000000FF000000) << 8) |
1560 ((v & 0x000000FF00000000) >> 8) |
1561 ((v & 0x0000FF0000000000) >> 24) |
1562 ((v & 0x00FF000000000000) >> 40) |
1563 ((v & 0xFF00000000000000) >> 56);
1564 #endif
1565 }
1566  
1567 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
1568 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
1569 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
1570 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
1571 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
1572 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
1573  
1574  
1575  
1576  
1577  
1578  
1579 #if (defined(__GNUC__) || defined(__clang__)) && \
1580 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
1581 defined(__ARM_FEATURE_UNALIGNED) || \
1582 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
1583 defined(__riscv_misaligned_fast) || \
1584 defined(__wasm__))
1585 # define UNALIGNED_ACCESS_IS_FAST 1
1586 #elif defined(_MSC_VER)
1587 # define UNALIGNED_ACCESS_IS_FAST 1
1588 #else
1589 # define UNALIGNED_ACCESS_IS_FAST 0
1590 #endif
1591  
1592  
1593  
1594 #ifdef FREESTANDING
1595 # define MEMCOPY __builtin_memcpy
1596 #else
1597 # define MEMCOPY memcpy
1598 #endif
1599  
1600  
1601  
1602 #define DEFINE_UNALIGNED_TYPE(type) \
1603 static forceinline type \
1604 load_##type##_unaligned(const void *p) \
1605 { \
1606 type v; \
1607 \
1608 MEMCOPY(&v, p, sizeof(v)); \
1609 return v; \
1610 } \
1611 \
1612 static forceinline void \
1613 store_##type##_unaligned(type v, void *p) \
1614 { \
1615 MEMCOPY(p, &v, sizeof(v)); \
1616 }
1617  
1618 DEFINE_UNALIGNED_TYPE(u16)
1619 DEFINE_UNALIGNED_TYPE(u32)
1620 DEFINE_UNALIGNED_TYPE(u64)
1621 DEFINE_UNALIGNED_TYPE(machine_word_t)
1622  
1623 #undef MEMCOPY
1624  
1625 #define load_word_unaligned load_machine_word_t_unaligned
1626 #define store_word_unaligned store_machine_word_t_unaligned
1627  
1628  
1629  
1630 static forceinline u16
1631 get_unaligned_le16(const u8 *p)
1632 {
1633 if (UNALIGNED_ACCESS_IS_FAST)
1634 return le16_bswap(load_u16_unaligned(p));
1635 else
1636 return ((u16)p[1] << 8) | p[0];
1637 }
1638  
1639 static forceinline u16
1640 get_unaligned_be16(const u8 *p)
1641 {
1642 if (UNALIGNED_ACCESS_IS_FAST)
1643 return be16_bswap(load_u16_unaligned(p));
1644 else
1645 return ((u16)p[0] << 8) | p[1];
1646 }
1647  
1648 static forceinline u32
1649 get_unaligned_le32(const u8 *p)
1650 {
1651 if (UNALIGNED_ACCESS_IS_FAST)
1652 return le32_bswap(load_u32_unaligned(p));
1653 else
1654 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
1655 ((u32)p[1] << 8) | p[0];
1656 }
1657  
1658 static forceinline u32
1659 get_unaligned_be32(const u8 *p)
1660 {
1661 if (UNALIGNED_ACCESS_IS_FAST)
1662 return be32_bswap(load_u32_unaligned(p));
1663 else
1664 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
1665 ((u32)p[2] << 8) | p[3];
1666 }
1667  
1668 static forceinline u64
1669 get_unaligned_le64(const u8 *p)
1670 {
1671 if (UNALIGNED_ACCESS_IS_FAST)
1672 return le64_bswap(load_u64_unaligned(p));
1673 else
1674 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
1675 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
1676 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
1677 ((u64)p[1] << 8) | p[0];
1678 }
1679  
1680 static forceinline machine_word_t
1681 get_unaligned_leword(const u8 *p)
1682 {
1683 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
1684 if (WORDBITS == 32)
1685 return get_unaligned_le32(p);
1686 else
1687 return get_unaligned_le64(p);
1688 }
1689  
1690  
1691  
1692 static forceinline void
1693 put_unaligned_le16(u16 v, u8 *p)
1694 {
1695 if (UNALIGNED_ACCESS_IS_FAST) {
1696 store_u16_unaligned(le16_bswap(v), p);
1697 } else {
1698 p[0] = (u8)(v >> 0);
1699 p[1] = (u8)(v >> 8);
1700 }
1701 }
1702  
1703 static forceinline void
1704 put_unaligned_be16(u16 v, u8 *p)
1705 {
1706 if (UNALIGNED_ACCESS_IS_FAST) {
1707 store_u16_unaligned(be16_bswap(v), p);
1708 } else {
1709 p[0] = (u8)(v >> 8);
1710 p[1] = (u8)(v >> 0);
1711 }
1712 }
1713  
1714 static forceinline void
1715 put_unaligned_le32(u32 v, u8 *p)
1716 {
1717 if (UNALIGNED_ACCESS_IS_FAST) {
1718 store_u32_unaligned(le32_bswap(v), p);
1719 } else {
1720 p[0] = (u8)(v >> 0);
1721 p[1] = (u8)(v >> 8);
1722 p[2] = (u8)(v >> 16);
1723 p[3] = (u8)(v >> 24);
1724 }
1725 }
1726  
1727 static forceinline void
1728 put_unaligned_be32(u32 v, u8 *p)
1729 {
1730 if (UNALIGNED_ACCESS_IS_FAST) {
1731 store_u32_unaligned(be32_bswap(v), p);
1732 } else {
1733 p[0] = (u8)(v >> 24);
1734 p[1] = (u8)(v >> 16);
1735 p[2] = (u8)(v >> 8);
1736 p[3] = (u8)(v >> 0);
1737 }
1738 }
1739  
1740 static forceinline void
1741 put_unaligned_le64(u64 v, u8 *p)
1742 {
1743 if (UNALIGNED_ACCESS_IS_FAST) {
1744 store_u64_unaligned(le64_bswap(v), p);
1745 } else {
1746 p[0] = (u8)(v >> 0);
1747 p[1] = (u8)(v >> 8);
1748 p[2] = (u8)(v >> 16);
1749 p[3] = (u8)(v >> 24);
1750 p[4] = (u8)(v >> 32);
1751 p[5] = (u8)(v >> 40);
1752 p[6] = (u8)(v >> 48);
1753 p[7] = (u8)(v >> 56);
1754 }
1755 }
1756  
1757 static forceinline void
1758 put_unaligned_leword(machine_word_t v, u8 *p)
1759 {
1760 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
1761 if (WORDBITS == 32)
1762 put_unaligned_le32(v, p);
1763 else
1764 put_unaligned_le64(v, p);
1765 }
1766  
1767  
1768  
1769  
1770  
1771  
1772  
1773 static forceinline unsigned
1774 bsr32(u32 v)
1775 {
1776 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
1777 return 31 - __builtin_clz(v);
1778 #elif defined(_MSC_VER)
1779 unsigned long i;
1780  
1781 _BitScanReverse(&i, v);
1782 return i;
1783 #else
1784 unsigned i = 0;
1785  
1786 while ((v >>= 1) != 0)
1787 i++;
1788 return i;
1789 #endif
1790 }
1791  
1792 static forceinline unsigned
1793 bsr64(u64 v)
1794 {
1795 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
1796 return 63 - __builtin_clzll(v);
1797 #elif defined(_MSC_VER) && defined(_WIN64)
1798 unsigned long i;
1799  
1800 _BitScanReverse64(&i, v);
1801 return i;
1802 #else
1803 unsigned i = 0;
1804  
1805 while ((v >>= 1) != 0)
1806 i++;
1807 return i;
1808 #endif
1809 }
1810  
1811 static forceinline unsigned
1812 bsrw(machine_word_t v)
1813 {
1814 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
1815 if (WORDBITS == 32)
1816 return bsr32(v);
1817 else
1818 return bsr64(v);
1819 }
1820  
1821  
1822  
1823 static forceinline unsigned
1824 bsf32(u32 v)
1825 {
1826 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
1827 return __builtin_ctz(v);
1828 #elif defined(_MSC_VER)
1829 unsigned long i;
1830  
1831 _BitScanForward(&i, v);
1832 return i;
1833 #else
1834 unsigned i = 0;
1835  
1836 for (; (v & 1) == 0; v >>= 1)
1837 i++;
1838 return i;
1839 #endif
1840 }
1841  
1842 static forceinline unsigned
1843 bsf64(u64 v)
1844 {
1845 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
1846 return __builtin_ctzll(v);
1847 #elif defined(_MSC_VER) && defined(_WIN64)
1848 unsigned long i;
1849  
1850 _BitScanForward64(&i, v);
1851 return i;
1852 #else
1853 unsigned i = 0;
1854  
1855 for (; (v & 1) == 0; v >>= 1)
1856 i++;
1857 return i;
1858 #endif
1859 }
1860  
1861 static forceinline unsigned
1862 bsfw(machine_word_t v)
1863 {
1864 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
1865 if (WORDBITS == 32)
1866 return bsf32(v);
1867 else
1868 return bsf64(v);
1869 }
1870  
1871  
1872 #undef rbit32
1873 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
1874 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
1875 static forceinline u32
1876 rbit32(u32 v)
1877 {
1878 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
1879 return v;
1880 }
1881 #define rbit32 rbit32
1882 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
1883 static forceinline u32
1884 rbit32(u32 v)
1885 {
1886 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
1887 return v;
1888 }
1889 #define rbit32 rbit32
1890 #endif
1891  
1892 #endif
1893  
1894  
1895 typedef void *(*malloc_func_t)(size_t);
1896 typedef void (*free_func_t)(void *);
1897  
1898 extern malloc_func_t libdeflate_default_malloc_func;
1899 extern free_func_t libdeflate_default_free_func;
1900  
1901 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
1902 size_t alignment, size_t size);
1903 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
1904  
1905 #ifdef FREESTANDING
1906  
1907 void *memset(void *s, int c, size_t n);
1908 #define memset(s, c, n) __builtin_memset((s), (c), (n))
1909  
1910 void *memcpy(void *dest, const void *src, size_t n);
1911 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
1912  
1913 void *memmove(void *dest, const void *src, size_t n);
1914 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
1915  
1916 int memcmp(const void *s1, const void *s2, size_t n);
1917 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
1918  
1919 #undef LIBDEFLATE_ENABLE_ASSERTIONS
1920 #else
1921 # include <string.h>
1922
1923 # ifdef __clang_analyzer__
1924 # define LIBDEFLATE_ENABLE_ASSERTIONS
1925 # endif
1926 #endif
1927  
1928  
1929 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
1930 NORETURN void
1931 libdeflate_assertion_failed(const char *expr, const char *file, int line);
1932 #define ASSERT(expr) { if (unlikely(!(expr))) \
1933 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
1934 #else
1935 #define ASSERT(expr) (void)(expr)
1936 #endif
1937  
1938 #define CONCAT_IMPL(a, b) a##b
1939 #define CONCAT(a, b) CONCAT_IMPL(a, b)
1940 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
1941  
1942 #endif
1943  
1944  
1945 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
1946  
1947 #define ARM_CPU_FEATURE_NEON (1 << 0)
1948 #define ARM_CPU_FEATURE_PMULL (1 << 1)
1949  
1950 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
1951 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
1952 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
1953 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
1954  
1955 #if !defined(FREESTANDING) && \
1956 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
1957 (defined(__linux__) || \
1958 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
1959 (defined(_WIN32) && defined(ARCH_ARM64)))
1960  
1961 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
1962 extern volatile u32 libdeflate_arm_cpu_features;
1963  
1964 void libdeflate_init_arm_cpu_features(void);
1965  
1966 static inline u32 get_arm_cpu_features(void)
1967 {
1968 if (libdeflate_arm_cpu_features == 0)
1969 libdeflate_init_arm_cpu_features();
1970 return libdeflate_arm_cpu_features;
1971 }
1972 #else
1973 static inline u32 get_arm_cpu_features(void) { return 0; }
1974 #endif
1975  
1976  
1977 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
1978 # define HAVE_NEON(features) 1
1979 # define HAVE_NEON_NATIVE 1
1980 #else
1981 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
1982 # define HAVE_NEON_NATIVE 0
1983 #endif
1984  
1985 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
1986 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
1987 # define HAVE_NEON_INTRIN 1
1988 # include <arm_neon.h>
1989 #else
1990 # define HAVE_NEON_INTRIN 0
1991 #endif
1992  
1993  
1994 #ifdef __ARM_FEATURE_CRYPTO
1995 # define HAVE_PMULL(features) 1
1996 #else
1997 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
1998 #endif
1999 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
2000 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
2001 CPU_IS_LITTLE_ENDIAN()
2002 # define HAVE_PMULL_INTRIN 1
2003
2004 # ifdef _MSC_VER
2005 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
2006 # else
2007 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
2008 # endif
2009 #else
2010 # define HAVE_PMULL_INTRIN 0
2011 #endif
2012  
2013  
2014 #ifdef __ARM_FEATURE_CRC32
2015 # define HAVE_CRC32(features) 1
2016 #else
2017 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
2018 #endif
2019 #if defined(ARCH_ARM64) && \
2020 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
2021 # define HAVE_CRC32_INTRIN 1
2022 # if defined(__GNUC__) || defined(__clang__)
2023 # include <arm_acle.h>
2024 # endif
2025
2026 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
2027 !defined(__ARM_FEATURE_CRC32)
2028 # undef __crc32b
2029 # define __crc32b(a, b) \
2030 ({ uint32_t res; \
2031 __asm__("crc32b %w0, %w1, %w2" \
2032 : "=r" (res) : "r" (a), "r" (b)); \
2033 res; })
2034 # undef __crc32h
2035 # define __crc32h(a, b) \
2036 ({ uint32_t res; \
2037 __asm__("crc32h %w0, %w1, %w2" \
2038 : "=r" (res) : "r" (a), "r" (b)); \
2039 res; })
2040 # undef __crc32w
2041 # define __crc32w(a, b) \
2042 ({ uint32_t res; \
2043 __asm__("crc32w %w0, %w1, %w2" \
2044 : "=r" (res) : "r" (a), "r" (b)); \
2045 res; })
2046 # undef __crc32d
2047 # define __crc32d(a, b) \
2048 ({ uint32_t res; \
2049 __asm__("crc32x %w0, %w1, %2" \
2050 : "=r" (res) : "r" (a), "r" (b)); \
2051 res; })
2052 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
2053 # endif
2054 #else
2055 # define HAVE_CRC32_INTRIN 0
2056 #endif
2057  
2058  
2059 #ifdef __ARM_FEATURE_SHA3
2060 # define HAVE_SHA3(features) 1
2061 #else
2062 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
2063 #endif
2064 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
2065 (GCC_PREREQ(9, 1) || \
2066 CLANG_PREREQ(7, 0, 10010463) )
2067 # define HAVE_SHA3_INTRIN 1
2068
2069 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
2070 !defined(__ARM_FEATURE_SHA3)
2071 # undef veor3q_u8
2072 # define veor3q_u8(a, b, c) \
2073 ({ uint8x16_t res; \
2074 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
2075 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
2076 res; })
2077 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
2078 # endif
2079 #else
2080 # define HAVE_SHA3_INTRIN 0
2081 #endif
2082  
2083  
2084 #ifdef __ARM_FEATURE_DOTPROD
2085 # define HAVE_DOTPROD(features) 1
2086 #else
2087 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
2088 #endif
2089 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
2090 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
2091 # define HAVE_DOTPROD_INTRIN 1
2092
2093 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
2094 !defined(__ARM_FEATURE_DOTPROD)
2095 # undef vdotq_u32
2096 # define vdotq_u32(a, b, c) \
2097 ({ uint32x4_t res = (a); \
2098 __asm__("udot %0.4s, %1.16b, %2.16b" \
2099 : "+w" (res) : "w" (b), "w" (c)); \
2100 res; })
2101 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
2102 # endif
2103 #else
2104 # define HAVE_DOTPROD_INTRIN 0
2105 #endif
2106  
2107 #endif
2108  
2109 #endif
2110  
2111  
2112  
2113 #if HAVE_NEON_INTRIN && CPU_IS_LITTLE_ENDIAN()
2114 # define adler32_arm_neon adler32_arm_neon
2115 # if HAVE_NEON_NATIVE
2116
2117 # define ATTRIBUTES
2118 # elif defined(ARCH_ARM32)
2119 # define ATTRIBUTES _target_attribute("fpu=neon")
2120 # elif defined(__clang__)
2121 # define ATTRIBUTES _target_attribute("simd")
2122 # else
2123 # define ATTRIBUTES _target_attribute("+simd")
2124 # endif
2125 static ATTRIBUTES MAYBE_UNUSED u32
2126 adler32_arm_neon(u32 adler, const u8 *p, size_t len)
2127 {
2128 static const u16 _aligned_attribute(16) mults[64] = {
2129 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
2130 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
2131 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
2132 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
2133 };
2134 const uint16x8_t mults_a = vld1q_u16(&mults[0]);
2135 const uint16x8_t mults_b = vld1q_u16(&mults[8]);
2136 const uint16x8_t mults_c = vld1q_u16(&mults[16]);
2137 const uint16x8_t mults_d = vld1q_u16(&mults[24]);
2138 const uint16x8_t mults_e = vld1q_u16(&mults[32]);
2139 const uint16x8_t mults_f = vld1q_u16(&mults[40]);
2140 const uint16x8_t mults_g = vld1q_u16(&mults[48]);
2141 const uint16x8_t mults_h = vld1q_u16(&mults[56]);
2142 u32 s1 = adler & 0xFFFF;
2143 u32 s2 = adler >> 16;
2144  
2145
2146 if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
2147 do {
2148 s1 += *p++;
2149 s2 += s1;
2150 len--;
2151 } while ((uintptr_t)p & 15);
2152 s1 %= DIVISOR;
2153 s2 %= DIVISOR;
2154 }
2155  
2156 while (len) {
2157
2158 size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
2159  
2160 len -= n;
2161  
2162 if (n >= 64) {
2163 uint32x4_t v_s1 = vdupq_n_u32(0);
2164 uint32x4_t v_s2 = vdupq_n_u32(0);
2165
2166 uint16x8_t v_byte_sums_a = vdupq_n_u16(0);
2167 uint16x8_t v_byte_sums_b = vdupq_n_u16(0);
2168 uint16x8_t v_byte_sums_c = vdupq_n_u16(0);
2169 uint16x8_t v_byte_sums_d = vdupq_n_u16(0);
2170 uint16x8_t v_byte_sums_e = vdupq_n_u16(0);
2171 uint16x8_t v_byte_sums_f = vdupq_n_u16(0);
2172 uint16x8_t v_byte_sums_g = vdupq_n_u16(0);
2173 uint16x8_t v_byte_sums_h = vdupq_n_u16(0);
2174  
2175 s2 += s1 * (n & ~63);
2176  
2177 do {
2178
2179 const uint8x16_t data_a = vld1q_u8(p + 0);
2180 const uint8x16_t data_b = vld1q_u8(p + 16);
2181 const uint8x16_t data_c = vld1q_u8(p + 32);
2182 const uint8x16_t data_d = vld1q_u8(p + 48);
2183 uint16x8_t tmp;
2184  
2185
2186 v_s2 = vaddq_u32(v_s2, v_s1);
2187  
2188
2189 tmp = vpaddlq_u8(data_a);
2190 v_byte_sums_a = vaddw_u8(v_byte_sums_a,
2191 vget_low_u8(data_a));
2192 v_byte_sums_b = vaddw_u8(v_byte_sums_b,
2193 vget_high_u8(data_a));
2194 tmp = vpadalq_u8(tmp, data_b);
2195 v_byte_sums_c = vaddw_u8(v_byte_sums_c,
2196 vget_low_u8(data_b));
2197 v_byte_sums_d = vaddw_u8(v_byte_sums_d,
2198 vget_high_u8(data_b));
2199 tmp = vpadalq_u8(tmp, data_c);
2200 v_byte_sums_e = vaddw_u8(v_byte_sums_e,
2201 vget_low_u8(data_c));
2202 v_byte_sums_f = vaddw_u8(v_byte_sums_f,
2203 vget_high_u8(data_c));
2204 tmp = vpadalq_u8(tmp, data_d);
2205 v_byte_sums_g = vaddw_u8(v_byte_sums_g,
2206 vget_low_u8(data_d));
2207 v_byte_sums_h = vaddw_u8(v_byte_sums_h,
2208 vget_high_u8(data_d));
2209 v_s1 = vpadalq_u16(v_s1, tmp);
2210  
2211 p += 64;
2212 n -= 64;
2213 } while (n >= 64);
2214  
2215
2216 #ifdef ARCH_ARM32
2217 # define umlal2(a, b, c) vmlal_u16((a), vget_high_u16(b), vget_high_u16(c))
2218 #else
2219 # define umlal2 vmlal_high_u16
2220 #endif
2221 v_s2 = vqshlq_n_u32(v_s2, 6);
2222 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),
2223 vget_low_u16(mults_a));
2224 v_s2 = umlal2(v_s2, v_byte_sums_a, mults_a);
2225 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),
2226 vget_low_u16(mults_b));
2227 v_s2 = umlal2(v_s2, v_byte_sums_b, mults_b);
2228 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),
2229 vget_low_u16(mults_c));
2230 v_s2 = umlal2(v_s2, v_byte_sums_c, mults_c);
2231 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_d),
2232 vget_low_u16(mults_d));
2233 v_s2 = umlal2(v_s2, v_byte_sums_d, mults_d);
2234 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_e),
2235 vget_low_u16(mults_e));
2236 v_s2 = umlal2(v_s2, v_byte_sums_e, mults_e);
2237 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_f),
2238 vget_low_u16(mults_f));
2239 v_s2 = umlal2(v_s2, v_byte_sums_f, mults_f);
2240 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_g),
2241 vget_low_u16(mults_g));
2242 v_s2 = umlal2(v_s2, v_byte_sums_g, mults_g);
2243 v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_h),
2244 vget_low_u16(mults_h));
2245 v_s2 = umlal2(v_s2, v_byte_sums_h, mults_h);
2246 #undef umlal2
2247  
2248
2249 #ifdef ARCH_ARM32
2250 s1 += vgetq_lane_u32(v_s1, 0) + vgetq_lane_u32(v_s1, 1) +
2251 vgetq_lane_u32(v_s1, 2) + vgetq_lane_u32(v_s1, 3);
2252 s2 += vgetq_lane_u32(v_s2, 0) + vgetq_lane_u32(v_s2, 1) +
2253 vgetq_lane_u32(v_s2, 2) + vgetq_lane_u32(v_s2, 3);
2254 #else
2255 s1 += vaddvq_u32(v_s1);
2256 s2 += vaddvq_u32(v_s2);
2257 #endif
2258 }
2259
2260 ADLER32_CHUNK(s1, s2, p, n);
2261 }
2262 return (s2 << 16) | s1;
2263 }
2264 #undef ATTRIBUTES
2265 #endif
2266  
2267  
2268 #if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN() && \
2269 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD)
2270 # define adler32_arm_neon_dotprod adler32_arm_neon_dotprod
2271 # ifdef __clang__
2272 # define ATTRIBUTES _target_attribute("dotprod")
2273
2274 # elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
2275 || defined(__ARM_FEATURE_DOTPROD)
2276 # define ATTRIBUTES _target_attribute("+dotprod")
2277 # else
2278 # define ATTRIBUTES _target_attribute("arch=armv8.2-a+dotprod")
2279 # endif
2280 static ATTRIBUTES u32
2281 adler32_arm_neon_dotprod(u32 adler, const u8 *p, size_t len)
2282 {
2283 static const u8 _aligned_attribute(16) mults[64] = {
2284 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
2285 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
2286 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
2287 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
2288 };
2289 const uint8x16_t mults_a = vld1q_u8(&mults[0]);
2290 const uint8x16_t mults_b = vld1q_u8(&mults[16]);
2291 const uint8x16_t mults_c = vld1q_u8(&mults[32]);
2292 const uint8x16_t mults_d = vld1q_u8(&mults[48]);
2293 const uint8x16_t ones = vdupq_n_u8(1);
2294 u32 s1 = adler & 0xFFFF;
2295 u32 s2 = adler >> 16;
2296  
2297
2298 if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
2299 do {
2300 s1 += *p++;
2301 s2 += s1;
2302 len--;
2303 } while ((uintptr_t)p & 15);
2304 s1 %= DIVISOR;
2305 s2 %= DIVISOR;
2306 }
2307  
2308 while (len) {
2309
2310 size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
2311  
2312 len -= n;
2313  
2314 if (n >= 64) {
2315 uint32x4_t v_s1_a = vdupq_n_u32(0);
2316 uint32x4_t v_s1_b = vdupq_n_u32(0);
2317 uint32x4_t v_s1_c = vdupq_n_u32(0);
2318 uint32x4_t v_s1_d = vdupq_n_u32(0);
2319 uint32x4_t v_s2_a = vdupq_n_u32(0);
2320 uint32x4_t v_s2_b = vdupq_n_u32(0);
2321 uint32x4_t v_s2_c = vdupq_n_u32(0);
2322 uint32x4_t v_s2_d = vdupq_n_u32(0);
2323 uint32x4_t v_s1_sums_a = vdupq_n_u32(0);
2324 uint32x4_t v_s1_sums_b = vdupq_n_u32(0);
2325 uint32x4_t v_s1_sums_c = vdupq_n_u32(0);
2326 uint32x4_t v_s1_sums_d = vdupq_n_u32(0);
2327 uint32x4_t v_s1;
2328 uint32x4_t v_s2;
2329 uint32x4_t v_s1_sums;
2330  
2331 s2 += s1 * (n & ~63);
2332  
2333 do {
2334 uint8x16_t data_a = vld1q_u8(p + 0);
2335 uint8x16_t data_b = vld1q_u8(p + 16);
2336 uint8x16_t data_c = vld1q_u8(p + 32);
2337 uint8x16_t data_d = vld1q_u8(p + 48);
2338  
2339 v_s1_sums_a = vaddq_u32(v_s1_sums_a, v_s1_a);
2340 v_s1_a = vdotq_u32(v_s1_a, data_a, ones);
2341 v_s2_a = vdotq_u32(v_s2_a, data_a, mults_a);
2342  
2343 v_s1_sums_b = vaddq_u32(v_s1_sums_b, v_s1_b);
2344 v_s1_b = vdotq_u32(v_s1_b, data_b, ones);
2345 v_s2_b = vdotq_u32(v_s2_b, data_b, mults_b);
2346  
2347 v_s1_sums_c = vaddq_u32(v_s1_sums_c, v_s1_c);
2348 v_s1_c = vdotq_u32(v_s1_c, data_c, ones);
2349 v_s2_c = vdotq_u32(v_s2_c, data_c, mults_c);
2350  
2351 v_s1_sums_d = vaddq_u32(v_s1_sums_d, v_s1_d);
2352 v_s1_d = vdotq_u32(v_s1_d, data_d, ones);
2353 v_s2_d = vdotq_u32(v_s2_d, data_d, mults_d);
2354  
2355 p += 64;
2356 n -= 64;
2357 } while (n >= 64);
2358  
2359 v_s1 = vaddq_u32(vaddq_u32(v_s1_a, v_s1_b),
2360 vaddq_u32(v_s1_c, v_s1_d));
2361 v_s2 = vaddq_u32(vaddq_u32(v_s2_a, v_s2_b),
2362 vaddq_u32(v_s2_c, v_s2_d));
2363 v_s1_sums = vaddq_u32(vaddq_u32(v_s1_sums_a,
2364 v_s1_sums_b),
2365 vaddq_u32(v_s1_sums_c,
2366 v_s1_sums_d));
2367 v_s2 = vaddq_u32(v_s2, vqshlq_n_u32(v_s1_sums, 6));
2368  
2369 s1 += vaddvq_u32(v_s1);
2370 s2 += vaddvq_u32(v_s2);
2371 }
2372
2373 ADLER32_CHUNK(s1, s2, p, n);
2374 }
2375 return (s2 << 16) | s1;
2376 }
2377 #undef ATTRIBUTES
2378 #endif
2379  
2380 #if defined(adler32_arm_neon_dotprod) && defined(__ARM_FEATURE_DOTPROD)
2381 #define DEFAULT_IMPL adler32_arm_neon_dotprod
2382 #else
2383 static inline adler32_func_t
2384 arch_select_adler32_func(void)
2385 {
2386 const u32 features MAYBE_UNUSED = get_arm_cpu_features();
2387  
2388 #ifdef adler32_arm_neon_dotprod
2389 if (HAVE_NEON(features) && HAVE_DOTPROD(features))
2390 return adler32_arm_neon_dotprod;
2391 #endif
2392 #ifdef adler32_arm_neon
2393 if (HAVE_NEON(features))
2394 return adler32_arm_neon;
2395 #endif
2396 return NULL;
2397 }
2398 #define arch_select_adler32_func arch_select_adler32_func
2399 #endif
2400  
2401 #endif
2402  
2403 #elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
2404 /* # include "x86/adler32_impl.h" */
2405  
2406  
2407 #ifndef LIB_X86_ADLER32_IMPL_H
2408 #define LIB_X86_ADLER32_IMPL_H
2409  
2410 /* #include "x86-cpu_features.h" */
2411  
2412  
2413 #ifndef LIB_X86_CPU_FEATURES_H
2414 #define LIB_X86_CPU_FEATURES_H
2415  
2416 /* #include "lib_common.h" */
2417  
2418  
2419 #ifndef LIB_LIB_COMMON_H
2420 #define LIB_LIB_COMMON_H
2421  
2422 #ifdef LIBDEFLATE_H
2423
2424 # error "lib_common.h must always be included before libdeflate.h"
2425 #endif
2426  
2427 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
2428 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
2429 #elif defined(__GNUC__)
2430 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
2431 #else
2432 # define LIBDEFLATE_EXPORT_SYM
2433 #endif
2434  
2435  
2436 #if defined(__GNUC__) && defined(__i386__)
2437 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
2438 #else
2439 # define LIBDEFLATE_ALIGN_STACK
2440 #endif
2441  
2442 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
2443  
2444 /* #include "../common_defs.h" */
2445  
2446  
2447 #ifndef COMMON_DEFS_H
2448 #define COMMON_DEFS_H
2449  
2450 /* #include "libdeflate.h" */
2451  
2452  
2453 #ifndef LIBDEFLATE_H
2454 #define LIBDEFLATE_H
2455  
2456 #include <stddef.h>
2457 #include <stdint.h>
2458  
2459 #ifdef __cplusplus
2460 extern "C" {
2461 #endif
2462  
2463 #define LIBDEFLATE_VERSION_MAJOR 1
2464 #define LIBDEFLATE_VERSION_MINOR 26
2465 #define LIBDEFLATE_VERSION_STRING "1.26"
2466  
2467  
2468 #ifndef LIBDEFLATEAPI
2469 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
2470 # define LIBDEFLATEAPI __declspec(dllimport)
2471 # else
2472 # define LIBDEFLATEAPI
2473 # endif
2474 #endif
2475  
2476  
2477  
2478  
2479  
2480 struct libdeflate_compressor;
2481 struct libdeflate_options;
2482  
2483  
2484 LIBDEFLATEAPI struct libdeflate_compressor *
2485 libdeflate_alloc_compressor(int compression_level);
2486  
2487  
2488 LIBDEFLATEAPI struct libdeflate_compressor *
2489 libdeflate_alloc_compressor_ex(int compression_level,
2490 const struct libdeflate_options *options);
2491  
2492  
2493 LIBDEFLATEAPI size_t
2494 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
2495 const void *in, size_t in_nbytes,
2496 void *out, size_t out_nbytes_avail);
2497  
2498  
2499 LIBDEFLATEAPI size_t
2500 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
2501 size_t in_nbytes);
2502  
2503  
2504 LIBDEFLATEAPI size_t
2505 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
2506 const void *in, size_t in_nbytes,
2507 void *out, size_t out_nbytes_avail);
2508  
2509  
2510 LIBDEFLATEAPI size_t
2511 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
2512 size_t in_nbytes);
2513  
2514  
2515 LIBDEFLATEAPI size_t
2516 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
2517 const void *in, size_t in_nbytes,
2518 void *out, size_t out_nbytes_avail);
2519  
2520  
2521 LIBDEFLATEAPI size_t
2522 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
2523 size_t in_nbytes);
2524  
2525  
2526 LIBDEFLATEAPI void
2527 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
2528  
2529  
2530  
2531  
2532  
2533 struct libdeflate_decompressor;
2534 struct libdeflate_options;
2535  
2536  
2537 LIBDEFLATEAPI struct libdeflate_decompressor *
2538 libdeflate_alloc_decompressor(void);
2539  
2540  
2541 LIBDEFLATEAPI struct libdeflate_decompressor *
2542 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
2543  
2544  
2545 enum libdeflate_result {
2546
2547 LIBDEFLATE_SUCCESS = 0,
2548  
2549
2550 LIBDEFLATE_BAD_DATA = 1,
2551  
2552
2553 LIBDEFLATE_SHORT_OUTPUT = 2,
2554  
2555
2556 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
2557 };
2558  
2559  
2560 LIBDEFLATEAPI enum libdeflate_result
2561 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
2562 const void *in, size_t in_nbytes,
2563 void *out, size_t out_nbytes_avail,
2564 size_t *actual_out_nbytes_ret);
2565  
2566  
2567 LIBDEFLATEAPI enum libdeflate_result
2568 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
2569 const void *in, size_t in_nbytes,
2570 void *out, size_t out_nbytes_avail,
2571 size_t *actual_in_nbytes_ret,
2572 size_t *actual_out_nbytes_ret);
2573  
2574  
2575 LIBDEFLATEAPI enum libdeflate_result
2576 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
2577 const void *in, size_t in_nbytes,
2578 void *out, size_t out_nbytes_avail,
2579 size_t *actual_out_nbytes_ret);
2580  
2581  
2582 LIBDEFLATEAPI enum libdeflate_result
2583 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
2584 const void *in, size_t in_nbytes,
2585 void *out, size_t out_nbytes_avail,
2586 size_t *actual_in_nbytes_ret,
2587 size_t *actual_out_nbytes_ret);
2588  
2589  
2590 LIBDEFLATEAPI enum libdeflate_result
2591 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
2592 const void *in, size_t in_nbytes,
2593 void *out, size_t out_nbytes_avail,
2594 size_t *actual_out_nbytes_ret);
2595  
2596  
2597 LIBDEFLATEAPI enum libdeflate_result
2598 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
2599 const void *in, size_t in_nbytes,
2600 void *out, size_t out_nbytes_avail,
2601 size_t *actual_in_nbytes_ret,
2602 size_t *actual_out_nbytes_ret);
2603  
2604  
2605 LIBDEFLATEAPI void
2606 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
2607  
2608  
2609  
2610  
2611  
2612  
2613 LIBDEFLATEAPI uint32_t
2614 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
2615  
2616  
2617  
2618 LIBDEFLATEAPI uint32_t
2619 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
2620  
2621  
2622  
2623  
2624  
2625  
2626 LIBDEFLATEAPI void
2627 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
2628 void (*free_func)(void *));
2629  
2630  
2631 struct libdeflate_options {
2632  
2633
2634 size_t sizeof_options;
2635  
2636
2637 void *(*malloc_func)(size_t);
2638 void (*free_func)(void *);
2639 };
2640  
2641 #ifdef __cplusplus
2642 }
2643 #endif
2644  
2645 #endif
2646  
2647  
2648 #include <stdbool.h>
2649 #include <stddef.h>
2650 #include <stdint.h>
2651 #ifdef _MSC_VER
2652 # include <intrin.h>
2653 # include <stdlib.h>
2654
2655
2656 # pragma warning(disable : 4146)
2657
2658 # pragma warning(disable : 4018)
2659 # pragma warning(disable : 4244)
2660 # pragma warning(disable : 4267)
2661 # pragma warning(disable : 4310)
2662
2663 # pragma warning(disable : 4100)
2664 # pragma warning(disable : 4127)
2665 # pragma warning(disable : 4189)
2666 # pragma warning(disable : 4232)
2667 # pragma warning(disable : 4245)
2668 # pragma warning(disable : 4295)
2669 #endif
2670 #ifndef FREESTANDING
2671 # include <string.h>
2672 #endif
2673  
2674  
2675  
2676  
2677  
2678  
2679 #undef ARCH_X86_64
2680 #undef ARCH_X86_32
2681 #undef ARCH_ARM64
2682 #undef ARCH_ARM32
2683 #undef ARCH_RISCV
2684 #ifdef _MSC_VER
2685
2686 # if defined(_M_X64) && !defined(_M_ARM64EC)
2687 # define ARCH_X86_64
2688 # elif defined(_M_IX86)
2689 # define ARCH_X86_32
2690 # elif defined(_M_ARM64)
2691 # define ARCH_ARM64
2692 # elif defined(_M_ARM)
2693 # define ARCH_ARM32
2694 # endif
2695 #else
2696 # if defined(__x86_64__)
2697 # define ARCH_X86_64
2698 # elif defined(__i386__)
2699 # define ARCH_X86_32
2700 # elif defined(__aarch64__)
2701 # define ARCH_ARM64
2702 # elif defined(__arm__)
2703 # define ARCH_ARM32
2704 # elif defined(__riscv)
2705 # define ARCH_RISCV
2706 # endif
2707 #endif
2708  
2709  
2710  
2711  
2712  
2713  
2714 typedef uint8_t u8;
2715 typedef uint16_t u16;
2716 typedef uint32_t u32;
2717 typedef uint64_t u64;
2718 typedef int8_t s8;
2719 typedef int16_t s16;
2720 typedef int32_t s32;
2721 typedef int64_t s64;
2722  
2723  
2724 #ifdef _MSC_VER
2725 # ifdef _WIN64
2726 typedef long long ssize_t;
2727 # else
2728 typedef long ssize_t;
2729 # endif
2730 #endif
2731  
2732  
2733 typedef size_t machine_word_t;
2734  
2735  
2736 #define WORDBYTES ((int)sizeof(machine_word_t))
2737  
2738  
2739 #define WORDBITS (8 * WORDBYTES)
2740  
2741  
2742  
2743  
2744  
2745  
2746 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
2747 # define GCC_PREREQ(major, minor) \
2748 (__GNUC__ > (major) || \
2749 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
2750 # if !GCC_PREREQ(4, 9)
2751 # error "gcc versions older than 4.9 are no longer supported"
2752 # endif
2753 #else
2754 # define GCC_PREREQ(major, minor) 0
2755 #endif
2756 #ifdef __clang__
2757 # ifdef __apple_build_version__
2758 # define CLANG_PREREQ(major, minor, apple_version) \
2759 (__apple_build_version__ >= (apple_version))
2760 # else
2761 # define CLANG_PREREQ(major, minor, apple_version) \
2762 (__clang_major__ > (major) || \
2763 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
2764 # endif
2765 # if !CLANG_PREREQ(3, 9, 8000000)
2766 # error "clang versions older than 3.9 are no longer supported"
2767 # endif
2768 #else
2769 # define CLANG_PREREQ(major, minor, apple_version) 0
2770 #endif
2771 #ifdef _MSC_VER
2772 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
2773 # if !MSVC_PREREQ(1928)
2774 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
2775 # endif
2776 #else
2777 # define MSVC_PREREQ(version) 0
2778 #endif
2779  
2780  
2781 #ifndef __has_attribute
2782 # define __has_attribute(attribute) 0
2783 #endif
2784  
2785  
2786 #ifndef __has_builtin
2787 # define __has_builtin(builtin) 0
2788 #endif
2789  
2790  
2791 #ifdef _MSC_VER
2792 # define inline __inline
2793 #endif
2794  
2795  
2796 #if defined(__GNUC__) || __has_attribute(always_inline)
2797 # define forceinline inline __attribute__((always_inline))
2798 #elif defined(_MSC_VER)
2799 # define forceinline __forceinline
2800 #else
2801 # define forceinline inline
2802 #endif
2803  
2804  
2805 #if defined(__GNUC__) || __has_attribute(unused)
2806 # define MAYBE_UNUSED __attribute__((unused))
2807 #else
2808 # define MAYBE_UNUSED
2809 #endif
2810  
2811  
2812 #if defined(__GNUC__) || __has_attribute(noreturn)
2813 # define NORETURN __attribute__((noreturn))
2814 #else
2815 # define NORETURN
2816 #endif
2817  
2818  
2819 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
2820 # define likely(expr) __builtin_expect(!!(expr), 1)
2821 #else
2822 # define likely(expr) (expr)
2823 #endif
2824  
2825  
2826 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
2827 # define unlikely(expr) __builtin_expect(!!(expr), 0)
2828 #else
2829 # define unlikely(expr) (expr)
2830 #endif
2831  
2832  
2833 #undef prefetchr
2834 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
2835 # define prefetchr(addr) __builtin_prefetch((addr), 0)
2836 #elif defined(_MSC_VER)
2837 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
2838 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
2839 # elif defined(ARCH_ARM64)
2840 # define prefetchr(addr) __prefetch2((addr), 0x00 )
2841 # elif defined(ARCH_ARM32)
2842 # define prefetchr(addr) __prefetch(addr)
2843 # endif
2844 #endif
2845 #ifndef prefetchr
2846 # define prefetchr(addr)
2847 #endif
2848  
2849  
2850 #undef prefetchw
2851 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
2852 # define prefetchw(addr) __builtin_prefetch((addr), 1)
2853 #elif defined(_MSC_VER)
2854 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
2855 # define prefetchw(addr) _m_prefetchw(addr)
2856 # elif defined(ARCH_ARM64)
2857 # define prefetchw(addr) __prefetch2((addr), 0x10 )
2858 # elif defined(ARCH_ARM32)
2859 # define prefetchw(addr) __prefetchw(addr)
2860 # endif
2861 #endif
2862 #ifndef prefetchw
2863 # define prefetchw(addr)
2864 #endif
2865  
2866  
2867 #undef _aligned_attribute
2868 #if defined(__GNUC__) || __has_attribute(aligned)
2869 # define _aligned_attribute(n) __attribute__((aligned(n)))
2870 #elif defined(_MSC_VER)
2871 # define _aligned_attribute(n) __declspec(align(n))
2872 #endif
2873  
2874  
2875 #if defined(__GNUC__) || __has_attribute(target)
2876 # define _target_attribute(attrs) __attribute__((target(attrs)))
2877 #else
2878 # define _target_attribute(attrs)
2879 #endif
2880  
2881  
2882  
2883  
2884  
2885 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
2886 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
2887 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
2888 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
2889 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
2890 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
2891 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
2892  
2893  
2894  
2895  
2896  
2897  
2898 #if defined(__BYTE_ORDER__)
2899 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
2900 #elif defined(_MSC_VER)
2901 # define CPU_IS_LITTLE_ENDIAN() true
2902 #else
2903 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
2904 {
2905 union {
2906 u32 w;
2907 u8 b;
2908 } u;
2909  
2910 u.w = 1;
2911 return u.b;
2912 }
2913 #endif
2914  
2915  
2916 static forceinline u16 bswap16(u16 v)
2917 {
2918 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
2919 return __builtin_bswap16(v);
2920 #elif defined(_MSC_VER)
2921 return _byteswap_ushort(v);
2922 #else
2923 return (v << 8) | (v >> 8);
2924 #endif
2925 }
2926  
2927  
2928 static forceinline u32 bswap32(u32 v)
2929 {
2930 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
2931 return __builtin_bswap32(v);
2932 #elif defined(_MSC_VER)
2933 return _byteswap_ulong(v);
2934 #else
2935 return ((v & 0x000000FF) << 24) |
2936 ((v & 0x0000FF00) << 8) |
2937 ((v & 0x00FF0000) >> 8) |
2938 ((v & 0xFF000000) >> 24);
2939 #endif
2940 }
2941  
2942  
2943 static forceinline u64 bswap64(u64 v)
2944 {
2945 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
2946 return __builtin_bswap64(v);
2947 #elif defined(_MSC_VER)
2948 return _byteswap_uint64(v);
2949 #else
2950 return ((v & 0x00000000000000FF) << 56) |
2951 ((v & 0x000000000000FF00) << 40) |
2952 ((v & 0x0000000000FF0000) << 24) |
2953 ((v & 0x00000000FF000000) << 8) |
2954 ((v & 0x000000FF00000000) >> 8) |
2955 ((v & 0x0000FF0000000000) >> 24) |
2956 ((v & 0x00FF000000000000) >> 40) |
2957 ((v & 0xFF00000000000000) >> 56);
2958 #endif
2959 }
2960  
2961 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
2962 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
2963 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
2964 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
2965 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
2966 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
2967  
2968  
2969  
2970  
2971  
2972  
2973 #if (defined(__GNUC__) || defined(__clang__)) && \
2974 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
2975 defined(__ARM_FEATURE_UNALIGNED) || \
2976 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
2977 defined(__riscv_misaligned_fast) || \
2978 defined(__wasm__))
2979 # define UNALIGNED_ACCESS_IS_FAST 1
2980 #elif defined(_MSC_VER)
2981 # define UNALIGNED_ACCESS_IS_FAST 1
2982 #else
2983 # define UNALIGNED_ACCESS_IS_FAST 0
2984 #endif
2985  
2986  
2987  
2988 #ifdef FREESTANDING
2989 # define MEMCOPY __builtin_memcpy
2990 #else
2991 # define MEMCOPY memcpy
2992 #endif
2993  
2994  
2995  
2996 #define DEFINE_UNALIGNED_TYPE(type) \
2997 static forceinline type \
2998 load_##type##_unaligned(const void *p) \
2999 { \
3000 type v; \
3001 \
3002 MEMCOPY(&v, p, sizeof(v)); \
3003 return v; \
3004 } \
3005 \
3006 static forceinline void \
3007 store_##type##_unaligned(type v, void *p) \
3008 { \
3009 MEMCOPY(p, &v, sizeof(v)); \
3010 }
3011  
3012 DEFINE_UNALIGNED_TYPE(u16)
3013 DEFINE_UNALIGNED_TYPE(u32)
3014 DEFINE_UNALIGNED_TYPE(u64)
3015 DEFINE_UNALIGNED_TYPE(machine_word_t)
3016  
3017 #undef MEMCOPY
3018  
3019 #define load_word_unaligned load_machine_word_t_unaligned
3020 #define store_word_unaligned store_machine_word_t_unaligned
3021  
3022  
3023  
3024 static forceinline u16
3025 get_unaligned_le16(const u8 *p)
3026 {
3027 if (UNALIGNED_ACCESS_IS_FAST)
3028 return le16_bswap(load_u16_unaligned(p));
3029 else
3030 return ((u16)p[1] << 8) | p[0];
3031 }
3032  
3033 static forceinline u16
3034 get_unaligned_be16(const u8 *p)
3035 {
3036 if (UNALIGNED_ACCESS_IS_FAST)
3037 return be16_bswap(load_u16_unaligned(p));
3038 else
3039 return ((u16)p[0] << 8) | p[1];
3040 }
3041  
3042 static forceinline u32
3043 get_unaligned_le32(const u8 *p)
3044 {
3045 if (UNALIGNED_ACCESS_IS_FAST)
3046 return le32_bswap(load_u32_unaligned(p));
3047 else
3048 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
3049 ((u32)p[1] << 8) | p[0];
3050 }
3051  
3052 static forceinline u32
3053 get_unaligned_be32(const u8 *p)
3054 {
3055 if (UNALIGNED_ACCESS_IS_FAST)
3056 return be32_bswap(load_u32_unaligned(p));
3057 else
3058 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
3059 ((u32)p[2] << 8) | p[3];
3060 }
3061  
3062 static forceinline u64
3063 get_unaligned_le64(const u8 *p)
3064 {
3065 if (UNALIGNED_ACCESS_IS_FAST)
3066 return le64_bswap(load_u64_unaligned(p));
3067 else
3068 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
3069 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
3070 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
3071 ((u64)p[1] << 8) | p[0];
3072 }
3073  
3074 static forceinline machine_word_t
3075 get_unaligned_leword(const u8 *p)
3076 {
3077 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
3078 if (WORDBITS == 32)
3079 return get_unaligned_le32(p);
3080 else
3081 return get_unaligned_le64(p);
3082 }
3083  
3084  
3085  
3086 static forceinline void
3087 put_unaligned_le16(u16 v, u8 *p)
3088 {
3089 if (UNALIGNED_ACCESS_IS_FAST) {
3090 store_u16_unaligned(le16_bswap(v), p);
3091 } else {
3092 p[0] = (u8)(v >> 0);
3093 p[1] = (u8)(v >> 8);
3094 }
3095 }
3096  
3097 static forceinline void
3098 put_unaligned_be16(u16 v, u8 *p)
3099 {
3100 if (UNALIGNED_ACCESS_IS_FAST) {
3101 store_u16_unaligned(be16_bswap(v), p);
3102 } else {
3103 p[0] = (u8)(v >> 8);
3104 p[1] = (u8)(v >> 0);
3105 }
3106 }
3107  
3108 static forceinline void
3109 put_unaligned_le32(u32 v, u8 *p)
3110 {
3111 if (UNALIGNED_ACCESS_IS_FAST) {
3112 store_u32_unaligned(le32_bswap(v), p);
3113 } else {
3114 p[0] = (u8)(v >> 0);
3115 p[1] = (u8)(v >> 8);
3116 p[2] = (u8)(v >> 16);
3117 p[3] = (u8)(v >> 24);
3118 }
3119 }
3120  
3121 static forceinline void
3122 put_unaligned_be32(u32 v, u8 *p)
3123 {
3124 if (UNALIGNED_ACCESS_IS_FAST) {
3125 store_u32_unaligned(be32_bswap(v), p);
3126 } else {
3127 p[0] = (u8)(v >> 24);
3128 p[1] = (u8)(v >> 16);
3129 p[2] = (u8)(v >> 8);
3130 p[3] = (u8)(v >> 0);
3131 }
3132 }
3133  
3134 static forceinline void
3135 put_unaligned_le64(u64 v, u8 *p)
3136 {
3137 if (UNALIGNED_ACCESS_IS_FAST) {
3138 store_u64_unaligned(le64_bswap(v), p);
3139 } else {
3140 p[0] = (u8)(v >> 0);
3141 p[1] = (u8)(v >> 8);
3142 p[2] = (u8)(v >> 16);
3143 p[3] = (u8)(v >> 24);
3144 p[4] = (u8)(v >> 32);
3145 p[5] = (u8)(v >> 40);
3146 p[6] = (u8)(v >> 48);
3147 p[7] = (u8)(v >> 56);
3148 }
3149 }
3150  
3151 static forceinline void
3152 put_unaligned_leword(machine_word_t v, u8 *p)
3153 {
3154 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
3155 if (WORDBITS == 32)
3156 put_unaligned_le32(v, p);
3157 else
3158 put_unaligned_le64(v, p);
3159 }
3160  
3161  
3162  
3163  
3164  
3165  
3166  
3167 static forceinline unsigned
3168 bsr32(u32 v)
3169 {
3170 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
3171 return 31 - __builtin_clz(v);
3172 #elif defined(_MSC_VER)
3173 unsigned long i;
3174  
3175 _BitScanReverse(&i, v);
3176 return i;
3177 #else
3178 unsigned i = 0;
3179  
3180 while ((v >>= 1) != 0)
3181 i++;
3182 return i;
3183 #endif
3184 }
3185  
3186 static forceinline unsigned
3187 bsr64(u64 v)
3188 {
3189 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
3190 return 63 - __builtin_clzll(v);
3191 #elif defined(_MSC_VER) && defined(_WIN64)
3192 unsigned long i;
3193  
3194 _BitScanReverse64(&i, v);
3195 return i;
3196 #else
3197 unsigned i = 0;
3198  
3199 while ((v >>= 1) != 0)
3200 i++;
3201 return i;
3202 #endif
3203 }
3204  
3205 static forceinline unsigned
3206 bsrw(machine_word_t v)
3207 {
3208 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
3209 if (WORDBITS == 32)
3210 return bsr32(v);
3211 else
3212 return bsr64(v);
3213 }
3214  
3215  
3216  
3217 static forceinline unsigned
3218 bsf32(u32 v)
3219 {
3220 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
3221 return __builtin_ctz(v);
3222 #elif defined(_MSC_VER)
3223 unsigned long i;
3224  
3225 _BitScanForward(&i, v);
3226 return i;
3227 #else
3228 unsigned i = 0;
3229  
3230 for (; (v & 1) == 0; v >>= 1)
3231 i++;
3232 return i;
3233 #endif
3234 }
3235  
3236 static forceinline unsigned
3237 bsf64(u64 v)
3238 {
3239 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
3240 return __builtin_ctzll(v);
3241 #elif defined(_MSC_VER) && defined(_WIN64)
3242 unsigned long i;
3243  
3244 _BitScanForward64(&i, v);
3245 return i;
3246 #else
3247 unsigned i = 0;
3248  
3249 for (; (v & 1) == 0; v >>= 1)
3250 i++;
3251 return i;
3252 #endif
3253 }
3254  
3255 static forceinline unsigned
3256 bsfw(machine_word_t v)
3257 {
3258 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
3259 if (WORDBITS == 32)
3260 return bsf32(v);
3261 else
3262 return bsf64(v);
3263 }
3264  
3265  
3266 #undef rbit32
3267 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
3268 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
3269 static forceinline u32
3270 rbit32(u32 v)
3271 {
3272 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
3273 return v;
3274 }
3275 #define rbit32 rbit32
3276 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
3277 static forceinline u32
3278 rbit32(u32 v)
3279 {
3280 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
3281 return v;
3282 }
3283 #define rbit32 rbit32
3284 #endif
3285  
3286 #endif
3287  
3288  
3289 typedef void *(*malloc_func_t)(size_t);
3290 typedef void (*free_func_t)(void *);
3291  
3292 extern malloc_func_t libdeflate_default_malloc_func;
3293 extern free_func_t libdeflate_default_free_func;
3294  
3295 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
3296 size_t alignment, size_t size);
3297 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
3298  
3299 #ifdef FREESTANDING
3300  
3301 void *memset(void *s, int c, size_t n);
3302 #define memset(s, c, n) __builtin_memset((s), (c), (n))
3303  
3304 void *memcpy(void *dest, const void *src, size_t n);
3305 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
3306  
3307 void *memmove(void *dest, const void *src, size_t n);
3308 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
3309  
3310 int memcmp(const void *s1, const void *s2, size_t n);
3311 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
3312  
3313 #undef LIBDEFLATE_ENABLE_ASSERTIONS
3314 #else
3315 # include <string.h>
3316
3317 # ifdef __clang_analyzer__
3318 # define LIBDEFLATE_ENABLE_ASSERTIONS
3319 # endif
3320 #endif
3321  
3322  
3323 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
3324 NORETURN void
3325 libdeflate_assertion_failed(const char *expr, const char *file, int line);
3326 #define ASSERT(expr) { if (unlikely(!(expr))) \
3327 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
3328 #else
3329 #define ASSERT(expr) (void)(expr)
3330 #endif
3331  
3332 #define CONCAT_IMPL(a, b) a##b
3333 #define CONCAT(a, b) CONCAT_IMPL(a, b)
3334 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
3335  
3336 #endif
3337  
3338  
3339 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
3340  
3341 #define X86_CPU_FEATURE_SSE2 (1 << 0)
3342 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
3343 #define X86_CPU_FEATURE_AVX (1 << 2)
3344 #define X86_CPU_FEATURE_AVX2 (1 << 3)
3345 #define X86_CPU_FEATURE_BMI2 (1 << 4)
3346  
3347 #define X86_CPU_FEATURE_ZMM (1 << 5)
3348 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
3349 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
3350 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
3351 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
3352 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
3353  
3354 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
3355  
3356 # define X86_CPU_FEATURES_KNOWN (1U << 31)
3357 extern volatile u32 libdeflate_x86_cpu_features;
3358  
3359 void libdeflate_init_x86_cpu_features(void);
3360  
3361 5 static inline u32 get_x86_cpu_features(void)
3362 {
3363 5 ▶ if (libdeflate_x86_cpu_features == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (libdeflate_x86_cpu_features == 0) 2 3 5
3364 2 libdeflate_init_x86_cpu_features();
3365 5 return libdeflate_x86_cpu_features;
3366 }
3367  
3368 # include <immintrin.h>
3369 # if defined(_MSC_VER) && defined(__clang__)
3370 # include <tmmintrin.h>
3371 # include <smmintrin.h>
3372 # include <wmmintrin.h>
3373 # include <avxintrin.h>
3374 # include <avx2intrin.h>
3375 # include <avx512fintrin.h>
3376 # include <avx512bwintrin.h>
3377 # include <avx512vlintrin.h>
3378 # if __has_include(<avx512vlbwintrin.h>)
3379 # include <avx512vlbwintrin.h>
3380 # endif
3381 # if __has_include(<vpclmulqdqintrin.h>)
3382 # include <vpclmulqdqintrin.h>
3383 # endif
3384 # if __has_include(<avx512vnniintrin.h>)
3385 # include <avx512vnniintrin.h>
3386 # endif
3387 # if __has_include(<avx512vlvnniintrin.h>)
3388 # include <avx512vlvnniintrin.h>
3389 # endif
3390 # if __has_include(<avxvnniintrin.h>)
3391 # include <avxvnniintrin.h>
3392 # endif
3393 # endif
3394 #else
3395 static inline u32 get_x86_cpu_features(void) { return 0; }
3396 #endif
3397  
3398 #if defined(__SSE2__) || \
3399 (defined(_MSC_VER) && \
3400 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
3401 # define HAVE_SSE2(features) 1
3402 # define HAVE_SSE2_NATIVE 1
3403 #else
3404 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
3405 # define HAVE_SSE2_NATIVE 0
3406 #endif
3407  
3408 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
3409 (defined(_MSC_VER) && defined(__AVX2__))
3410 # define HAVE_PCLMULQDQ(features) 1
3411 #else
3412 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
3413 #endif
3414  
3415 #ifdef __AVX__
3416 # define HAVE_AVX(features) 1
3417 #else
3418 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
3419 #endif
3420  
3421 #ifdef __AVX2__
3422 # define HAVE_AVX2(features) 1
3423 #else
3424 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
3425 #endif
3426  
3427 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
3428 # define HAVE_BMI2(features) 1
3429 # define HAVE_BMI2_NATIVE 1
3430 #else
3431 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
3432 # define HAVE_BMI2_NATIVE 0
3433 #endif
3434  
3435 #ifdef __AVX512BW__
3436 # define HAVE_AVX512BW(features) 1
3437 #else
3438 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
3439 #endif
3440  
3441 #ifdef __AVX512VL__
3442 # define HAVE_AVX512VL(features) 1
3443 #else
3444 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
3445 #endif
3446  
3447 #ifdef __VPCLMULQDQ__
3448 # define HAVE_VPCLMULQDQ(features) 1
3449 #else
3450 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
3451 #endif
3452  
3453 #ifdef __AVX512VNNI__
3454 # define HAVE_AVX512VNNI(features) 1
3455 #else
3456 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
3457 #endif
3458  
3459 #ifdef __AVXVNNI__
3460 # define HAVE_AVXVNNI(features) 1
3461 #else
3462 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
3463 #endif
3464  
3465 #endif
3466  
3467 #endif
3468  
3469  
3470  
3471 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
3472 # define adler32_x86_sse2 adler32_x86_sse2
3473 # define SUFFIX _sse2
3474 # define ATTRIBUTES _target_attribute("sse2")
3475 # define VL 16
3476 # define USE_VNNI 0
3477 # define USE_AVX512 0
3478 /* #include "x86-adler32_template.h" */
3479  
3480  
3481  
3482  
3483 #if VL == 16
3484 # define vec_t __m128i
3485 # define mask_t u16
3486 # define LOG2_VL 4
3487 # define VADD8(a, b) _mm_add_epi8((a), (b))
3488 # define VADD16(a, b) _mm_add_epi16((a), (b))
3489 # define VADD32(a, b) _mm_add_epi32((a), (b))
3490 # if USE_AVX512
3491 # define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
3492 # else
3493 # define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
3494 # endif
3495 # define VLOAD(p) _mm_load_si128((const void *)(p))
3496 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
3497 # define VMADD16(a, b) _mm_madd_epi16((a), (b))
3498 # define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
3499 # define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
3500 # define VSAD8(a, b) _mm_sad_epu8((a), (b))
3501 # define VSET1_8(a) _mm_set1_epi8(a)
3502 # define VSET1_32(a) _mm_set1_epi32(a)
3503 # define VSETZERO() _mm_setzero_si128()
3504 # define VSLL32(a, b) _mm_slli_epi32((a), (b))
3505 # define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
3506 # define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
3507 #elif VL == 32
3508 # define vec_t __m256i
3509 # define mask_t u32
3510 # define LOG2_VL 5
3511 # define VADD8(a, b) _mm256_add_epi8((a), (b))
3512 # define VADD16(a, b) _mm256_add_epi16((a), (b))
3513 # define VADD32(a, b) _mm256_add_epi32((a), (b))
3514 # if USE_AVX512
3515 # define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
3516 # else
3517 # define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
3518 # endif
3519 # define VLOAD(p) _mm256_load_si256((const void *)(p))
3520 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
3521 # define VMADD16(a, b) _mm256_madd_epi16((a), (b))
3522 # define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
3523 # define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
3524 # define VSAD8(a, b) _mm256_sad_epu8((a), (b))
3525 # define VSET1_8(a) _mm256_set1_epi8(a)
3526 # define VSET1_32(a) _mm256_set1_epi32(a)
3527 # define VSETZERO() _mm256_setzero_si256()
3528 # define VSLL32(a, b) _mm256_slli_epi32((a), (b))
3529 # define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
3530 # define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
3531 #elif VL == 64
3532 # define vec_t __m512i
3533 # define mask_t u64
3534 # define LOG2_VL 6
3535 # define VADD8(a, b) _mm512_add_epi8((a), (b))
3536 # define VADD16(a, b) _mm512_add_epi16((a), (b))
3537 # define VADD32(a, b) _mm512_add_epi32((a), (b))
3538 # define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
3539 # define VLOAD(p) _mm512_load_si512((const void *)(p))
3540 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
3541 # define VMADD16(a, b) _mm512_madd_epi16((a), (b))
3542 # define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
3543 # define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
3544 # define VSAD8(a, b) _mm512_sad_epu8((a), (b))
3545 # define VSET1_8(a) _mm512_set1_epi8(a)
3546 # define VSET1_32(a) _mm512_set1_epi32(a)
3547 # define VSETZERO() _mm512_setzero_si512()
3548 # define VSLL32(a, b) _mm512_slli_epi32((a), (b))
3549 # define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
3550 # define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
3551 #else
3552 # error "unsupported vector length"
3553 #endif
3554  
3555 #define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
3556 #define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
3557 #define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
3558 #define VADD32_7X(a, b, c, d, e, f, g) \
3559 VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
3560  
3561  
3562 #undef reduce_to_32bits
3563 static forceinline ATTRIBUTES void
3564 ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
3565 {
3566 __m128i v_s1_128, v_s2_128;
3567 #if VL == 16
3568 {
3569 0 v_s1_128 = v_s1;
3570 0 v_s2_128 = v_s2;
3571 }
3572 #else
3573 {
3574 __m256i v_s1_256, v_s2_256;
3575 #if VL == 32
3576 v_s1_256 = v_s1;
3577 v_s2_256 = v_s2;
3578 #else
3579
3580 v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
3581 _mm512_extracti64x4_epi64(v_s1, 1));
3582 v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
3583 _mm512_extracti64x4_epi64(v_s2, 1));
3584 #endif
3585
3586 v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
3587 _mm256_extracti128_si256(v_s1_256, 1));
3588 v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
3589 _mm256_extracti128_si256(v_s2_256, 1));
3590 }
3591 #endif
3592  
3593
3594 #if USE_VNNI
3595 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
3596 #endif
3597 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
3598 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
3599 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
3600  
3601 0 *s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
3602 0 *s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
3603 0 }
3604 #define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
3605  
3606 static ATTRIBUTES u32
3607 0 ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
3608 {
3609 #if USE_VNNI
3610
3611 static const u8 _aligned_attribute(VL) raw_mults[VL] = {
3612 #if VL == 64
3613 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
3614 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
3615 #endif
3616 #if VL >= 32
3617 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
3618 #endif
3619 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
3620 };
3621 const vec_t ones = VSET1_8(1);
3622 #else
3623
3624 static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
3625 #if VL == 16
3626 { 32, 31, 30, 29, 28, 27, 26, 25 },
3627 { 24, 23, 22, 21, 20, 19, 18, 17 },
3628 { 16, 15, 14, 13, 12, 11, 10, 9 },
3629 { 8, 7, 6, 5, 4, 3, 2, 1 },
3630 #elif VL == 32
3631 { 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
3632 { 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
3633 { 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
3634 { 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
3635 #else
3636 # error "unsupported parameters"
3637 #endif
3638 };
3639 0 const vec_t mults_a = VLOAD(raw_mults[0]);
3640 0 const vec_t mults_b = VLOAD(raw_mults[1]);
3641 0 const vec_t mults_c = VLOAD(raw_mults[2]);
3642 0 const vec_t mults_d = VLOAD(raw_mults[3]);
3643 #endif
3644 0 const vec_t zeroes = VSETZERO();
3645 0 u32 s1 = adler & 0xFFFF;
3646 0 u32 s2 = adler >> 16;
3647  
3648
3649 0 ▶ if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
Branch: 2 branches
BranchTrueFalseTotal
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
3650 do {
3651 0 s1 += *p++;
3652 0 s2 += s1;
3653 0 len--;
3654 0 ▶ } while ((uintptr_t)p & (VL-1));
Branch: 1 branch
BranchTrueFalseTotal
} while ((uintptr_t)p & (VL-1)); 0 0 0
3655 0 s1 %= DIVISOR;
3656 0 s2 %= DIVISOR;
3657 }
3658  
3659 #if USE_VNNI
3660
3661 while (len) {
3662
3663 size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
3664 vec_t mults = VLOAD(raw_mults);
3665 vec_t v_s1 = zeroes;
3666 vec_t v_s2 = zeroes;
3667  
3668 s2 += s1 * n;
3669 len -= n;
3670  
3671 if (n >= 4*VL) {
3672 vec_t v_s1_b = zeroes;
3673 vec_t v_s1_c = zeroes;
3674 vec_t v_s1_d = zeroes;
3675 vec_t v_s2_b = zeroes;
3676 vec_t v_s2_c = zeroes;
3677 vec_t v_s2_d = zeroes;
3678 vec_t v_s1_sums = zeroes;
3679 vec_t v_s1_sums_b = zeroes;
3680 vec_t v_s1_sums_c = zeroes;
3681 vec_t v_s1_sums_d = zeroes;
3682 vec_t tmp0, tmp1;
3683  
3684 do {
3685 vec_t data_a = VLOADU(p + 0*VL);
3686 vec_t data_b = VLOADU(p + 1*VL);
3687 vec_t data_c = VLOADU(p + 2*VL);
3688 vec_t data_d = VLOADU(p + 3*VL);
3689  
3690
3691 #if GCC_PREREQ(1, 0)
3692 __asm__("" : "+v" (data_a), "+v" (data_b),
3693 "+v" (data_c), "+v" (data_d));
3694 #endif
3695  
3696 v_s2 = VDPBUSD(v_s2, data_a, mults);
3697 v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
3698 v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
3699 v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
3700  
3701 v_s1_sums = VADD32(v_s1_sums, v_s1);
3702 v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
3703 v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
3704 v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
3705  
3706 v_s1 = VDPBUSD(v_s1, data_a, ones);
3707 v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
3708 v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
3709 v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
3710  
3711
3712 #if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
3713 __asm__("" : "+v" (v_s2), "+v" (v_s2_b),
3714 "+v" (v_s2_c), "+v" (v_s2_d),
3715 "+v" (v_s1_sums),
3716 "+v" (v_s1_sums_b),
3717 "+v" (v_s1_sums_c),
3718 "+v" (v_s1_sums_d),
3719 "+v" (v_s1), "+v" (v_s1_b),
3720 "+v" (v_s1_c), "+v" (v_s1_d));
3721 #endif
3722 p += 4*VL;
3723 n -= 4*VL;
3724 } while (n >= 4*VL);
3725  
3726
3727 tmp0 = VADD32(v_s1, v_s1_b);
3728 tmp1 = VADD32(v_s1, v_s1_c);
3729 v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
3730 v_s1_sums_c, v_s1_sums_d);
3731 v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
3732 v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
3733 VSLL32(tmp0, LOG2_VL + 1),
3734 VSLL32(tmp1, LOG2_VL),
3735 v_s2, v_s2_b, v_s2_c, v_s2_d);
3736 }
3737  
3738
3739 if (n >= 2*VL) {
3740 const vec_t data_a = VLOADU(p + 0*VL);
3741 const vec_t data_b = VLOADU(p + 1*VL);
3742  
3743 v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
3744 v_s1 = VDPBUSD(v_s1, data_a, ones);
3745 v_s1 = VDPBUSD(v_s1, data_b, ones);
3746 v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
3747 v_s2 = VDPBUSD(v_s2, data_a, mults);
3748 v_s2 = VDPBUSD(v_s2, data_b, mults);
3749 p += 2*VL;
3750 n -= 2*VL;
3751 }
3752 if (n) {
3753
3754 vec_t data;
3755  
3756 v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
3757  
3758 mults = VADD8(mults, VSET1_8((int)n - VL));
3759 if (n > VL) {
3760 data = VLOADU(p);
3761 v_s1 = VDPBUSD(v_s1, data, ones);
3762 v_s2 = VDPBUSD(v_s2, data, mults);
3763 p += VL;
3764 n -= VL;
3765 mults = VADD8(mults, VSET1_8(-VL));
3766 }
3767
3768 #if USE_AVX512
3769 data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
3770 #else
3771 data = zeroes;
3772 memcpy(&data, p, n);
3773 #endif
3774 v_s1 = VDPBUSD(v_s1, data, ones);
3775 v_s2 = VDPBUSD(v_s2, data, mults);
3776 p += n;
3777 }
3778  
3779 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
3780 s1 %= DIVISOR;
3781 s2 %= DIVISOR;
3782 }
3783 #else
3784
3785 0 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 0 0 0
3786
3787 0 size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
3788 MAX_CHUNK_LEN) & ~(2*VL - 1));
3789 0 len -= n;
3790  
3791 0 ▶ if (n >= 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 2*VL) { 0 0 0
3792 0 vec_t v_s1 = zeroes;
3793 0 vec_t v_s1_sums = zeroes;
3794 0 vec_t v_byte_sums_a = zeroes;
3795 0 vec_t v_byte_sums_b = zeroes;
3796 0 vec_t v_byte_sums_c = zeroes;
3797 0 vec_t v_byte_sums_d = zeroes;
3798 vec_t v_s2;
3799  
3800 0 s2 += s1 * (n & ~(2*VL - 1));
3801  
3802 do {
3803 0 vec_t data_a = VLOADU(p + 0*VL);
3804 0 vec_t data_b = VLOADU(p + 1*VL);
3805  
3806 0 v_s1_sums = VADD32(v_s1_sums, v_s1);
3807 0 v_byte_sums_a = VADD16(v_byte_sums_a,
3808 VUNPACKLO8(data_a, zeroes));
3809 0 v_byte_sums_b = VADD16(v_byte_sums_b,
3810 VUNPACKHI8(data_a, zeroes));
3811 0 v_byte_sums_c = VADD16(v_byte_sums_c,
3812 VUNPACKLO8(data_b, zeroes));
3813 0 v_byte_sums_d = VADD16(v_byte_sums_d,
3814 VUNPACKHI8(data_b, zeroes));
3815 0 v_s1 = VADD32(v_s1,
3816 VADD32(VSAD8(data_a, zeroes),
3817 VSAD8(data_b, zeroes)));
3818
3819 #if GCC_PREREQ(1, 0)
3820 0 __asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
3821 "+x" (v_byte_sums_a),
3822 "+x" (v_byte_sums_b),
3823 "+x" (v_byte_sums_c),
3824 "+x" (v_byte_sums_d));
3825 #endif
3826 0 p += 2*VL;
3827 0 n -= 2*VL;
3828 0 ▶ } while (n >= 2*VL);
Branch: 1 branch
BranchTrueFalseTotal
} while (n >= 2*VL); 0 0 0
3829  
3830
3831 0 v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
3832 VMADD16(v_byte_sums_a, mults_a),
3833 VMADD16(v_byte_sums_b, mults_b),
3834 VMADD16(v_byte_sums_c, mults_c),
3835 VMADD16(v_byte_sums_d, mults_d));
3836 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
3837 }
3838
3839 0 ▶ ADLER32_CHUNK(s1, s2, p, n);
Branch: 3 branches
BranchTrueFalseTotal
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
ADLER32_CHUNK(s1, s2, p, n); 0 0 0
3840 }
3841 #endif
3842 0 return (s2 << 16) | s1;
3843 }
3844  
3845 #undef vec_t
3846 #undef mask_t
3847 #undef LOG2_VL
3848 #undef VADD8
3849 #undef VADD16
3850 #undef VADD32
3851 #undef VDPBUSD
3852 #undef VLOAD
3853 #undef VLOADU
3854 #undef VMADD16
3855 #undef VMASKZ_LOADU
3856 #undef VMULLO32
3857 #undef VSAD8
3858 #undef VSET1_8
3859 #undef VSET1_32
3860 #undef VSETZERO
3861 #undef VSLL32
3862 #undef VUNPACKLO8
3863 #undef VUNPACKHI8
3864  
3865 #undef SUFFIX
3866 #undef ATTRIBUTES
3867 #undef VL
3868 #undef USE_VNNI
3869 #undef USE_AVX512
3870  
3871  
3872 # define adler32_x86_avx2 adler32_x86_avx2
3873 # define SUFFIX _avx2
3874 # define ATTRIBUTES _target_attribute("avx2")
3875 # define VL 32
3876 # define USE_VNNI 0
3877 # define USE_AVX512 0
3878 /* #include "x86-adler32_template.h" */
3879  
3880  
3881  
3882  
3883 #if VL == 16
3884 # define vec_t __m128i
3885 # define mask_t u16
3886 # define LOG2_VL 4
3887 # define VADD8(a, b) _mm_add_epi8((a), (b))
3888 # define VADD16(a, b) _mm_add_epi16((a), (b))
3889 # define VADD32(a, b) _mm_add_epi32((a), (b))
3890 # if USE_AVX512
3891 # define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
3892 # else
3893 # define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
3894 # endif
3895 # define VLOAD(p) _mm_load_si128((const void *)(p))
3896 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
3897 # define VMADD16(a, b) _mm_madd_epi16((a), (b))
3898 # define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
3899 # define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
3900 # define VSAD8(a, b) _mm_sad_epu8((a), (b))
3901 # define VSET1_8(a) _mm_set1_epi8(a)
3902 # define VSET1_32(a) _mm_set1_epi32(a)
3903 # define VSETZERO() _mm_setzero_si128()
3904 # define VSLL32(a, b) _mm_slli_epi32((a), (b))
3905 # define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
3906 # define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
3907 #elif VL == 32
3908 # define vec_t __m256i
3909 # define mask_t u32
3910 # define LOG2_VL 5
3911 # define VADD8(a, b) _mm256_add_epi8((a), (b))
3912 # define VADD16(a, b) _mm256_add_epi16((a), (b))
3913 # define VADD32(a, b) _mm256_add_epi32((a), (b))
3914 # if USE_AVX512
3915 # define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
3916 # else
3917 # define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
3918 # endif
3919 # define VLOAD(p) _mm256_load_si256((const void *)(p))
3920 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
3921 # define VMADD16(a, b) _mm256_madd_epi16((a), (b))
3922 # define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
3923 # define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
3924 # define VSAD8(a, b) _mm256_sad_epu8((a), (b))
3925 # define VSET1_8(a) _mm256_set1_epi8(a)
3926 # define VSET1_32(a) _mm256_set1_epi32(a)
3927 # define VSETZERO() _mm256_setzero_si256()
3928 # define VSLL32(a, b) _mm256_slli_epi32((a), (b))
3929 # define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
3930 # define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
3931 #elif VL == 64
3932 # define vec_t __m512i
3933 # define mask_t u64
3934 # define LOG2_VL 6
3935 # define VADD8(a, b) _mm512_add_epi8((a), (b))
3936 # define VADD16(a, b) _mm512_add_epi16((a), (b))
3937 # define VADD32(a, b) _mm512_add_epi32((a), (b))
3938 # define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
3939 # define VLOAD(p) _mm512_load_si512((const void *)(p))
3940 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
3941 # define VMADD16(a, b) _mm512_madd_epi16((a), (b))
3942 # define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
3943 # define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
3944 # define VSAD8(a, b) _mm512_sad_epu8((a), (b))
3945 # define VSET1_8(a) _mm512_set1_epi8(a)
3946 # define VSET1_32(a) _mm512_set1_epi32(a)
3947 # define VSETZERO() _mm512_setzero_si512()
3948 # define VSLL32(a, b) _mm512_slli_epi32((a), (b))
3949 # define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
3950 # define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
3951 #else
3952 # error "unsupported vector length"
3953 #endif
3954  
3955 #define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
3956 #define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
3957 #define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
3958 #define VADD32_7X(a, b, c, d, e, f, g) \
3959 VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
3960  
3961  
3962 #undef reduce_to_32bits
3963 static forceinline ATTRIBUTES void
3964 ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
3965 {
3966 __m128i v_s1_128, v_s2_128;
3967 #if VL == 16
3968 {
3969 v_s1_128 = v_s1;
3970 v_s2_128 = v_s2;
3971 }
3972 #else
3973 {
3974 __m256i v_s1_256, v_s2_256;
3975 #if VL == 32
3976 24 v_s1_256 = v_s1;
3977 24 v_s2_256 = v_s2;
3978 #else
3979
3980 v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
3981 _mm512_extracti64x4_epi64(v_s1, 1));
3982 v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
3983 _mm512_extracti64x4_epi64(v_s2, 1));
3984 #endif
3985
3986 48 v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
3987 24 _mm256_extracti128_si256(v_s1_256, 1));
3988 48 v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
3989 24 _mm256_extracti128_si256(v_s2_256, 1));
3990 }
3991 #endif
3992  
3993
3994 #if USE_VNNI
3995 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
3996 #endif
3997 24 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
3998 24 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
3999 48 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
4000  
4001 24 *s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
4002 24 *s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
4003 24 }
4004 #define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
4005  
4006 static ATTRIBUTES u32
4007 24 ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
4008 {
4009 #if USE_VNNI
4010
4011 static const u8 _aligned_attribute(VL) raw_mults[VL] = {
4012 #if VL == 64
4013 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
4014 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
4015 #endif
4016 #if VL >= 32
4017 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
4018 #endif
4019 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
4020 };
4021 const vec_t ones = VSET1_8(1);
4022 #else
4023
4024 static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
4025 #if VL == 16
4026 { 32, 31, 30, 29, 28, 27, 26, 25 },
4027 { 24, 23, 22, 21, 20, 19, 18, 17 },
4028 { 16, 15, 14, 13, 12, 11, 10, 9 },
4029 { 8, 7, 6, 5, 4, 3, 2, 1 },
4030 #elif VL == 32
4031 { 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
4032 { 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
4033 { 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
4034 { 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
4035 #else
4036 # error "unsupported parameters"
4037 #endif
4038 };
4039 24 const vec_t mults_a = VLOAD(raw_mults[0]);
4040 24 const vec_t mults_b = VLOAD(raw_mults[1]);
4041 24 const vec_t mults_c = VLOAD(raw_mults[2]);
4042 24 const vec_t mults_d = VLOAD(raw_mults[3]);
4043 #endif
4044 24 const vec_t zeroes = VSETZERO();
4045 24 u32 s1 = adler & 0xFFFF;
4046 24 u32 s2 = adler >> 16;
4047  
4048
4049 24 ▶ if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
Branch: 2 branches
BranchTrueFalseTotal
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 24 24
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
4050 do {
4051 0 s1 += *p++;
4052 0 s2 += s1;
4053 0 len--;
4054 0 ▶ } while ((uintptr_t)p & (VL-1));
Branch: 1 branch
BranchTrueFalseTotal
} while ((uintptr_t)p & (VL-1)); 0 0 0
4055 0 s1 %= DIVISOR;
4056 0 s2 %= DIVISOR;
4057 }
4058  
4059 #if USE_VNNI
4060
4061 while (len) {
4062
4063 size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
4064 vec_t mults = VLOAD(raw_mults);
4065 vec_t v_s1 = zeroes;
4066 vec_t v_s2 = zeroes;
4067  
4068 s2 += s1 * n;
4069 len -= n;
4070  
4071 if (n >= 4*VL) {
4072 vec_t v_s1_b = zeroes;
4073 vec_t v_s1_c = zeroes;
4074 vec_t v_s1_d = zeroes;
4075 vec_t v_s2_b = zeroes;
4076 vec_t v_s2_c = zeroes;
4077 vec_t v_s2_d = zeroes;
4078 vec_t v_s1_sums = zeroes;
4079 vec_t v_s1_sums_b = zeroes;
4080 vec_t v_s1_sums_c = zeroes;
4081 vec_t v_s1_sums_d = zeroes;
4082 vec_t tmp0, tmp1;
4083  
4084 do {
4085 vec_t data_a = VLOADU(p + 0*VL);
4086 vec_t data_b = VLOADU(p + 1*VL);
4087 vec_t data_c = VLOADU(p + 2*VL);
4088 vec_t data_d = VLOADU(p + 3*VL);
4089  
4090
4091 #if GCC_PREREQ(1, 0)
4092 __asm__("" : "+v" (data_a), "+v" (data_b),
4093 "+v" (data_c), "+v" (data_d));
4094 #endif
4095  
4096 v_s2 = VDPBUSD(v_s2, data_a, mults);
4097 v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
4098 v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
4099 v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
4100  
4101 v_s1_sums = VADD32(v_s1_sums, v_s1);
4102 v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
4103 v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
4104 v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
4105  
4106 v_s1 = VDPBUSD(v_s1, data_a, ones);
4107 v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
4108 v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
4109 v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
4110  
4111
4112 #if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
4113 __asm__("" : "+v" (v_s2), "+v" (v_s2_b),
4114 "+v" (v_s2_c), "+v" (v_s2_d),
4115 "+v" (v_s1_sums),
4116 "+v" (v_s1_sums_b),
4117 "+v" (v_s1_sums_c),
4118 "+v" (v_s1_sums_d),
4119 "+v" (v_s1), "+v" (v_s1_b),
4120 "+v" (v_s1_c), "+v" (v_s1_d));
4121 #endif
4122 p += 4*VL;
4123 n -= 4*VL;
4124 } while (n >= 4*VL);
4125  
4126
4127 tmp0 = VADD32(v_s1, v_s1_b);
4128 tmp1 = VADD32(v_s1, v_s1_c);
4129 v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
4130 v_s1_sums_c, v_s1_sums_d);
4131 v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
4132 v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
4133 VSLL32(tmp0, LOG2_VL + 1),
4134 VSLL32(tmp1, LOG2_VL),
4135 v_s2, v_s2_b, v_s2_c, v_s2_d);
4136 }
4137  
4138
4139 if (n >= 2*VL) {
4140 const vec_t data_a = VLOADU(p + 0*VL);
4141 const vec_t data_b = VLOADU(p + 1*VL);
4142  
4143 v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
4144 v_s1 = VDPBUSD(v_s1, data_a, ones);
4145 v_s1 = VDPBUSD(v_s1, data_b, ones);
4146 v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
4147 v_s2 = VDPBUSD(v_s2, data_a, mults);
4148 v_s2 = VDPBUSD(v_s2, data_b, mults);
4149 p += 2*VL;
4150 n -= 2*VL;
4151 }
4152 if (n) {
4153
4154 vec_t data;
4155  
4156 v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
4157  
4158 mults = VADD8(mults, VSET1_8((int)n - VL));
4159 if (n > VL) {
4160 data = VLOADU(p);
4161 v_s1 = VDPBUSD(v_s1, data, ones);
4162 v_s2 = VDPBUSD(v_s2, data, mults);
4163 p += VL;
4164 n -= VL;
4165 mults = VADD8(mults, VSET1_8(-VL));
4166 }
4167
4168 #if USE_AVX512
4169 data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
4170 #else
4171 data = zeroes;
4172 memcpy(&data, p, n);
4173 #endif
4174 v_s1 = VDPBUSD(v_s1, data, ones);
4175 v_s2 = VDPBUSD(v_s2, data, mults);
4176 p += n;
4177 }
4178  
4179 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
4180 s1 %= DIVISOR;
4181 s2 %= DIVISOR;
4182 }
4183 #else
4184
4185 48 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 24 24 48
4186
4187 24 size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
4188 MAX_CHUNK_LEN) & ~(2*VL - 1));
4189 24 len -= n;
4190  
4191 24 ▶ if (n >= 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 2*VL) { 24 0 24
4192 24 vec_t v_s1 = zeroes;
4193 24 vec_t v_s1_sums = zeroes;
4194 24 vec_t v_byte_sums_a = zeroes;
4195 24 vec_t v_byte_sums_b = zeroes;
4196 24 vec_t v_byte_sums_c = zeroes;
4197 24 vec_t v_byte_sums_d = zeroes;
4198 vec_t v_s2;
4199  
4200 24 s2 += s1 * (n & ~(2*VL - 1));
4201  
4202 do {
4203 624 vec_t data_a = VLOADU(p + 0*VL);
4204 1248 vec_t data_b = VLOADU(p + 1*VL);
4205  
4206 624 v_s1_sums = VADD32(v_s1_sums, v_s1);
4207 1248 v_byte_sums_a = VADD16(v_byte_sums_a,
4208 VUNPACKLO8(data_a, zeroes));
4209 1248 v_byte_sums_b = VADD16(v_byte_sums_b,
4210 VUNPACKHI8(data_a, zeroes));
4211 1248 v_byte_sums_c = VADD16(v_byte_sums_c,
4212 VUNPACKLO8(data_b, zeroes));
4213 1248 v_byte_sums_d = VADD16(v_byte_sums_d,
4214 VUNPACKHI8(data_b, zeroes));
4215 1872 v_s1 = VADD32(v_s1,
4216 VADD32(VSAD8(data_a, zeroes),
4217 VSAD8(data_b, zeroes)));
4218
4219 #if GCC_PREREQ(1, 0)
4220 624 __asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
4221 "+x" (v_byte_sums_a),
4222 "+x" (v_byte_sums_b),
4223 "+x" (v_byte_sums_c),
4224 "+x" (v_byte_sums_d));
4225 #endif
4226 624 p += 2*VL;
4227 624 n -= 2*VL;
4228 624 ▶ } while (n >= 2*VL);
Branch: 1 branch
BranchTrueFalseTotal
} while (n >= 2*VL); 600 24 624
4229  
4230
4231 216 v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
4232 VMADD16(v_byte_sums_a, mults_a),
4233 VMADD16(v_byte_sums_b, mults_b),
4234 VMADD16(v_byte_sums_c, mults_c),
4235 VMADD16(v_byte_sums_d, mults_d));
4236 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
4237 }
4238
4239 216 ▶ ADLER32_CHUNK(s1, s2, p, n);
Branch: 3 branches
BranchTrueFalseTotal
ADLER32_CHUNK(s1, s2, p, n); 24 0 24
ADLER32_CHUNK(s1, s2, p, n); 192 24 216
ADLER32_CHUNK(s1, s2, p, n); 0 24 24
4240 }
4241 #endif
4242 24 return (s2 << 16) | s1;
4243 }
4244  
4245 #undef vec_t
4246 #undef mask_t
4247 #undef LOG2_VL
4248 #undef VADD8
4249 #undef VADD16
4250 #undef VADD32
4251 #undef VDPBUSD
4252 #undef VLOAD
4253 #undef VLOADU
4254 #undef VMADD16
4255 #undef VMASKZ_LOADU
4256 #undef VMULLO32
4257 #undef VSAD8
4258 #undef VSET1_8
4259 #undef VSET1_32
4260 #undef VSETZERO
4261 #undef VSLL32
4262 #undef VUNPACKLO8
4263 #undef VUNPACKHI8
4264  
4265 #undef SUFFIX
4266 #undef ATTRIBUTES
4267 #undef VL
4268 #undef USE_VNNI
4269 #undef USE_AVX512
4270  
4271 #endif
4272  
4273  
4274 #if (GCC_PREREQ(12, 1) || CLANG_PREREQ(12, 0, 13000000) || MSVC_PREREQ(1930)) && \
4275 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX_VNNI)
4276 # define adler32_x86_avx2_vnni adler32_x86_avx2_vnni
4277 # define SUFFIX _avx2_vnni
4278 # define ATTRIBUTES _target_attribute("avx2,avxvnni")
4279 # define VL 32
4280 # define USE_VNNI 1
4281 # define USE_AVX512 0
4282 /* #include "x86-adler32_template.h" */
4283  
4284  
4285  
4286  
4287 #if VL == 16
4288 # define vec_t __m128i
4289 # define mask_t u16
4290 # define LOG2_VL 4
4291 # define VADD8(a, b) _mm_add_epi8((a), (b))
4292 # define VADD16(a, b) _mm_add_epi16((a), (b))
4293 # define VADD32(a, b) _mm_add_epi32((a), (b))
4294 # if USE_AVX512
4295 # define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
4296 # else
4297 # define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
4298 # endif
4299 # define VLOAD(p) _mm_load_si128((const void *)(p))
4300 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
4301 # define VMADD16(a, b) _mm_madd_epi16((a), (b))
4302 # define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
4303 # define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
4304 # define VSAD8(a, b) _mm_sad_epu8((a), (b))
4305 # define VSET1_8(a) _mm_set1_epi8(a)
4306 # define VSET1_32(a) _mm_set1_epi32(a)
4307 # define VSETZERO() _mm_setzero_si128()
4308 # define VSLL32(a, b) _mm_slli_epi32((a), (b))
4309 # define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
4310 # define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
4311 #elif VL == 32
4312 # define vec_t __m256i
4313 # define mask_t u32
4314 # define LOG2_VL 5
4315 # define VADD8(a, b) _mm256_add_epi8((a), (b))
4316 # define VADD16(a, b) _mm256_add_epi16((a), (b))
4317 # define VADD32(a, b) _mm256_add_epi32((a), (b))
4318 # if USE_AVX512
4319 # define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
4320 # else
4321 # define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
4322 # endif
4323 # define VLOAD(p) _mm256_load_si256((const void *)(p))
4324 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
4325 # define VMADD16(a, b) _mm256_madd_epi16((a), (b))
4326 # define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
4327 # define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
4328 # define VSAD8(a, b) _mm256_sad_epu8((a), (b))
4329 # define VSET1_8(a) _mm256_set1_epi8(a)
4330 # define VSET1_32(a) _mm256_set1_epi32(a)
4331 # define VSETZERO() _mm256_setzero_si256()
4332 # define VSLL32(a, b) _mm256_slli_epi32((a), (b))
4333 # define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
4334 # define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
4335 #elif VL == 64
4336 # define vec_t __m512i
4337 # define mask_t u64
4338 # define LOG2_VL 6
4339 # define VADD8(a, b) _mm512_add_epi8((a), (b))
4340 # define VADD16(a, b) _mm512_add_epi16((a), (b))
4341 # define VADD32(a, b) _mm512_add_epi32((a), (b))
4342 # define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
4343 # define VLOAD(p) _mm512_load_si512((const void *)(p))
4344 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
4345 # define VMADD16(a, b) _mm512_madd_epi16((a), (b))
4346 # define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
4347 # define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
4348 # define VSAD8(a, b) _mm512_sad_epu8((a), (b))
4349 # define VSET1_8(a) _mm512_set1_epi8(a)
4350 # define VSET1_32(a) _mm512_set1_epi32(a)
4351 # define VSETZERO() _mm512_setzero_si512()
4352 # define VSLL32(a, b) _mm512_slli_epi32((a), (b))
4353 # define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
4354 # define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
4355 #else
4356 # error "unsupported vector length"
4357 #endif
4358  
4359 #define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
4360 #define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
4361 #define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
4362 #define VADD32_7X(a, b, c, d, e, f, g) \
4363 VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
4364  
4365  
4366 #undef reduce_to_32bits
4367 static forceinline ATTRIBUTES void
4368 ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
4369 {
4370 __m128i v_s1_128, v_s2_128;
4371 #if VL == 16
4372 {
4373 v_s1_128 = v_s1;
4374 v_s2_128 = v_s2;
4375 }
4376 #else
4377 {
4378 __m256i v_s1_256, v_s2_256;
4379 #if VL == 32
4380 0 v_s1_256 = v_s1;
4381 0 v_s2_256 = v_s2;
4382 #else
4383
4384 v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
4385 _mm512_extracti64x4_epi64(v_s1, 1));
4386 v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
4387 _mm512_extracti64x4_epi64(v_s2, 1));
4388 #endif
4389
4390 0 v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
4391 0 _mm256_extracti128_si256(v_s1_256, 1));
4392 0 v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
4393 0 _mm256_extracti128_si256(v_s2_256, 1));
4394 }
4395 #endif
4396  
4397
4398 #if USE_VNNI
4399 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
4400 #endif
4401 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
4402 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
4403 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
4404  
4405 0 *s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
4406 0 *s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
4407 0 }
4408 #define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
4409  
4410 static ATTRIBUTES u32
4411 0 ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
4412 {
4413 #if USE_VNNI
4414
4415 static const u8 _aligned_attribute(VL) raw_mults[VL] = {
4416 #if VL == 64
4417 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
4418 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
4419 #endif
4420 #if VL >= 32
4421 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
4422 #endif
4423 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
4424 };
4425 0 const vec_t ones = VSET1_8(1);
4426 #else
4427
4428 static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
4429 #if VL == 16
4430 { 32, 31, 30, 29, 28, 27, 26, 25 },
4431 { 24, 23, 22, 21, 20, 19, 18, 17 },
4432 { 16, 15, 14, 13, 12, 11, 10, 9 },
4433 { 8, 7, 6, 5, 4, 3, 2, 1 },
4434 #elif VL == 32
4435 { 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
4436 { 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
4437 { 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
4438 { 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
4439 #else
4440 # error "unsupported parameters"
4441 #endif
4442 };
4443 const vec_t mults_a = VLOAD(raw_mults[0]);
4444 const vec_t mults_b = VLOAD(raw_mults[1]);
4445 const vec_t mults_c = VLOAD(raw_mults[2]);
4446 const vec_t mults_d = VLOAD(raw_mults[3]);
4447 #endif
4448 0 const vec_t zeroes = VSETZERO();
4449 0 u32 s1 = adler & 0xFFFF;
4450 0 u32 s2 = adler >> 16;
4451  
4452
4453 0 ▶ if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
Branch: 2 branches
BranchTrueFalseTotal
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
4454 do {
4455 0 s1 += *p++;
4456 0 s2 += s1;
4457 0 len--;
4458 0 ▶ } while ((uintptr_t)p & (VL-1));
Branch: 1 branch
BranchTrueFalseTotal
} while ((uintptr_t)p & (VL-1)); 0 0 0
4459 0 s1 %= DIVISOR;
4460 0 s2 %= DIVISOR;
4461 }
4462  
4463 #if USE_VNNI
4464
4465 0 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 0 0 0
4466
4467 0 size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
4468 0 vec_t mults = VLOAD(raw_mults);
4469 0 vec_t v_s1 = zeroes;
4470 0 vec_t v_s2 = zeroes;
4471  
4472 0 s2 += s1 * n;
4473 0 len -= n;
4474  
4475 0 ▶ if (n >= 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 4*VL) { 0 0 0
4476 0 vec_t v_s1_b = zeroes;
4477 0 vec_t v_s1_c = zeroes;
4478 0 vec_t v_s1_d = zeroes;
4479 0 vec_t v_s2_b = zeroes;
4480 0 vec_t v_s2_c = zeroes;
4481 0 vec_t v_s2_d = zeroes;
4482 0 vec_t v_s1_sums = zeroes;
4483 0 vec_t v_s1_sums_b = zeroes;
4484 0 vec_t v_s1_sums_c = zeroes;
4485 0 vec_t v_s1_sums_d = zeroes;
4486 vec_t tmp0, tmp1;
4487  
4488 do {
4489 0 vec_t data_a = VLOADU(p + 0*VL);
4490 0 vec_t data_b = VLOADU(p + 1*VL);
4491 0 vec_t data_c = VLOADU(p + 2*VL);
4492 0 vec_t data_d = VLOADU(p + 3*VL);
4493  
4494
4495 #if GCC_PREREQ(1, 0)
4496 0 __asm__("" : "+v" (data_a), "+v" (data_b),
4497 "+v" (data_c), "+v" (data_d));
4498 #endif
4499  
4500 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
4501 0 v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
4502 0 v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
4503 0 v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
4504  
4505 0 v_s1_sums = VADD32(v_s1_sums, v_s1);
4506 0 v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
4507 0 v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
4508 0 v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
4509  
4510 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
4511 0 v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
4512 0 v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
4513 0 v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
4514  
4515
4516 #if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
4517 0 __asm__("" : "+v" (v_s2), "+v" (v_s2_b),
4518 "+v" (v_s2_c), "+v" (v_s2_d),
4519 "+v" (v_s1_sums),
4520 "+v" (v_s1_sums_b),
4521 "+v" (v_s1_sums_c),
4522 "+v" (v_s1_sums_d),
4523 "+v" (v_s1), "+v" (v_s1_b),
4524 "+v" (v_s1_c), "+v" (v_s1_d));
4525 #endif
4526 0 p += 4*VL;
4527 0 n -= 4*VL;
4528 0 ▶ } while (n >= 4*VL);
Branch: 1 branch
BranchTrueFalseTotal
} while (n >= 4*VL); 0 0 0
4529  
4530
4531 0 tmp0 = VADD32(v_s1, v_s1_b);
4532 0 tmp1 = VADD32(v_s1, v_s1_c);
4533 0 v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
4534 v_s1_sums_c, v_s1_sums_d);
4535 0 v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
4536 0 v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
4537 VSLL32(tmp0, LOG2_VL + 1),
4538 VSLL32(tmp1, LOG2_VL),
4539 v_s2, v_s2_b, v_s2_c, v_s2_d);
4540 }
4541  
4542
4543 0 ▶ if (n >= 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 2*VL) { 0 0 0
4544 0 const vec_t data_a = VLOADU(p + 0*VL);
4545 0 const vec_t data_b = VLOADU(p + 1*VL);
4546  
4547 0 v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
4548 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
4549 0 v_s1 = VDPBUSD(v_s1, data_b, ones);
4550 0 v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
4551 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
4552 0 v_s2 = VDPBUSD(v_s2, data_b, mults);
4553 0 p += 2*VL;
4554 0 n -= 2*VL;
4555 }
4556 0 ▶ if (n) {
Branch: 1 branch
BranchTrueFalseTotal
if (n) { 0 0 0
4557
4558 vec_t data;
4559  
4560 0 v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
4561  
4562 0 mults = VADD8(mults, VSET1_8((int)n - VL));
4563 0 ▶ if (n > VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n > VL) { 0 0 0
4564 0 data = VLOADU(p);
4565 0 v_s1 = VDPBUSD(v_s1, data, ones);
4566 0 v_s2 = VDPBUSD(v_s2, data, mults);
4567 0 p += VL;
4568 0 n -= VL;
4569 0 mults = VADD8(mults, VSET1_8(-VL));
4570 }
4571
4572 #if USE_AVX512
4573 data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
4574 #else
4575 0 data = zeroes;
4576 0 memcpy(&data, p, n);
4577 #endif
4578 0 v_s1 = VDPBUSD(v_s1, data, ones);
4579 0 v_s2 = VDPBUSD(v_s2, data, mults);
4580 0 p += n;
4581 }
4582  
4583 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
4584 0 s1 %= DIVISOR;
4585 0 s2 %= DIVISOR;
4586 }
4587 #else
4588
4589 while (len) {
4590
4591 size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
4592 MAX_CHUNK_LEN) & ~(2*VL - 1));
4593 len -= n;
4594  
4595 if (n >= 2*VL) {
4596 vec_t v_s1 = zeroes;
4597 vec_t v_s1_sums = zeroes;
4598 vec_t v_byte_sums_a = zeroes;
4599 vec_t v_byte_sums_b = zeroes;
4600 vec_t v_byte_sums_c = zeroes;
4601 vec_t v_byte_sums_d = zeroes;
4602 vec_t v_s2;
4603  
4604 s2 += s1 * (n & ~(2*VL - 1));
4605  
4606 do {
4607 vec_t data_a = VLOADU(p + 0*VL);
4608 vec_t data_b = VLOADU(p + 1*VL);
4609  
4610 v_s1_sums = VADD32(v_s1_sums, v_s1);
4611 v_byte_sums_a = VADD16(v_byte_sums_a,
4612 VUNPACKLO8(data_a, zeroes));
4613 v_byte_sums_b = VADD16(v_byte_sums_b,
4614 VUNPACKHI8(data_a, zeroes));
4615 v_byte_sums_c = VADD16(v_byte_sums_c,
4616 VUNPACKLO8(data_b, zeroes));
4617 v_byte_sums_d = VADD16(v_byte_sums_d,
4618 VUNPACKHI8(data_b, zeroes));
4619 v_s1 = VADD32(v_s1,
4620 VADD32(VSAD8(data_a, zeroes),
4621 VSAD8(data_b, zeroes)));
4622
4623 #if GCC_PREREQ(1, 0)
4624 __asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
4625 "+x" (v_byte_sums_a),
4626 "+x" (v_byte_sums_b),
4627 "+x" (v_byte_sums_c),
4628 "+x" (v_byte_sums_d));
4629 #endif
4630 p += 2*VL;
4631 n -= 2*VL;
4632 } while (n >= 2*VL);
4633  
4634
4635 v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
4636 VMADD16(v_byte_sums_a, mults_a),
4637 VMADD16(v_byte_sums_b, mults_b),
4638 VMADD16(v_byte_sums_c, mults_c),
4639 VMADD16(v_byte_sums_d, mults_d));
4640 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
4641 }
4642
4643 ADLER32_CHUNK(s1, s2, p, n);
4644 }
4645 #endif
4646 0 return (s2 << 16) | s1;
4647 }
4648  
4649 #undef vec_t
4650 #undef mask_t
4651 #undef LOG2_VL
4652 #undef VADD8
4653 #undef VADD16
4654 #undef VADD32
4655 #undef VDPBUSD
4656 #undef VLOAD
4657 #undef VLOADU
4658 #undef VMADD16
4659 #undef VMASKZ_LOADU
4660 #undef VMULLO32
4661 #undef VSAD8
4662 #undef VSET1_8
4663 #undef VSET1_32
4664 #undef VSETZERO
4665 #undef VSLL32
4666 #undef VUNPACKLO8
4667 #undef VUNPACKHI8
4668  
4669 #undef SUFFIX
4670 #undef ATTRIBUTES
4671 #undef VL
4672 #undef USE_VNNI
4673 #undef USE_AVX512
4674  
4675 #endif
4676  
4677 #if (GCC_PREREQ(8, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
4678 !(CLANG_PREREQ(18, 0, 18000000) && !CLANG_PREREQ(19, 0, 19000000)) && \
4679 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_AVX512VNNI)
4680  
4681 # define adler32_x86_avx512_vl256_vnni adler32_x86_avx512_vl256_vnni
4682 # define SUFFIX _avx512_vl256_vnni
4683 # define ATTRIBUTES _target_attribute("avx512bw,avx512vl,avx512vnni")
4684 # define VL 32
4685 # define USE_VNNI 1
4686 # define USE_AVX512 1
4687 /* #include "x86-adler32_template.h" */
4688  
4689  
4690  
4691  
4692 #if VL == 16
4693 # define vec_t __m128i
4694 # define mask_t u16
4695 # define LOG2_VL 4
4696 # define VADD8(a, b) _mm_add_epi8((a), (b))
4697 # define VADD16(a, b) _mm_add_epi16((a), (b))
4698 # define VADD32(a, b) _mm_add_epi32((a), (b))
4699 # if USE_AVX512
4700 # define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
4701 # else
4702 # define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
4703 # endif
4704 # define VLOAD(p) _mm_load_si128((const void *)(p))
4705 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
4706 # define VMADD16(a, b) _mm_madd_epi16((a), (b))
4707 # define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
4708 # define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
4709 # define VSAD8(a, b) _mm_sad_epu8((a), (b))
4710 # define VSET1_8(a) _mm_set1_epi8(a)
4711 # define VSET1_32(a) _mm_set1_epi32(a)
4712 # define VSETZERO() _mm_setzero_si128()
4713 # define VSLL32(a, b) _mm_slli_epi32((a), (b))
4714 # define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
4715 # define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
4716 #elif VL == 32
4717 # define vec_t __m256i
4718 # define mask_t u32
4719 # define LOG2_VL 5
4720 # define VADD8(a, b) _mm256_add_epi8((a), (b))
4721 # define VADD16(a, b) _mm256_add_epi16((a), (b))
4722 # define VADD32(a, b) _mm256_add_epi32((a), (b))
4723 # if USE_AVX512
4724 # define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
4725 # else
4726 # define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
4727 # endif
4728 # define VLOAD(p) _mm256_load_si256((const void *)(p))
4729 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
4730 # define VMADD16(a, b) _mm256_madd_epi16((a), (b))
4731 # define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
4732 # define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
4733 # define VSAD8(a, b) _mm256_sad_epu8((a), (b))
4734 # define VSET1_8(a) _mm256_set1_epi8(a)
4735 # define VSET1_32(a) _mm256_set1_epi32(a)
4736 # define VSETZERO() _mm256_setzero_si256()
4737 # define VSLL32(a, b) _mm256_slli_epi32((a), (b))
4738 # define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
4739 # define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
4740 #elif VL == 64
4741 # define vec_t __m512i
4742 # define mask_t u64
4743 # define LOG2_VL 6
4744 # define VADD8(a, b) _mm512_add_epi8((a), (b))
4745 # define VADD16(a, b) _mm512_add_epi16((a), (b))
4746 # define VADD32(a, b) _mm512_add_epi32((a), (b))
4747 # define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
4748 # define VLOAD(p) _mm512_load_si512((const void *)(p))
4749 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
4750 # define VMADD16(a, b) _mm512_madd_epi16((a), (b))
4751 # define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
4752 # define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
4753 # define VSAD8(a, b) _mm512_sad_epu8((a), (b))
4754 # define VSET1_8(a) _mm512_set1_epi8(a)
4755 # define VSET1_32(a) _mm512_set1_epi32(a)
4756 # define VSETZERO() _mm512_setzero_si512()
4757 # define VSLL32(a, b) _mm512_slli_epi32((a), (b))
4758 # define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
4759 # define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
4760 #else
4761 # error "unsupported vector length"
4762 #endif
4763  
4764 #define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
4765 #define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
4766 #define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
4767 #define VADD32_7X(a, b, c, d, e, f, g) \
4768 VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
4769  
4770  
4771 #undef reduce_to_32bits
4772 static forceinline ATTRIBUTES void
4773 ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
4774 {
4775 __m128i v_s1_128, v_s2_128;
4776 #if VL == 16
4777 {
4778 v_s1_128 = v_s1;
4779 v_s2_128 = v_s2;
4780 }
4781 #else
4782 {
4783 __m256i v_s1_256, v_s2_256;
4784 #if VL == 32
4785 0 v_s1_256 = v_s1;
4786 0 v_s2_256 = v_s2;
4787 #else
4788
4789 v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
4790 _mm512_extracti64x4_epi64(v_s1, 1));
4791 v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
4792 _mm512_extracti64x4_epi64(v_s2, 1));
4793 #endif
4794
4795 0 v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
4796 0 _mm256_extracti128_si256(v_s1_256, 1));
4797 0 v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
4798 0 _mm256_extracti128_si256(v_s2_256, 1));
4799 }
4800 #endif
4801  
4802
4803 #if USE_VNNI
4804 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
4805 #endif
4806 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
4807 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
4808 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
4809  
4810 0 *s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
4811 0 *s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
4812 0 }
4813 #define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
4814  
4815 static ATTRIBUTES u32
4816 0 ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
4817 {
4818 #if USE_VNNI
4819
4820 static const u8 _aligned_attribute(VL) raw_mults[VL] = {
4821 #if VL == 64
4822 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
4823 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
4824 #endif
4825 #if VL >= 32
4826 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
4827 #endif
4828 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
4829 };
4830 0 const vec_t ones = VSET1_8(1);
4831 #else
4832
4833 static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
4834 #if VL == 16
4835 { 32, 31, 30, 29, 28, 27, 26, 25 },
4836 { 24, 23, 22, 21, 20, 19, 18, 17 },
4837 { 16, 15, 14, 13, 12, 11, 10, 9 },
4838 { 8, 7, 6, 5, 4, 3, 2, 1 },
4839 #elif VL == 32
4840 { 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
4841 { 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
4842 { 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
4843 { 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
4844 #else
4845 # error "unsupported parameters"
4846 #endif
4847 };
4848 const vec_t mults_a = VLOAD(raw_mults[0]);
4849 const vec_t mults_b = VLOAD(raw_mults[1]);
4850 const vec_t mults_c = VLOAD(raw_mults[2]);
4851 const vec_t mults_d = VLOAD(raw_mults[3]);
4852 #endif
4853 0 const vec_t zeroes = VSETZERO();
4854 0 u32 s1 = adler & 0xFFFF;
4855 0 u32 s2 = adler >> 16;
4856  
4857
4858 0 ▶ if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
Branch: 2 branches
BranchTrueFalseTotal
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
4859 do {
4860 0 s1 += *p++;
4861 0 s2 += s1;
4862 0 len--;
4863 0 ▶ } while ((uintptr_t)p & (VL-1));
Branch: 1 branch
BranchTrueFalseTotal
} while ((uintptr_t)p & (VL-1)); 0 0 0
4864 0 s1 %= DIVISOR;
4865 0 s2 %= DIVISOR;
4866 }
4867  
4868 #if USE_VNNI
4869
4870 0 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 0 0 0
4871
4872 0 size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
4873 0 vec_t mults = VLOAD(raw_mults);
4874 0 vec_t v_s1 = zeroes;
4875 0 vec_t v_s2 = zeroes;
4876  
4877 0 s2 += s1 * n;
4878 0 len -= n;
4879  
4880 0 ▶ if (n >= 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 4*VL) { 0 0 0
4881 0 vec_t v_s1_b = zeroes;
4882 0 vec_t v_s1_c = zeroes;
4883 0 vec_t v_s1_d = zeroes;
4884 0 vec_t v_s2_b = zeroes;
4885 0 vec_t v_s2_c = zeroes;
4886 0 vec_t v_s2_d = zeroes;
4887 0 vec_t v_s1_sums = zeroes;
4888 0 vec_t v_s1_sums_b = zeroes;
4889 0 vec_t v_s1_sums_c = zeroes;
4890 0 vec_t v_s1_sums_d = zeroes;
4891 vec_t tmp0, tmp1;
4892  
4893 do {
4894 0 vec_t data_a = VLOADU(p + 0*VL);
4895 0 vec_t data_b = VLOADU(p + 1*VL);
4896 0 vec_t data_c = VLOADU(p + 2*VL);
4897 0 vec_t data_d = VLOADU(p + 3*VL);
4898  
4899
4900 #if GCC_PREREQ(1, 0)
4901 0 __asm__("" : "+v" (data_a), "+v" (data_b),
4902 "+v" (data_c), "+v" (data_d));
4903 #endif
4904  
4905 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
4906 0 v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
4907 0 v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
4908 0 v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
4909  
4910 0 v_s1_sums = VADD32(v_s1_sums, v_s1);
4911 0 v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
4912 0 v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
4913 0 v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
4914  
4915 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
4916 0 v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
4917 0 v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
4918 0 v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
4919  
4920
4921 #if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
4922 0 __asm__("" : "+v" (v_s2), "+v" (v_s2_b),
4923 "+v" (v_s2_c), "+v" (v_s2_d),
4924 "+v" (v_s1_sums),
4925 "+v" (v_s1_sums_b),
4926 "+v" (v_s1_sums_c),
4927 "+v" (v_s1_sums_d),
4928 "+v" (v_s1), "+v" (v_s1_b),
4929 "+v" (v_s1_c), "+v" (v_s1_d));
4930 #endif
4931 0 p += 4*VL;
4932 0 n -= 4*VL;
4933 0 ▶ } while (n >= 4*VL);
Branch: 1 branch
BranchTrueFalseTotal
} while (n >= 4*VL); 0 0 0
4934  
4935
4936 0 tmp0 = VADD32(v_s1, v_s1_b);
4937 0 tmp1 = VADD32(v_s1, v_s1_c);
4938 0 v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
4939 v_s1_sums_c, v_s1_sums_d);
4940 0 v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
4941 0 v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
4942 VSLL32(tmp0, LOG2_VL + 1),
4943 VSLL32(tmp1, LOG2_VL),
4944 v_s2, v_s2_b, v_s2_c, v_s2_d);
4945 }
4946  
4947
4948 0 ▶ if (n >= 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 2*VL) { 0 0 0
4949 0 const vec_t data_a = VLOADU(p + 0*VL);
4950 0 const vec_t data_b = VLOADU(p + 1*VL);
4951  
4952 0 v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
4953 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
4954 0 v_s1 = VDPBUSD(v_s1, data_b, ones);
4955 0 v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
4956 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
4957 0 v_s2 = VDPBUSD(v_s2, data_b, mults);
4958 0 p += 2*VL;
4959 0 n -= 2*VL;
4960 }
4961 0 ▶ if (n) {
Branch: 1 branch
BranchTrueFalseTotal
if (n) { 0 0 0
4962
4963 vec_t data;
4964  
4965 0 v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
4966  
4967 0 mults = VADD8(mults, VSET1_8((int)n - VL));
4968 0 ▶ if (n > VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n > VL) { 0 0 0
4969 0 data = VLOADU(p);
4970 0 v_s1 = VDPBUSD(v_s1, data, ones);
4971 0 v_s2 = VDPBUSD(v_s2, data, mults);
4972 0 p += VL;
4973 0 n -= VL;
4974 0 mults = VADD8(mults, VSET1_8(-VL));
4975 }
4976
4977 #if USE_AVX512
4978 0 data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
4979 #else
4980 data = zeroes;
4981 memcpy(&data, p, n);
4982 #endif
4983 0 v_s1 = VDPBUSD(v_s1, data, ones);
4984 0 v_s2 = VDPBUSD(v_s2, data, mults);
4985 0 p += n;
4986 }
4987  
4988 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
4989 0 s1 %= DIVISOR;
4990 0 s2 %= DIVISOR;
4991 }
4992 #else
4993
4994 while (len) {
4995
4996 size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
4997 MAX_CHUNK_LEN) & ~(2*VL - 1));
4998 len -= n;
4999  
5000 if (n >= 2*VL) {
5001 vec_t v_s1 = zeroes;
5002 vec_t v_s1_sums = zeroes;
5003 vec_t v_byte_sums_a = zeroes;
5004 vec_t v_byte_sums_b = zeroes;
5005 vec_t v_byte_sums_c = zeroes;
5006 vec_t v_byte_sums_d = zeroes;
5007 vec_t v_s2;
5008  
5009 s2 += s1 * (n & ~(2*VL - 1));
5010  
5011 do {
5012 vec_t data_a = VLOADU(p + 0*VL);
5013 vec_t data_b = VLOADU(p + 1*VL);
5014  
5015 v_s1_sums = VADD32(v_s1_sums, v_s1);
5016 v_byte_sums_a = VADD16(v_byte_sums_a,
5017 VUNPACKLO8(data_a, zeroes));
5018 v_byte_sums_b = VADD16(v_byte_sums_b,
5019 VUNPACKHI8(data_a, zeroes));
5020 v_byte_sums_c = VADD16(v_byte_sums_c,
5021 VUNPACKLO8(data_b, zeroes));
5022 v_byte_sums_d = VADD16(v_byte_sums_d,
5023 VUNPACKHI8(data_b, zeroes));
5024 v_s1 = VADD32(v_s1,
5025 VADD32(VSAD8(data_a, zeroes),
5026 VSAD8(data_b, zeroes)));
5027
5028 #if GCC_PREREQ(1, 0)
5029 __asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
5030 "+x" (v_byte_sums_a),
5031 "+x" (v_byte_sums_b),
5032 "+x" (v_byte_sums_c),
5033 "+x" (v_byte_sums_d));
5034 #endif
5035 p += 2*VL;
5036 n -= 2*VL;
5037 } while (n >= 2*VL);
5038  
5039
5040 v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
5041 VMADD16(v_byte_sums_a, mults_a),
5042 VMADD16(v_byte_sums_b, mults_b),
5043 VMADD16(v_byte_sums_c, mults_c),
5044 VMADD16(v_byte_sums_d, mults_d));
5045 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
5046 }
5047
5048 ADLER32_CHUNK(s1, s2, p, n);
5049 }
5050 #endif
5051 0 return (s2 << 16) | s1;
5052 }
5053  
5054 #undef vec_t
5055 #undef mask_t
5056 #undef LOG2_VL
5057 #undef VADD8
5058 #undef VADD16
5059 #undef VADD32
5060 #undef VDPBUSD
5061 #undef VLOAD
5062 #undef VLOADU
5063 #undef VMADD16
5064 #undef VMASKZ_LOADU
5065 #undef VMULLO32
5066 #undef VSAD8
5067 #undef VSET1_8
5068 #undef VSET1_32
5069 #undef VSETZERO
5070 #undef VSLL32
5071 #undef VUNPACKLO8
5072 #undef VUNPACKHI8
5073  
5074 #undef SUFFIX
5075 #undef ATTRIBUTES
5076 #undef VL
5077 #undef USE_VNNI
5078 #undef USE_AVX512
5079  
5080  
5081  
5082 # define adler32_x86_avx512_vl512_vnni adler32_x86_avx512_vl512_vnni
5083 # define SUFFIX _avx512_vl512_vnni
5084 # define ATTRIBUTES _target_attribute("avx512bw,avx512vnni")
5085 # define VL 64
5086 # define USE_VNNI 1
5087 # define USE_AVX512 1
5088 /* #include "x86-adler32_template.h" */
5089  
5090  
5091  
5092  
5093 #if VL == 16
5094 # define vec_t __m128i
5095 # define mask_t u16
5096 # define LOG2_VL 4
5097 # define VADD8(a, b) _mm_add_epi8((a), (b))
5098 # define VADD16(a, b) _mm_add_epi16((a), (b))
5099 # define VADD32(a, b) _mm_add_epi32((a), (b))
5100 # if USE_AVX512
5101 # define VDPBUSD(a, b, c) _mm_dpbusd_epi32((a), (b), (c))
5102 # else
5103 # define VDPBUSD(a, b, c) _mm_dpbusd_avx_epi32((a), (b), (c))
5104 # endif
5105 # define VLOAD(p) _mm_load_si128((const void *)(p))
5106 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
5107 # define VMADD16(a, b) _mm_madd_epi16((a), (b))
5108 # define VMASKZ_LOADU(mask, p) _mm_maskz_loadu_epi8((mask), (p))
5109 # define VMULLO32(a, b) _mm_mullo_epi32((a), (b))
5110 # define VSAD8(a, b) _mm_sad_epu8((a), (b))
5111 # define VSET1_8(a) _mm_set1_epi8(a)
5112 # define VSET1_32(a) _mm_set1_epi32(a)
5113 # define VSETZERO() _mm_setzero_si128()
5114 # define VSLL32(a, b) _mm_slli_epi32((a), (b))
5115 # define VUNPACKLO8(a, b) _mm_unpacklo_epi8((a), (b))
5116 # define VUNPACKHI8(a, b) _mm_unpackhi_epi8((a), (b))
5117 #elif VL == 32
5118 # define vec_t __m256i
5119 # define mask_t u32
5120 # define LOG2_VL 5
5121 # define VADD8(a, b) _mm256_add_epi8((a), (b))
5122 # define VADD16(a, b) _mm256_add_epi16((a), (b))
5123 # define VADD32(a, b) _mm256_add_epi32((a), (b))
5124 # if USE_AVX512
5125 # define VDPBUSD(a, b, c) _mm256_dpbusd_epi32((a), (b), (c))
5126 # else
5127 # define VDPBUSD(a, b, c) _mm256_dpbusd_avx_epi32((a), (b), (c))
5128 # endif
5129 # define VLOAD(p) _mm256_load_si256((const void *)(p))
5130 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
5131 # define VMADD16(a, b) _mm256_madd_epi16((a), (b))
5132 # define VMASKZ_LOADU(mask, p) _mm256_maskz_loadu_epi8((mask), (p))
5133 # define VMULLO32(a, b) _mm256_mullo_epi32((a), (b))
5134 # define VSAD8(a, b) _mm256_sad_epu8((a), (b))
5135 # define VSET1_8(a) _mm256_set1_epi8(a)
5136 # define VSET1_32(a) _mm256_set1_epi32(a)
5137 # define VSETZERO() _mm256_setzero_si256()
5138 # define VSLL32(a, b) _mm256_slli_epi32((a), (b))
5139 # define VUNPACKLO8(a, b) _mm256_unpacklo_epi8((a), (b))
5140 # define VUNPACKHI8(a, b) _mm256_unpackhi_epi8((a), (b))
5141 #elif VL == 64
5142 # define vec_t __m512i
5143 # define mask_t u64
5144 # define LOG2_VL 6
5145 # define VADD8(a, b) _mm512_add_epi8((a), (b))
5146 # define VADD16(a, b) _mm512_add_epi16((a), (b))
5147 # define VADD32(a, b) _mm512_add_epi32((a), (b))
5148 # define VDPBUSD(a, b, c) _mm512_dpbusd_epi32((a), (b), (c))
5149 # define VLOAD(p) _mm512_load_si512((const void *)(p))
5150 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
5151 # define VMADD16(a, b) _mm512_madd_epi16((a), (b))
5152 # define VMASKZ_LOADU(mask, p) _mm512_maskz_loadu_epi8((mask), (p))
5153 # define VMULLO32(a, b) _mm512_mullo_epi32((a), (b))
5154 # define VSAD8(a, b) _mm512_sad_epu8((a), (b))
5155 # define VSET1_8(a) _mm512_set1_epi8(a)
5156 # define VSET1_32(a) _mm512_set1_epi32(a)
5157 # define VSETZERO() _mm512_setzero_si512()
5158 # define VSLL32(a, b) _mm512_slli_epi32((a), (b))
5159 # define VUNPACKLO8(a, b) _mm512_unpacklo_epi8((a), (b))
5160 # define VUNPACKHI8(a, b) _mm512_unpackhi_epi8((a), (b))
5161 #else
5162 # error "unsupported vector length"
5163 #endif
5164  
5165 #define VADD32_3X(a, b, c) VADD32(VADD32((a), (b)), (c))
5166 #define VADD32_4X(a, b, c, d) VADD32(VADD32((a), (b)), VADD32((c), (d)))
5167 #define VADD32_5X(a, b, c, d, e) VADD32((a), VADD32_4X((b), (c), (d), (e)))
5168 #define VADD32_7X(a, b, c, d, e, f, g) \
5169 VADD32(VADD32_3X((a), (b), (c)), VADD32_4X((d), (e), (f), (g)))
5170  
5171  
5172 #undef reduce_to_32bits
5173 static forceinline ATTRIBUTES void
5174 ADD_SUFFIX(reduce_to_32bits)(vec_t v_s1, vec_t v_s2, u32 *s1_p, u32 *s2_p)
5175 {
5176 __m128i v_s1_128, v_s2_128;
5177 #if VL == 16
5178 {
5179 v_s1_128 = v_s1;
5180 v_s2_128 = v_s2;
5181 }
5182 #else
5183 {
5184 __m256i v_s1_256, v_s2_256;
5185 #if VL == 32
5186 v_s1_256 = v_s1;
5187 v_s2_256 = v_s2;
5188 #else
5189
5190 0 v_s1_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s1, 0),
5191 0 _mm512_extracti64x4_epi64(v_s1, 1));
5192 0 v_s2_256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v_s2, 0),
5193 0 _mm512_extracti64x4_epi64(v_s2, 1));
5194 #endif
5195
5196 0 v_s1_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s1_256, 0),
5197 0 _mm256_extracti128_si256(v_s1_256, 1));
5198 0 v_s2_128 = _mm_add_epi32(_mm256_extracti128_si256(v_s2_256, 0),
5199 0 _mm256_extracti128_si256(v_s2_256, 1));
5200 }
5201 #endif
5202  
5203
5204 #if USE_VNNI
5205 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x31));
5206 #endif
5207 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x31));
5208 0 v_s1_128 = _mm_add_epi32(v_s1_128, _mm_shuffle_epi32(v_s1_128, 0x02));
5209 0 v_s2_128 = _mm_add_epi32(v_s2_128, _mm_shuffle_epi32(v_s2_128, 0x02));
5210  
5211 0 *s1_p += (u32)_mm_cvtsi128_si32(v_s1_128);
5212 0 *s2_p += (u32)_mm_cvtsi128_si32(v_s2_128);
5213 0 }
5214 #define reduce_to_32bits ADD_SUFFIX(reduce_to_32bits)
5215  
5216 static ATTRIBUTES u32
5217 0 ADD_SUFFIX(adler32_x86)(u32 adler, const u8 *p, size_t len)
5218 {
5219 #if USE_VNNI
5220
5221 static const u8 _aligned_attribute(VL) raw_mults[VL] = {
5222 #if VL == 64
5223 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
5224 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
5225 #endif
5226 #if VL >= 32
5227 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
5228 #endif
5229 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
5230 };
5231 0 const vec_t ones = VSET1_8(1);
5232 #else
5233
5234 static const u16 _aligned_attribute(VL) raw_mults[4][VL / 2] = {
5235 #if VL == 16
5236 { 32, 31, 30, 29, 28, 27, 26, 25 },
5237 { 24, 23, 22, 21, 20, 19, 18, 17 },
5238 { 16, 15, 14, 13, 12, 11, 10, 9 },
5239 { 8, 7, 6, 5, 4, 3, 2, 1 },
5240 #elif VL == 32
5241 { 64, 63, 62, 61, 60, 59, 58, 57, 48, 47, 46, 45, 44, 43, 42, 41 },
5242 { 56, 55, 54, 53, 52, 51, 50, 49, 40, 39, 38, 37, 36, 35, 34, 33 },
5243 { 32, 31, 30, 29, 28, 27, 26, 25, 16, 15, 14, 13, 12, 11, 10, 9 },
5244 { 24, 23, 22, 21, 20, 19, 18, 17, 8, 7, 6, 5, 4, 3, 2, 1 },
5245 #else
5246 # error "unsupported parameters"
5247 #endif
5248 };
5249 const vec_t mults_a = VLOAD(raw_mults[0]);
5250 const vec_t mults_b = VLOAD(raw_mults[1]);
5251 const vec_t mults_c = VLOAD(raw_mults[2]);
5252 const vec_t mults_d = VLOAD(raw_mults[3]);
5253 #endif
5254 0 const vec_t zeroes = VSETZERO();
5255 0 u32 s1 = adler & 0xFFFF;
5256 0 u32 s2 = adler >> 16;
5257  
5258
5259 0 ▶ if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) {
Branch: 2 branches
BranchTrueFalseTotal
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
if (unlikely(len > 65536 && ((uintptr_t)p & (VL-1)))) { 0 0 0
5260 do {
5261 0 s1 += *p++;
5262 0 s2 += s1;
5263 0 len--;
5264 0 ▶ } while ((uintptr_t)p & (VL-1));
Branch: 1 branch
BranchTrueFalseTotal
} while ((uintptr_t)p & (VL-1)); 0 0 0
5265 0 s1 %= DIVISOR;
5266 0 s2 %= DIVISOR;
5267 }
5268  
5269 #if USE_VNNI
5270
5271 0 ▶ while (len) {
Branch: 1 branch
BranchTrueFalseTotal
while (len) { 0 0 0
5272
5273 0 size_t n = MIN(len, MAX_CHUNK_LEN & ~(4*VL - 1));
5274 0 vec_t mults = VLOAD(raw_mults);
5275 0 vec_t v_s1 = zeroes;
5276 0 vec_t v_s2 = zeroes;
5277  
5278 0 s2 += s1 * n;
5279 0 len -= n;
5280  
5281 0 ▶ if (n >= 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 4*VL) { 0 0 0
5282 0 vec_t v_s1_b = zeroes;
5283 0 vec_t v_s1_c = zeroes;
5284 0 vec_t v_s1_d = zeroes;
5285 0 vec_t v_s2_b = zeroes;
5286 0 vec_t v_s2_c = zeroes;
5287 0 vec_t v_s2_d = zeroes;
5288 0 vec_t v_s1_sums = zeroes;
5289 0 vec_t v_s1_sums_b = zeroes;
5290 0 vec_t v_s1_sums_c = zeroes;
5291 0 vec_t v_s1_sums_d = zeroes;
5292 vec_t tmp0, tmp1;
5293  
5294 do {
5295 0 vec_t data_a = VLOADU(p + 0*VL);
5296 0 vec_t data_b = VLOADU(p + 1*VL);
5297 0 vec_t data_c = VLOADU(p + 2*VL);
5298 0 vec_t data_d = VLOADU(p + 3*VL);
5299  
5300
5301 #if GCC_PREREQ(1, 0)
5302 0 __asm__("" : "+v" (data_a), "+v" (data_b),
5303 "+v" (data_c), "+v" (data_d));
5304 #endif
5305  
5306 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
5307 0 v_s2_b = VDPBUSD(v_s2_b, data_b, mults);
5308 0 v_s2_c = VDPBUSD(v_s2_c, data_c, mults);
5309 0 v_s2_d = VDPBUSD(v_s2_d, data_d, mults);
5310  
5311 0 v_s1_sums = VADD32(v_s1_sums, v_s1);
5312 0 v_s1_sums_b = VADD32(v_s1_sums_b, v_s1_b);
5313 0 v_s1_sums_c = VADD32(v_s1_sums_c, v_s1_c);
5314 0 v_s1_sums_d = VADD32(v_s1_sums_d, v_s1_d);
5315  
5316 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
5317 0 v_s1_b = VDPBUSD(v_s1_b, data_b, ones);
5318 0 v_s1_c = VDPBUSD(v_s1_c, data_c, ones);
5319 0 v_s1_d = VDPBUSD(v_s1_d, data_d, ones);
5320  
5321
5322 #if GCC_PREREQ(1, 0) && !defined(ARCH_X86_32)
5323 0 __asm__("" : "+v" (v_s2), "+v" (v_s2_b),
5324 "+v" (v_s2_c), "+v" (v_s2_d),
5325 "+v" (v_s1_sums),
5326 "+v" (v_s1_sums_b),
5327 "+v" (v_s1_sums_c),
5328 "+v" (v_s1_sums_d),
5329 "+v" (v_s1), "+v" (v_s1_b),
5330 "+v" (v_s1_c), "+v" (v_s1_d));
5331 #endif
5332 0 p += 4*VL;
5333 0 n -= 4*VL;
5334 0 ▶ } while (n >= 4*VL);
Branch: 1 branch
BranchTrueFalseTotal
} while (n >= 4*VL); 0 0 0
5335  
5336
5337 0 tmp0 = VADD32(v_s1, v_s1_b);
5338 0 tmp1 = VADD32(v_s1, v_s1_c);
5339 0 v_s1_sums = VADD32_4X(v_s1_sums, v_s1_sums_b,
5340 v_s1_sums_c, v_s1_sums_d);
5341 0 v_s1 = VADD32_3X(tmp0, v_s1_c, v_s1_d);
5342 0 v_s2 = VADD32_7X(VSLL32(v_s1_sums, LOG2_VL + 2),
5343 VSLL32(tmp0, LOG2_VL + 1),
5344 VSLL32(tmp1, LOG2_VL),
5345 v_s2, v_s2_b, v_s2_c, v_s2_d);
5346 }
5347  
5348
5349 0 ▶ if (n >= 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n >= 2*VL) { 0 0 0
5350 0 const vec_t data_a = VLOADU(p + 0*VL);
5351 0 const vec_t data_b = VLOADU(p + 1*VL);
5352  
5353 0 v_s2 = VADD32(v_s2, VSLL32(v_s1, LOG2_VL + 1));
5354 0 v_s1 = VDPBUSD(v_s1, data_a, ones);
5355 0 v_s1 = VDPBUSD(v_s1, data_b, ones);
5356 0 v_s2 = VDPBUSD(v_s2, data_a, VSET1_8(VL));
5357 0 v_s2 = VDPBUSD(v_s2, data_a, mults);
5358 0 v_s2 = VDPBUSD(v_s2, data_b, mults);
5359 0 p += 2*VL;
5360 0 n -= 2*VL;
5361 }
5362 0 ▶ if (n) {
Branch: 1 branch
BranchTrueFalseTotal
if (n) { 0 0 0
5363
5364 vec_t data;
5365  
5366 0 v_s2 = VADD32(v_s2, VMULLO32(v_s1, VSET1_32(n)));
5367  
5368 0 mults = VADD8(mults, VSET1_8((int)n - VL));
5369 0 ▶ if (n > VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (n > VL) { 0 0 0
5370 0 data = VLOADU(p);
5371 0 v_s1 = VDPBUSD(v_s1, data, ones);
5372 0 v_s2 = VDPBUSD(v_s2, data, mults);
5373 0 p += VL;
5374 0 n -= VL;
5375 0 mults = VADD8(mults, VSET1_8(-VL));
5376 }
5377
5378 #if USE_AVX512
5379 0 data = VMASKZ_LOADU((mask_t)-1 >> (VL - n), p);
5380 #else
5381 data = zeroes;
5382 memcpy(&data, p, n);
5383 #endif
5384 0 v_s1 = VDPBUSD(v_s1, data, ones);
5385 0 v_s2 = VDPBUSD(v_s2, data, mults);
5386 0 p += n;
5387 }
5388  
5389 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
5390 0 s1 %= DIVISOR;
5391 0 s2 %= DIVISOR;
5392 }
5393 #else
5394
5395 while (len) {
5396
5397 size_t n = MIN(len, MIN(2 * VL * (INT16_MAX / UINT8_MAX),
5398 MAX_CHUNK_LEN) & ~(2*VL - 1));
5399 len -= n;
5400  
5401 if (n >= 2*VL) {
5402 vec_t v_s1 = zeroes;
5403 vec_t v_s1_sums = zeroes;
5404 vec_t v_byte_sums_a = zeroes;
5405 vec_t v_byte_sums_b = zeroes;
5406 vec_t v_byte_sums_c = zeroes;
5407 vec_t v_byte_sums_d = zeroes;
5408 vec_t v_s2;
5409  
5410 s2 += s1 * (n & ~(2*VL - 1));
5411  
5412 do {
5413 vec_t data_a = VLOADU(p + 0*VL);
5414 vec_t data_b = VLOADU(p + 1*VL);
5415  
5416 v_s1_sums = VADD32(v_s1_sums, v_s1);
5417 v_byte_sums_a = VADD16(v_byte_sums_a,
5418 VUNPACKLO8(data_a, zeroes));
5419 v_byte_sums_b = VADD16(v_byte_sums_b,
5420 VUNPACKHI8(data_a, zeroes));
5421 v_byte_sums_c = VADD16(v_byte_sums_c,
5422 VUNPACKLO8(data_b, zeroes));
5423 v_byte_sums_d = VADD16(v_byte_sums_d,
5424 VUNPACKHI8(data_b, zeroes));
5425 v_s1 = VADD32(v_s1,
5426 VADD32(VSAD8(data_a, zeroes),
5427 VSAD8(data_b, zeroes)));
5428
5429 #if GCC_PREREQ(1, 0)
5430 __asm__("" : "+x" (v_s1), "+x" (v_s1_sums),
5431 "+x" (v_byte_sums_a),
5432 "+x" (v_byte_sums_b),
5433 "+x" (v_byte_sums_c),
5434 "+x" (v_byte_sums_d));
5435 #endif
5436 p += 2*VL;
5437 n -= 2*VL;
5438 } while (n >= 2*VL);
5439  
5440
5441 v_s2 = VADD32_5X(VSLL32(v_s1_sums, LOG2_VL + 1),
5442 VMADD16(v_byte_sums_a, mults_a),
5443 VMADD16(v_byte_sums_b, mults_b),
5444 VMADD16(v_byte_sums_c, mults_c),
5445 VMADD16(v_byte_sums_d, mults_d));
5446 reduce_to_32bits(v_s1, v_s2, &s1, &s2);
5447 }
5448
5449 ADLER32_CHUNK(s1, s2, p, n);
5450 }
5451 #endif
5452 0 return (s2 << 16) | s1;
5453 }
5454  
5455 #undef vec_t
5456 #undef mask_t
5457 #undef LOG2_VL
5458 #undef VADD8
5459 #undef VADD16
5460 #undef VADD32
5461 #undef VDPBUSD
5462 #undef VLOAD
5463 #undef VLOADU
5464 #undef VMADD16
5465 #undef VMASKZ_LOADU
5466 #undef VMULLO32
5467 #undef VSAD8
5468 #undef VSET1_8
5469 #undef VSET1_32
5470 #undef VSETZERO
5471 #undef VSLL32
5472 #undef VUNPACKLO8
5473 #undef VUNPACKHI8
5474  
5475 #undef SUFFIX
5476 #undef ATTRIBUTES
5477 #undef VL
5478 #undef USE_VNNI
5479 #undef USE_AVX512
5480  
5481 #endif
5482  
5483 static inline adler32_func_t
5484 1 arch_select_adler32_func(void)
5485 {
5486 1 const u32 features MAYBE_UNUSED = get_x86_cpu_features();
5487  
5488 #ifdef adler32_x86_avx512_vl512_vnni
5489 1 ▶ if ((features & X86_CPU_FEATURE_ZMM) &&
Branch: 1 branch
BranchTrueFalseTotal
if ((features & X86_CPU_FEATURE_ZMM) && 0 1 1
5490 0 ▶ HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features))
Branch: 2 branches
BranchTrueFalseTotal
HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features)) 0 0 0
HAVE_AVX512BW(features) && HAVE_AVX512VNNI(features)) 0 0 0
5491 0 return adler32_x86_avx512_vl512_vnni;
5492 #endif
5493 #ifdef adler32_x86_avx512_vl256_vnni
5494 1 ▶ if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) &&
Branch: 2 branches
BranchTrueFalseTotal
if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) && 0 1 1
if (HAVE_AVX512BW(features) && HAVE_AVX512VL(features) && 0 0 0
5495 0 ▶ HAVE_AVX512VNNI(features))
Branch: 1 branch
BranchTrueFalseTotal
HAVE_AVX512VNNI(features)) 0 0 0
5496 0 return adler32_x86_avx512_vl256_vnni;
5497 #endif
5498 #ifdef adler32_x86_avx2_vnni
5499 1 ▶ if (HAVE_AVX2(features) && HAVE_AVXVNNI(features))
Branch: 2 branches
BranchTrueFalseTotal
if (HAVE_AVX2(features) && HAVE_AVXVNNI(features)) 1 0 1
if (HAVE_AVX2(features) && HAVE_AVXVNNI(features)) 0 1 1
5500 0 return adler32_x86_avx2_vnni;
5501 #endif
5502 #ifdef adler32_x86_avx2
5503 1 ▶ if (HAVE_AVX2(features))
Branch: 1 branch
BranchTrueFalseTotal
if (HAVE_AVX2(features)) 1 0 1
5504 1 return adler32_x86_avx2;
5505 #endif
5506 #ifdef adler32_x86_sse2
5507 if (HAVE_SSE2(features))
5508 0 return adler32_x86_sse2;
5509 #endif
5510 return NULL;
5511 }
5512 #define arch_select_adler32_func arch_select_adler32_func
5513  
5514 #endif
5515  
5516 #endif
5517  
5518 #ifndef DEFAULT_IMPL
5519 # define DEFAULT_IMPL adler32_generic
5520 #endif
5521  
5522 #ifdef arch_select_adler32_func
5523 static u32 adler32_dispatch_adler32(u32 adler, const u8 *p, size_t len);
5524  
5525 static volatile adler32_func_t adler32_impl = adler32_dispatch_adler32;
5526  
5527  
5528 1 static u32 adler32_dispatch_adler32(u32 adler, const u8 *p, size_t len)
5529 {
5530 1 adler32_func_t f = arch_select_adler32_func();
5531  
5532 1 ▶ if (f == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (f == NULL) 0 1 1
5533 0 f = DEFAULT_IMPL;
5534  
5535 1 adler32_impl = f;
5536 1 return f(adler, p, len);
5537 }
5538 #else
5539  
5540 #define adler32_impl DEFAULT_IMPL
5541 #endif
5542  
5543 LIBDEFLATEAPI u32
5544 24 libdeflate_adler32(u32 adler, const void *buffer, size_t len)
5545 {
5546 24 ▶ if (buffer == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (buffer == NULL) 0 24 24
5547 0 return 1;
5548 24 return adler32_impl(adler, buffer, len);
5549 }
5550 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/crc32.c */
5551  
5552  
5553  
5554  
5555 /* #include "lib_common.h" */
5556  
5557  
5558 #ifndef LIB_LIB_COMMON_H
5559 #define LIB_LIB_COMMON_H
5560  
5561 #ifdef LIBDEFLATE_H
5562
5563 # error "lib_common.h must always be included before libdeflate.h"
5564 #endif
5565  
5566 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
5567 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
5568 #elif defined(__GNUC__)
5569 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
5570 #else
5571 # define LIBDEFLATE_EXPORT_SYM
5572 #endif
5573  
5574  
5575 #if defined(__GNUC__) && defined(__i386__)
5576 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
5577 #else
5578 # define LIBDEFLATE_ALIGN_STACK
5579 #endif
5580  
5581 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
5582  
5583 /* #include "../common_defs.h" */
5584  
5585  
5586 #ifndef COMMON_DEFS_H
5587 #define COMMON_DEFS_H
5588  
5589 /* #include "libdeflate.h" */
5590  
5591  
5592 #ifndef LIBDEFLATE_H
5593 #define LIBDEFLATE_H
5594  
5595 #include <stddef.h>
5596 #include <stdint.h>
5597  
5598 #ifdef __cplusplus
5599 extern "C" {
5600 #endif
5601  
5602 #define LIBDEFLATE_VERSION_MAJOR 1
5603 #define LIBDEFLATE_VERSION_MINOR 26
5604 #define LIBDEFLATE_VERSION_STRING "1.26"
5605  
5606  
5607 #ifndef LIBDEFLATEAPI
5608 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
5609 # define LIBDEFLATEAPI __declspec(dllimport)
5610 # else
5611 # define LIBDEFLATEAPI
5612 # endif
5613 #endif
5614  
5615  
5616  
5617  
5618  
5619 struct libdeflate_compressor;
5620 struct libdeflate_options;
5621  
5622  
5623 LIBDEFLATEAPI struct libdeflate_compressor *
5624 libdeflate_alloc_compressor(int compression_level);
5625  
5626  
5627 LIBDEFLATEAPI struct libdeflate_compressor *
5628 libdeflate_alloc_compressor_ex(int compression_level,
5629 const struct libdeflate_options *options);
5630  
5631  
5632 LIBDEFLATEAPI size_t
5633 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
5634 const void *in, size_t in_nbytes,
5635 void *out, size_t out_nbytes_avail);
5636  
5637  
5638 LIBDEFLATEAPI size_t
5639 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
5640 size_t in_nbytes);
5641  
5642  
5643 LIBDEFLATEAPI size_t
5644 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
5645 const void *in, size_t in_nbytes,
5646 void *out, size_t out_nbytes_avail);
5647  
5648  
5649 LIBDEFLATEAPI size_t
5650 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
5651 size_t in_nbytes);
5652  
5653  
5654 LIBDEFLATEAPI size_t
5655 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
5656 const void *in, size_t in_nbytes,
5657 void *out, size_t out_nbytes_avail);
5658  
5659  
5660 LIBDEFLATEAPI size_t
5661 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
5662 size_t in_nbytes);
5663  
5664  
5665 LIBDEFLATEAPI void
5666 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
5667  
5668  
5669  
5670  
5671  
5672 struct libdeflate_decompressor;
5673 struct libdeflate_options;
5674  
5675  
5676 LIBDEFLATEAPI struct libdeflate_decompressor *
5677 libdeflate_alloc_decompressor(void);
5678  
5679  
5680 LIBDEFLATEAPI struct libdeflate_decompressor *
5681 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
5682  
5683  
5684 enum libdeflate_result {
5685
5686 LIBDEFLATE_SUCCESS = 0,
5687  
5688
5689 LIBDEFLATE_BAD_DATA = 1,
5690  
5691
5692 LIBDEFLATE_SHORT_OUTPUT = 2,
5693  
5694
5695 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
5696 };
5697  
5698  
5699 LIBDEFLATEAPI enum libdeflate_result
5700 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
5701 const void *in, size_t in_nbytes,
5702 void *out, size_t out_nbytes_avail,
5703 size_t *actual_out_nbytes_ret);
5704  
5705  
5706 LIBDEFLATEAPI enum libdeflate_result
5707 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
5708 const void *in, size_t in_nbytes,
5709 void *out, size_t out_nbytes_avail,
5710 size_t *actual_in_nbytes_ret,
5711 size_t *actual_out_nbytes_ret);
5712  
5713  
5714 LIBDEFLATEAPI enum libdeflate_result
5715 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
5716 const void *in, size_t in_nbytes,
5717 void *out, size_t out_nbytes_avail,
5718 size_t *actual_out_nbytes_ret);
5719  
5720  
5721 LIBDEFLATEAPI enum libdeflate_result
5722 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
5723 const void *in, size_t in_nbytes,
5724 void *out, size_t out_nbytes_avail,
5725 size_t *actual_in_nbytes_ret,
5726 size_t *actual_out_nbytes_ret);
5727  
5728  
5729 LIBDEFLATEAPI enum libdeflate_result
5730 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
5731 const void *in, size_t in_nbytes,
5732 void *out, size_t out_nbytes_avail,
5733 size_t *actual_out_nbytes_ret);
5734  
5735  
5736 LIBDEFLATEAPI enum libdeflate_result
5737 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
5738 const void *in, size_t in_nbytes,
5739 void *out, size_t out_nbytes_avail,
5740 size_t *actual_in_nbytes_ret,
5741 size_t *actual_out_nbytes_ret);
5742  
5743  
5744 LIBDEFLATEAPI void
5745 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
5746  
5747  
5748  
5749  
5750  
5751  
5752 LIBDEFLATEAPI uint32_t
5753 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
5754  
5755  
5756  
5757 LIBDEFLATEAPI uint32_t
5758 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
5759  
5760  
5761  
5762  
5763  
5764  
5765 LIBDEFLATEAPI void
5766 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
5767 void (*free_func)(void *));
5768  
5769  
5770 struct libdeflate_options {
5771  
5772
5773 size_t sizeof_options;
5774  
5775
5776 void *(*malloc_func)(size_t);
5777 void (*free_func)(void *);
5778 };
5779  
5780 #ifdef __cplusplus
5781 }
5782 #endif
5783  
5784 #endif
5785  
5786  
5787 #include <stdbool.h>
5788 #include <stddef.h>
5789 #include <stdint.h>
5790 #ifdef _MSC_VER
5791 # include <intrin.h>
5792 # include <stdlib.h>
5793
5794
5795 # pragma warning(disable : 4146)
5796
5797 # pragma warning(disable : 4018)
5798 # pragma warning(disable : 4244)
5799 # pragma warning(disable : 4267)
5800 # pragma warning(disable : 4310)
5801
5802 # pragma warning(disable : 4100)
5803 # pragma warning(disable : 4127)
5804 # pragma warning(disable : 4189)
5805 # pragma warning(disable : 4232)
5806 # pragma warning(disable : 4245)
5807 # pragma warning(disable : 4295)
5808 #endif
5809 #ifndef FREESTANDING
5810 # include <string.h>
5811 #endif
5812  
5813  
5814  
5815  
5816  
5817  
5818 #undef ARCH_X86_64
5819 #undef ARCH_X86_32
5820 #undef ARCH_ARM64
5821 #undef ARCH_ARM32
5822 #undef ARCH_RISCV
5823 #ifdef _MSC_VER
5824
5825 # if defined(_M_X64) && !defined(_M_ARM64EC)
5826 # define ARCH_X86_64
5827 # elif defined(_M_IX86)
5828 # define ARCH_X86_32
5829 # elif defined(_M_ARM64)
5830 # define ARCH_ARM64
5831 # elif defined(_M_ARM)
5832 # define ARCH_ARM32
5833 # endif
5834 #else
5835 # if defined(__x86_64__)
5836 # define ARCH_X86_64
5837 # elif defined(__i386__)
5838 # define ARCH_X86_32
5839 # elif defined(__aarch64__)
5840 # define ARCH_ARM64
5841 # elif defined(__arm__)
5842 # define ARCH_ARM32
5843 # elif defined(__riscv)
5844 # define ARCH_RISCV
5845 # endif
5846 #endif
5847  
5848  
5849  
5850  
5851  
5852  
5853 typedef uint8_t u8;
5854 typedef uint16_t u16;
5855 typedef uint32_t u32;
5856 typedef uint64_t u64;
5857 typedef int8_t s8;
5858 typedef int16_t s16;
5859 typedef int32_t s32;
5860 typedef int64_t s64;
5861  
5862  
5863 #ifdef _MSC_VER
5864 # ifdef _WIN64
5865 typedef long long ssize_t;
5866 # else
5867 typedef long ssize_t;
5868 # endif
5869 #endif
5870  
5871  
5872 typedef size_t machine_word_t;
5873  
5874  
5875 #define WORDBYTES ((int)sizeof(machine_word_t))
5876  
5877  
5878 #define WORDBITS (8 * WORDBYTES)
5879  
5880  
5881  
5882  
5883  
5884  
5885 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
5886 # define GCC_PREREQ(major, minor) \
5887 (__GNUC__ > (major) || \
5888 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
5889 # if !GCC_PREREQ(4, 9)
5890 # error "gcc versions older than 4.9 are no longer supported"
5891 # endif
5892 #else
5893 # define GCC_PREREQ(major, minor) 0
5894 #endif
5895 #ifdef __clang__
5896 # ifdef __apple_build_version__
5897 # define CLANG_PREREQ(major, minor, apple_version) \
5898 (__apple_build_version__ >= (apple_version))
5899 # else
5900 # define CLANG_PREREQ(major, minor, apple_version) \
5901 (__clang_major__ > (major) || \
5902 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
5903 # endif
5904 # if !CLANG_PREREQ(3, 9, 8000000)
5905 # error "clang versions older than 3.9 are no longer supported"
5906 # endif
5907 #else
5908 # define CLANG_PREREQ(major, minor, apple_version) 0
5909 #endif
5910 #ifdef _MSC_VER
5911 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
5912 # if !MSVC_PREREQ(1928)
5913 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
5914 # endif
5915 #else
5916 # define MSVC_PREREQ(version) 0
5917 #endif
5918  
5919  
5920 #ifndef __has_attribute
5921 # define __has_attribute(attribute) 0
5922 #endif
5923  
5924  
5925 #ifndef __has_builtin
5926 # define __has_builtin(builtin) 0
5927 #endif
5928  
5929  
5930 #ifdef _MSC_VER
5931 # define inline __inline
5932 #endif
5933  
5934  
5935 #if defined(__GNUC__) || __has_attribute(always_inline)
5936 # define forceinline inline __attribute__((always_inline))
5937 #elif defined(_MSC_VER)
5938 # define forceinline __forceinline
5939 #else
5940 # define forceinline inline
5941 #endif
5942  
5943  
5944 #if defined(__GNUC__) || __has_attribute(unused)
5945 # define MAYBE_UNUSED __attribute__((unused))
5946 #else
5947 # define MAYBE_UNUSED
5948 #endif
5949  
5950  
5951 #if defined(__GNUC__) || __has_attribute(noreturn)
5952 # define NORETURN __attribute__((noreturn))
5953 #else
5954 # define NORETURN
5955 #endif
5956  
5957  
5958 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
5959 # define likely(expr) __builtin_expect(!!(expr), 1)
5960 #else
5961 # define likely(expr) (expr)
5962 #endif
5963  
5964  
5965 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
5966 # define unlikely(expr) __builtin_expect(!!(expr), 0)
5967 #else
5968 # define unlikely(expr) (expr)
5969 #endif
5970  
5971  
5972 #undef prefetchr
5973 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
5974 # define prefetchr(addr) __builtin_prefetch((addr), 0)
5975 #elif defined(_MSC_VER)
5976 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
5977 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
5978 # elif defined(ARCH_ARM64)
5979 # define prefetchr(addr) __prefetch2((addr), 0x00 )
5980 # elif defined(ARCH_ARM32)
5981 # define prefetchr(addr) __prefetch(addr)
5982 # endif
5983 #endif
5984 #ifndef prefetchr
5985 # define prefetchr(addr)
5986 #endif
5987  
5988  
5989 #undef prefetchw
5990 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
5991 # define prefetchw(addr) __builtin_prefetch((addr), 1)
5992 #elif defined(_MSC_VER)
5993 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
5994 # define prefetchw(addr) _m_prefetchw(addr)
5995 # elif defined(ARCH_ARM64)
5996 # define prefetchw(addr) __prefetch2((addr), 0x10 )
5997 # elif defined(ARCH_ARM32)
5998 # define prefetchw(addr) __prefetchw(addr)
5999 # endif
6000 #endif
6001 #ifndef prefetchw
6002 # define prefetchw(addr)
6003 #endif
6004  
6005  
6006 #undef _aligned_attribute
6007 #if defined(__GNUC__) || __has_attribute(aligned)
6008 # define _aligned_attribute(n) __attribute__((aligned(n)))
6009 #elif defined(_MSC_VER)
6010 # define _aligned_attribute(n) __declspec(align(n))
6011 #endif
6012  
6013  
6014 #if defined(__GNUC__) || __has_attribute(target)
6015 # define _target_attribute(attrs) __attribute__((target(attrs)))
6016 #else
6017 # define _target_attribute(attrs)
6018 #endif
6019  
6020  
6021  
6022  
6023  
6024 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
6025 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
6026 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
6027 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
6028 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
6029 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
6030 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
6031  
6032  
6033  
6034  
6035  
6036  
6037 #if defined(__BYTE_ORDER__)
6038 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
6039 #elif defined(_MSC_VER)
6040 # define CPU_IS_LITTLE_ENDIAN() true
6041 #else
6042 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
6043 {
6044 union {
6045 u32 w;
6046 u8 b;
6047 } u;
6048  
6049 u.w = 1;
6050 return u.b;
6051 }
6052 #endif
6053  
6054  
6055 static forceinline u16 bswap16(u16 v)
6056 {
6057 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
6058 return __builtin_bswap16(v);
6059 #elif defined(_MSC_VER)
6060 return _byteswap_ushort(v);
6061 #else
6062 return (v << 8) | (v >> 8);
6063 #endif
6064 }
6065  
6066  
6067 static forceinline u32 bswap32(u32 v)
6068 {
6069 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
6070 return __builtin_bswap32(v);
6071 #elif defined(_MSC_VER)
6072 return _byteswap_ulong(v);
6073 #else
6074 return ((v & 0x000000FF) << 24) |
6075 ((v & 0x0000FF00) << 8) |
6076 ((v & 0x00FF0000) >> 8) |
6077 ((v & 0xFF000000) >> 24);
6078 #endif
6079 }
6080  
6081  
6082 static forceinline u64 bswap64(u64 v)
6083 {
6084 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
6085 return __builtin_bswap64(v);
6086 #elif defined(_MSC_VER)
6087 return _byteswap_uint64(v);
6088 #else
6089 return ((v & 0x00000000000000FF) << 56) |
6090 ((v & 0x000000000000FF00) << 40) |
6091 ((v & 0x0000000000FF0000) << 24) |
6092 ((v & 0x00000000FF000000) << 8) |
6093 ((v & 0x000000FF00000000) >> 8) |
6094 ((v & 0x0000FF0000000000) >> 24) |
6095 ((v & 0x00FF000000000000) >> 40) |
6096 ((v & 0xFF00000000000000) >> 56);
6097 #endif
6098 }
6099  
6100 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
6101 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
6102 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
6103 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
6104 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
6105 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
6106  
6107  
6108  
6109  
6110  
6111  
6112 #if (defined(__GNUC__) || defined(__clang__)) && \
6113 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
6114 defined(__ARM_FEATURE_UNALIGNED) || \
6115 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
6116 defined(__riscv_misaligned_fast) || \
6117 defined(__wasm__))
6118 # define UNALIGNED_ACCESS_IS_FAST 1
6119 #elif defined(_MSC_VER)
6120 # define UNALIGNED_ACCESS_IS_FAST 1
6121 #else
6122 # define UNALIGNED_ACCESS_IS_FAST 0
6123 #endif
6124  
6125  
6126  
6127 #ifdef FREESTANDING
6128 # define MEMCOPY __builtin_memcpy
6129 #else
6130 # define MEMCOPY memcpy
6131 #endif
6132  
6133  
6134  
6135 #define DEFINE_UNALIGNED_TYPE(type) \
6136 static forceinline type \
6137 load_##type##_unaligned(const void *p) \
6138 { \
6139 type v; \
6140 \
6141 MEMCOPY(&v, p, sizeof(v)); \
6142 return v; \
6143 } \
6144 \
6145 static forceinline void \
6146 store_##type##_unaligned(type v, void *p) \
6147 { \
6148 MEMCOPY(p, &v, sizeof(v)); \
6149 }
6150  
6151 DEFINE_UNALIGNED_TYPE(u16)
6152 DEFINE_UNALIGNED_TYPE(u32)
6153 DEFINE_UNALIGNED_TYPE(u64)
6154 DEFINE_UNALIGNED_TYPE(machine_word_t)
6155  
6156 #undef MEMCOPY
6157  
6158 #define load_word_unaligned load_machine_word_t_unaligned
6159 #define store_word_unaligned store_machine_word_t_unaligned
6160  
6161  
6162  
6163 static forceinline u16
6164 get_unaligned_le16(const u8 *p)
6165 {
6166 if (UNALIGNED_ACCESS_IS_FAST)
6167 return le16_bswap(load_u16_unaligned(p));
6168 else
6169 return ((u16)p[1] << 8) | p[0];
6170 }
6171  
6172 static forceinline u16
6173 get_unaligned_be16(const u8 *p)
6174 {
6175 if (UNALIGNED_ACCESS_IS_FAST)
6176 return be16_bswap(load_u16_unaligned(p));
6177 else
6178 return ((u16)p[0] << 8) | p[1];
6179 }
6180  
6181 static forceinline u32
6182 get_unaligned_le32(const u8 *p)
6183 {
6184 if (UNALIGNED_ACCESS_IS_FAST)
6185 return le32_bswap(load_u32_unaligned(p));
6186 else
6187 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
6188 ((u32)p[1] << 8) | p[0];
6189 }
6190  
6191 static forceinline u32
6192 get_unaligned_be32(const u8 *p)
6193 {
6194 if (UNALIGNED_ACCESS_IS_FAST)
6195 return be32_bswap(load_u32_unaligned(p));
6196 else
6197 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
6198 ((u32)p[2] << 8) | p[3];
6199 }
6200  
6201 static forceinline u64
6202 get_unaligned_le64(const u8 *p)
6203 {
6204 if (UNALIGNED_ACCESS_IS_FAST)
6205 return le64_bswap(load_u64_unaligned(p));
6206 else
6207 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
6208 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
6209 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
6210 ((u64)p[1] << 8) | p[0];
6211 }
6212  
6213 static forceinline machine_word_t
6214 get_unaligned_leword(const u8 *p)
6215 {
6216 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
6217 if (WORDBITS == 32)
6218 return get_unaligned_le32(p);
6219 else
6220 return get_unaligned_le64(p);
6221 }
6222  
6223  
6224  
6225 static forceinline void
6226 put_unaligned_le16(u16 v, u8 *p)
6227 {
6228 if (UNALIGNED_ACCESS_IS_FAST) {
6229 store_u16_unaligned(le16_bswap(v), p);
6230 } else {
6231 p[0] = (u8)(v >> 0);
6232 p[1] = (u8)(v >> 8);
6233 }
6234 }
6235  
6236 static forceinline void
6237 put_unaligned_be16(u16 v, u8 *p)
6238 {
6239 if (UNALIGNED_ACCESS_IS_FAST) {
6240 store_u16_unaligned(be16_bswap(v), p);
6241 } else {
6242 p[0] = (u8)(v >> 8);
6243 p[1] = (u8)(v >> 0);
6244 }
6245 }
6246  
6247 static forceinline void
6248 put_unaligned_le32(u32 v, u8 *p)
6249 {
6250 if (UNALIGNED_ACCESS_IS_FAST) {
6251 store_u32_unaligned(le32_bswap(v), p);
6252 } else {
6253 p[0] = (u8)(v >> 0);
6254 p[1] = (u8)(v >> 8);
6255 p[2] = (u8)(v >> 16);
6256 p[3] = (u8)(v >> 24);
6257 }
6258 }
6259  
6260 static forceinline void
6261 put_unaligned_be32(u32 v, u8 *p)
6262 {
6263 if (UNALIGNED_ACCESS_IS_FAST) {
6264 store_u32_unaligned(be32_bswap(v), p);
6265 } else {
6266 p[0] = (u8)(v >> 24);
6267 p[1] = (u8)(v >> 16);
6268 p[2] = (u8)(v >> 8);
6269 p[3] = (u8)(v >> 0);
6270 }
6271 }
6272  
6273 static forceinline void
6274 put_unaligned_le64(u64 v, u8 *p)
6275 {
6276 if (UNALIGNED_ACCESS_IS_FAST) {
6277 store_u64_unaligned(le64_bswap(v), p);
6278 } else {
6279 p[0] = (u8)(v >> 0);
6280 p[1] = (u8)(v >> 8);
6281 p[2] = (u8)(v >> 16);
6282 p[3] = (u8)(v >> 24);
6283 p[4] = (u8)(v >> 32);
6284 p[5] = (u8)(v >> 40);
6285 p[6] = (u8)(v >> 48);
6286 p[7] = (u8)(v >> 56);
6287 }
6288 }
6289  
6290 static forceinline void
6291 put_unaligned_leword(machine_word_t v, u8 *p)
6292 {
6293 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
6294 if (WORDBITS == 32)
6295 put_unaligned_le32(v, p);
6296 else
6297 put_unaligned_le64(v, p);
6298 }
6299  
6300  
6301  
6302  
6303  
6304  
6305  
6306 static forceinline unsigned
6307 bsr32(u32 v)
6308 {
6309 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
6310 return 31 - __builtin_clz(v);
6311 #elif defined(_MSC_VER)
6312 unsigned long i;
6313  
6314 _BitScanReverse(&i, v);
6315 return i;
6316 #else
6317 unsigned i = 0;
6318  
6319 while ((v >>= 1) != 0)
6320 i++;
6321 return i;
6322 #endif
6323 }
6324  
6325 static forceinline unsigned
6326 bsr64(u64 v)
6327 {
6328 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
6329 return 63 - __builtin_clzll(v);
6330 #elif defined(_MSC_VER) && defined(_WIN64)
6331 unsigned long i;
6332  
6333 _BitScanReverse64(&i, v);
6334 return i;
6335 #else
6336 unsigned i = 0;
6337  
6338 while ((v >>= 1) != 0)
6339 i++;
6340 return i;
6341 #endif
6342 }
6343  
6344 static forceinline unsigned
6345 bsrw(machine_word_t v)
6346 {
6347 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
6348 if (WORDBITS == 32)
6349 return bsr32(v);
6350 else
6351 return bsr64(v);
6352 }
6353  
6354  
6355  
6356 static forceinline unsigned
6357 bsf32(u32 v)
6358 {
6359 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
6360 return __builtin_ctz(v);
6361 #elif defined(_MSC_VER)
6362 unsigned long i;
6363  
6364 _BitScanForward(&i, v);
6365 return i;
6366 #else
6367 unsigned i = 0;
6368  
6369 for (; (v & 1) == 0; v >>= 1)
6370 i++;
6371 return i;
6372 #endif
6373 }
6374  
6375 static forceinline unsigned
6376 bsf64(u64 v)
6377 {
6378 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
6379 return __builtin_ctzll(v);
6380 #elif defined(_MSC_VER) && defined(_WIN64)
6381 unsigned long i;
6382  
6383 _BitScanForward64(&i, v);
6384 return i;
6385 #else
6386 unsigned i = 0;
6387  
6388 for (; (v & 1) == 0; v >>= 1)
6389 i++;
6390 return i;
6391 #endif
6392 }
6393  
6394 static forceinline unsigned
6395 bsfw(machine_word_t v)
6396 {
6397 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
6398 if (WORDBITS == 32)
6399 return bsf32(v);
6400 else
6401 return bsf64(v);
6402 }
6403  
6404  
6405 #undef rbit32
6406 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
6407 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
6408 static forceinline u32
6409 rbit32(u32 v)
6410 {
6411 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
6412 return v;
6413 }
6414 #define rbit32 rbit32
6415 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
6416 static forceinline u32
6417 rbit32(u32 v)
6418 {
6419 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
6420 return v;
6421 }
6422 #define rbit32 rbit32
6423 #endif
6424  
6425 #endif
6426  
6427  
6428 typedef void *(*malloc_func_t)(size_t);
6429 typedef void (*free_func_t)(void *);
6430  
6431 extern malloc_func_t libdeflate_default_malloc_func;
6432 extern free_func_t libdeflate_default_free_func;
6433  
6434 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
6435 size_t alignment, size_t size);
6436 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
6437  
6438 #ifdef FREESTANDING
6439  
6440 void *memset(void *s, int c, size_t n);
6441 #define memset(s, c, n) __builtin_memset((s), (c), (n))
6442  
6443 void *memcpy(void *dest, const void *src, size_t n);
6444 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
6445  
6446 void *memmove(void *dest, const void *src, size_t n);
6447 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
6448  
6449 int memcmp(const void *s1, const void *s2, size_t n);
6450 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
6451  
6452 #undef LIBDEFLATE_ENABLE_ASSERTIONS
6453 #else
6454 # include <string.h>
6455
6456 # ifdef __clang_analyzer__
6457 # define LIBDEFLATE_ENABLE_ASSERTIONS
6458 # endif
6459 #endif
6460  
6461  
6462 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
6463 NORETURN void
6464 libdeflate_assertion_failed(const char *expr, const char *file, int line);
6465 #define ASSERT(expr) { if (unlikely(!(expr))) \
6466 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
6467 #else
6468 #define ASSERT(expr) (void)(expr)
6469 #endif
6470  
6471 #define CONCAT_IMPL(a, b) a##b
6472 #define CONCAT(a, b) CONCAT_IMPL(a, b)
6473 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
6474  
6475 #endif
6476  
6477 /* #include "crc32_multipliers.h" */
6478  
6479  
6480 #define CRC32_X159_MODG 0xae689191
6481 #define CRC32_X95_MODG 0xccaa009e
6482  
6483 #define CRC32_X287_MODG 0xf1da05aa
6484 #define CRC32_X223_MODG 0x81256527
6485  
6486 #define CRC32_X415_MODG 0x3db1ecdc
6487 #define CRC32_X351_MODG 0xaf449247
6488  
6489 #define CRC32_X543_MODG 0x8f352d95
6490 #define CRC32_X479_MODG 0x1d9513d7
6491  
6492 #define CRC32_X671_MODG 0x1c279815
6493 #define CRC32_X607_MODG 0xae0b5394
6494  
6495 #define CRC32_X799_MODG 0xdf068dc2
6496 #define CRC32_X735_MODG 0x57c54819
6497  
6498 #define CRC32_X927_MODG 0x31f8303f
6499 #define CRC32_X863_MODG 0x0cbec0ed
6500  
6501 #define CRC32_X1055_MODG 0x33fff533
6502 #define CRC32_X991_MODG 0x910eeec1
6503  
6504 #define CRC32_X1183_MODG 0x26b70c3d
6505 #define CRC32_X1119_MODG 0x3f41287a
6506  
6507 #define CRC32_X1311_MODG 0xe3543be0
6508 #define CRC32_X1247_MODG 0x9026d5b1
6509  
6510 #define CRC32_X1439_MODG 0x5a1bb05d
6511 #define CRC32_X1375_MODG 0xd1df2327
6512  
6513 #define CRC32_X1567_MODG 0x596c8d81
6514 #define CRC32_X1503_MODG 0xf5e48c85
6515  
6516 #define CRC32_X1695_MODG 0x682bdd4f
6517 #define CRC32_X1631_MODG 0x3c656ced
6518  
6519 #define CRC32_X1823_MODG 0x4a28bd43
6520 #define CRC32_X1759_MODG 0xfe807bbd
6521  
6522 #define CRC32_X1951_MODG 0x0077f00d
6523 #define CRC32_X1887_MODG 0x1f0c2cdd
6524  
6525 #define CRC32_X2079_MODG 0xce3371cb
6526 #define CRC32_X2015_MODG 0xe95c1271
6527  
6528 #define CRC32_X2207_MODG 0xa749e894
6529 #define CRC32_X2143_MODG 0xb918a347
6530  
6531 #define CRC32_X2335_MODG 0x2c538639
6532 #define CRC32_X2271_MODG 0x71d54a59
6533  
6534 #define CRC32_X2463_MODG 0x32b0733c
6535 #define CRC32_X2399_MODG 0xff6f2fc2
6536  
6537 #define CRC32_X2591_MODG 0x0e9bd5cc
6538 #define CRC32_X2527_MODG 0xcec97417
6539  
6540 #define CRC32_X2719_MODG 0x76278617
6541 #define CRC32_X2655_MODG 0x1c63267b
6542  
6543 #define CRC32_X2847_MODG 0xc51b93e3
6544 #define CRC32_X2783_MODG 0xf183c71b
6545  
6546 #define CRC32_X2975_MODG 0x7eaed122
6547 #define CRC32_X2911_MODG 0x9b9bdbd0
6548  
6549 #define CRC32_X3103_MODG 0x2ce423f1
6550 #define CRC32_X3039_MODG 0xd31343ea
6551  
6552 #define CRC32_X3231_MODG 0x8b8d8645
6553 #define CRC32_X3167_MODG 0x4470ac44
6554  
6555 #define CRC32_X3359_MODG 0x4b700aa8
6556 #define CRC32_X3295_MODG 0xeea395c4
6557  
6558 #define CRC32_X3487_MODG 0xeff5e99d
6559 #define CRC32_X3423_MODG 0xf9d9c7ee
6560  
6561 #define CRC32_X3615_MODG 0xad0d2bb2
6562 #define CRC32_X3551_MODG 0xcd669a40
6563  
6564 #define CRC32_X3743_MODG 0x9fb66bd3
6565 #define CRC32_X3679_MODG 0x6d40f445
6566  
6567 #define CRC32_X3871_MODG 0xc2dcc467
6568 #define CRC32_X3807_MODG 0x9ee62949
6569  
6570 #define CRC32_X3999_MODG 0x398e2ff2
6571 #define CRC32_X3935_MODG 0x145575d5
6572  
6573 #define CRC32_X4127_MODG 0x1072db28
6574 #define CRC32_X4063_MODG 0x0c30f51d
6575  
6576 #define CRC32_BARRETT_CONSTANT_1 0xb4e5b025f7011641ULL
6577 #define CRC32_BARRETT_CONSTANT_2 0x00000001db710641ULL
6578  
6579 #define CRC32_NUM_CHUNKS 4
6580 #define CRC32_MIN_VARIABLE_CHUNK_LEN 128UL
6581 #define CRC32_MAX_VARIABLE_CHUNK_LEN 16384UL
6582  
6583  
6584 static const u32 crc32_mults_for_chunklen[][CRC32_NUM_CHUNKS - 1] MAYBE_UNUSED = {
6585 { 0 },
6586
6587 { 0xd31343ea , 0xe95c1271 , 0x910eeec1 , },
6588
6589 { 0x1d6708a0 , 0x0c30f51d , 0xe95c1271 , },
6590
6591 { 0xdb3839f3 , 0x1d6708a0 , 0xd31343ea , },
6592
6593 { 0x1753ab84 , 0xbbf2f6d6 , 0x0c30f51d , },
6594
6595 { 0x3796455c , 0xb8e0e4a8 , 0xc352f6de , },
6596
6597 { 0x3954de39 , 0x1753ab84 , 0x1d6708a0 , },
6598
6599 { 0x632d78c5 , 0x3fc33de4 , 0x9a1b53c8 , },
6600
6601 { 0xa0decef3 , 0x7b4aa8b7 , 0xbbf2f6d6 , },
6602
6603 { 0xe9c09bb0 , 0x3954de39 , 0xdb3839f3 , },
6604
6605 { 0xd51917a4 , 0xcae68461 , 0xb8e0e4a8 , },
6606
6607 { 0x154a8a62 , 0x41e7589c , 0x3e9a43cd , },
6608
6609 { 0xf196555d , 0xa0decef3 , 0x1753ab84 , },
6610
6611 { 0x8eec2999 , 0xefb0a128 , 0x6044fbb0 , },
6612
6613 { 0x27892abf , 0x48d72bb1 , 0x3fc33de4 , },
6614
6615 { 0x77bc2419 , 0xd51917a4 , 0x3796455c , },
6616
6617 { 0xcea114a5 , 0x68c0a2c5 , 0x7b4aa8b7 , },
6618
6619 { 0xa1077e85 , 0x188cc628 , 0x0c21f835 , },
6620
6621 { 0xc5ed75e1 , 0xf196555d , 0x3954de39 , },
6622
6623 { 0xca4fba3f , 0x0acfa26f , 0x6cb21510 , },
6624
6625 { 0xcf5bcdc4 , 0x4fae7fc0 , 0xcae68461 , },
6626
6627 { 0xf36b9d16 , 0x27892abf , 0x632d78c5 , },
6628
6629 { 0xf76fd988 , 0xed5c39b1 , 0x41e7589c , },
6630
6631 { 0x6c45d92e , 0xff809fcd , 0x0c46baec , },
6632
6633 { 0x6116b82b , 0xcea114a5 , 0xa0decef3 , },
6634
6635 { 0x4d9899bb , 0x9f9d8d9c , 0x53deb236 , },
6636
6637 { 0x3e7c93b9 , 0x6666b805 , 0xefb0a128 , },
6638
6639 { 0x388b20ac , 0xc5ed75e1 , 0xe9c09bb0 , },
6640
6641 { 0x0956d953 , 0x97fbdb14 , 0x48d72bb1 , },
6642
6643 { 0x55cb4dfe , 0x1b37c832 , 0xc07331b3 , },
6644
6645 { 0x52222fea , 0xcf5bcdc4 , 0xd51917a4 , },
6646
6647 { 0x0603989b , 0xb03c8112 , 0x5e04b9a5 , },
6648
6649 { 0x4470c029 , 0x2339d155 , 0x68c0a2c5 , },
6650
6651 { 0xb6f35093 , 0xf76fd988 , 0x154a8a62 , },
6652
6653 { 0xc46805ba , 0x416f9449 , 0x188cc628 , },
6654
6655 { 0xc3876592 , 0x4b809189 , 0xc35cf6e7 , },
6656
6657 { 0x5b0c98b9 , 0x6116b82b , 0xf196555d , },
6658
6659 { 0x30d13e5f , 0x4c5a315a , 0x8c224466 , },
6660
6661 { 0x54afca53 , 0xbccfa2c1 , 0x0acfa26f , },
6662
6663 { 0x93102436 , 0x3e7c93b9 , 0x8eec2999 , },
6664
6665 { 0xbd2655a8 , 0x3e116c9d , 0x4fae7fc0 , },
6666
6667 { 0x70cd7f26 , 0x408e57f2 , 0x1691be45 , },
6668
6669 { 0x2d546c53 , 0x0956d953 , 0x27892abf , },
6670
6671 { 0xb53410a8 , 0x42ebf0ad , 0x161f3c12 , },
6672
6673 { 0x67a93f75 , 0xcf3233e4 , 0xed5c39b1 , },
6674
6675 { 0x9830ac33 , 0x52222fea , 0x77bc2419 , },
6676
6677 { 0xb0b6fc3e , 0x2fde73f8 , 0xff809fcd , },
6678
6679 { 0x84170f16 , 0xced90d99 , 0x30de0f98 , },
6680
6681 { 0xd7017a0c , 0x4470c029 , 0xcea114a5 , },
6682
6683 { 0xadb25de6 , 0x84f40beb , 0x2b7e0e1b , },
6684
6685 { 0x8282fddc , 0xec855937 , 0x9f9d8d9c , },
6686
6687 { 0x46362bee , 0xc46805ba , 0xa1077e85 , },
6688
6689 { 0xb9077a01 , 0xdf7a24ac , 0x6666b805 , },
6690
6691 { 0xf51d9bc6 , 0x2b52dc39 , 0x7e774cf6 , },
6692
6693 { 0x4ca19a29 , 0x5b0c98b9 , 0xc5ed75e1 , },
6694
6695 { 0xdc0fc3fc , 0xb939fcdf , 0x3678fed2 , },
6696
6697 { 0x63c3d167 , 0x70f9947d , 0x97fbdb14 , },
6698
6699 { 0x5851d254 , 0x54afca53 , 0xca4fba3f , },
6700
6701 { 0xfeacf2a1 , 0x7a3c0a6a , 0x1b37c832 , },
6702
6703 { 0x93b7edc8 , 0x1fea4d2a , 0x58fa96ee , },
6704
6705 { 0x5539e44a , 0xbd2655a8 , 0xcf5bcdc4 , },
6706
6707 { 0xde32a3d2 , 0x4ff61aa1 , 0x6a6a3694 , },
6708
6709 { 0xf0baeeb6 , 0x7ae2f6f4 , 0xb03c8112 , },
6710
6711 { 0xbe15887f , 0x2d546c53 , 0xf36b9d16 , },
6712
6713 { 0x64f34a05 , 0xe0ee5efe , 0x2339d155 , },
6714
6715 { 0x1b6d1aea , 0xfeafb67c , 0x4fb001a8 , },
6716
6717 { 0x82adb0b8 , 0x67a93f75 , 0xf76fd988 , },
6718
6719 { 0x694587c7 , 0x3b34408b , 0xeccb2978 , },
6720
6721 { 0xd2fc57c3 , 0x07fcf8c6 , 0x416f9449 , },
6722
6723 { 0x9dd6837c , 0xb0b6fc3e , 0x6c45d92e , },
6724
6725 { 0x3a9d1f97 , 0xefd033b2 , 0x4b809189 , },
6726
6727 { 0x1eee1d2a , 0xf2a6e46e , 0x55b4c814 , },
6728
6729 { 0xb57c7728 , 0xd7017a0c , 0x6116b82b , },
6730
6731 { 0xf2fc5d61 , 0x242aac86 , 0x05245cf0 , },
6732
6733 { 0x26387824 , 0xc15c4ca5 , 0x4c5a315a , },
6734
6735 { 0x8c151e77 , 0x8282fddc , 0x4d9899bb , },
6736
6737 { 0x8ea1f680 , 0xf5ff6cdd , 0xbccfa2c1 , },
6738
6739 { 0xe8cf3d2a , 0x338b1fb1 , 0xeda61f70 , },
6740
6741 { 0x21f15b59 , 0xb9077a01 , 0x3e7c93b9 , },
6742
6743 { 0x6f68d64a , 0x901b0161 , 0xb9fd3537 , },
6744
6745 { 0x71b74d95 , 0xf5ddd5ad , 0x3e116c9d , },
6746
6747 { 0x4c2e7261 , 0x4ca19a29 , 0x388b20ac , },
6748
6749 { 0x8a2d38e8 , 0xd27ee0a1 , 0x408e57f2 , },
6750
6751 { 0x7e58ca17 , 0x69dfedd2 , 0x3a76805e , },
6752
6753 { 0xf997967f , 0x63c3d167 , 0x0956d953 , },
6754
6755 { 0x48215963 , 0x71e1dfe0 , 0x42a6d410 , },
6756
6757 { 0xa704b94c , 0x679f198a , 0x42ebf0ad , },
6758
6759 { 0x1d699056 , 0xfeacf2a1 , 0x55cb4dfe , },
6760
6761 { 0x6800bcc5 , 0x16024f15 , 0xcf3233e4 , },
6762
6763 { 0x2d48e4ca , 0xbe61582f , 0x46026283 , },
6764
6765 { 0x4c4c2b55 , 0x5539e44a , 0x52222fea , },
6766
6767 { 0xd8ce94cb , 0xbc613c26 , 0x33776b4b , },
6768
6769 { 0xd0b5a02b , 0x490d3cc6 , 0x2fde73f8 , },
6770
6771 { 0xa223f7ec , 0xf0baeeb6 , 0x0603989b , },
6772
6773 { 0x58de337a , 0x3bf3d597 , 0xced90d99 , },
6774
6775 { 0x37f5d8f4 , 0x4d5b699b , 0xd7262e5f , },
6776
6777 { 0xfa8a435d , 0x64f34a05 , 0x4470c029 , },
6778
6779 { 0x238709fe , 0x52e7458f , 0x9a174cd3 , },
6780
6781 { 0x9e1ba6f5 , 0xef0272f7 , 0x84f40beb , },
6782
6783 { 0xcd8b57fa , 0x82adb0b8 , 0xb6f35093 , },
6784
6785 { 0x0aed142f , 0xb1650290 , 0xec855937 , },
6786
6787 { 0xd1f064db , 0x6e7340d3 , 0x5c28cb52 , },
6788
6789 { 0x464ac895 , 0xd2fc57c3 , 0xc46805ba , },
6790
6791 { 0xa0e6beea , 0xcfeec3d0 , 0x0225d214 , },
6792
6793 { 0x78703ce0 , 0xc60f6075 , 0xdf7a24ac , },
6794
6795 { 0xfea48165 , 0x3a9d1f97 , 0xc3876592 , },
6796
6797 { 0xdb89b8db , 0xa6172211 , 0x2b52dc39 , },
6798
6799 { 0x7ca03731 , 0x1db42849 , 0xc5df246e , },
6800
6801 { 0x8801d0aa , 0xb57c7728 , 0x5b0c98b9 , },
6802
6803 { 0xf89cd7f0 , 0xcc396a0b , 0xdb799c51 , },
6804
6805 { 0x1611a808 , 0xaeae6105 , 0xb939fcdf , },
6806
6807 { 0xe3cdb888 , 0x26387824 , 0x30d13e5f , },
6808
6809 { 0x552a4cf6 , 0xee2d04bb , 0x70f9947d , },
6810
6811 { 0x85e248e9 , 0x0a79663f , 0x53339cf7 , },
6812
6813 { 0x1c61c3e9 , 0x8ea1f680 , 0x54afca53 , },
6814
6815 { 0xb14cfc2b , 0x2e073302 , 0x10897992 , },
6816
6817 { 0x6ec444cc , 0x9e819f13 , 0x7a3c0a6a , },
6818
6819 { 0xe2fa5f80 , 0x21f15b59 , 0x93102436 , },
6820
6821 { 0x6d33f4c6 , 0x31a27455 , 0x1fea4d2a , },
6822
6823 { 0xb6dec609 , 0x4d437056 , 0x42eb1e2a , },
6824
6825 { 0x1846c518 , 0x71b74d95 , 0xbd2655a8 , },
6826
6827 { 0x9f947f8a , 0x2b501619 , 0xa4924b0e , },
6828
6829 { 0xb7442f4d , 0xba30a5d8 , 0x4ff61aa1 , },
6830
6831 { 0xe2c93242 , 0x8a2d38e8 , 0x70cd7f26 , },
6832
6833 { 0xcd6863df , 0x78fd88dc , 0x7ae2f6f4 , },
6834
6835 { 0xd512001d , 0xe6612dff , 0x5c4d0ca9 , },
6836
6837 { 0x4e8d6b6c , 0xf997967f , 0x2d546c53 , },
6838
6839 { 0xfa653ba1 , 0xc99014d4 , 0xa0c9fd27 , },
6840
6841 { 0x49893408 , 0x29c2448b , 0xe0ee5efe , },
6842 };
6843  
6844  
6845 #define CRC32_FIXED_CHUNK_LEN 32768UL
6846 #define CRC32_FIXED_CHUNK_MULT_1 0x29c2448b
6847 #define CRC32_FIXED_CHUNK_MULT_2 0x4b912f53
6848 #define CRC32_FIXED_CHUNK_MULT_3 0x454c93be
6849  
6850 /* #include "crc32_tables.h" */
6851  
6852  
6853 static const u32 crc32_slice1_table[] MAYBE_UNUSED = {
6854 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
6855 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
6856 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
6857 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
6858 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
6859 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
6860 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
6861 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
6862 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
6863 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
6864 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
6865 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
6866 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
6867 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
6868 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
6869 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
6870 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
6871 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
6872 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
6873 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
6874 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
6875 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
6876 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
6877 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
6878 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
6879 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
6880 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
6881 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
6882 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
6883 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
6884 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
6885 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
6886 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
6887 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
6888 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
6889 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
6890 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
6891 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
6892 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
6893 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
6894 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
6895 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
6896 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
6897 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
6898 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
6899 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
6900 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
6901 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
6902 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
6903 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
6904 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
6905 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
6906 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
6907 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
6908 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
6909 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
6910 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
6911 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
6912 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
6913 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
6914 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
6915 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
6916 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
6917 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
6918 };
6919  
6920 static const u32 crc32_slice8_table[] MAYBE_UNUSED = {
6921 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
6922 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
6923 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
6924 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
6925 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
6926 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
6927 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
6928 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
6929 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
6930 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
6931 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
6932 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
6933 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
6934 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
6935 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
6936 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
6937 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
6938 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
6939 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
6940 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
6941 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
6942 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
6943 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
6944 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
6945 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
6946 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
6947 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
6948 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
6949 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
6950 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
6951 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
6952 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
6953 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
6954 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
6955 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
6956 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
6957 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
6958 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
6959 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
6960 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
6961 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
6962 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
6963 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
6964 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
6965 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
6966 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
6967 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
6968 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
6969 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
6970 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
6971 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
6972 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
6973 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
6974 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
6975 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
6976 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
6977 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
6978 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
6979 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
6980 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
6981 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
6982 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
6983 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
6984 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
6985 0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3,
6986 0x646cc504, 0x7d77f445, 0x565aa786, 0x4f4196c7,
6987 0xc8d98a08, 0xd1c2bb49, 0xfaefe88a, 0xe3f4d9cb,
6988 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e, 0x87981ccf,
6989 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192,
6990 0x2eaed755, 0x37b5e614, 0x1c98b5d7, 0x05838496,
6991 0x821b9859, 0x9b00a918, 0xb02dfadb, 0xa936cb9a,
6992 0xe6775d5d, 0xff6c6c1c, 0xd4413fdf, 0xcd5a0e9e,
6993 0x958424a2, 0x8c9f15e3, 0xa7b24620, 0xbea97761,
6994 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265,
6995 0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69,
6996 0x39316bae, 0x202a5aef, 0x0b07092c, 0x121c386d,
6997 0xdf4636f3, 0xc65d07b2, 0xed705471, 0xf46b6530,
6998 0xbb2af3f7, 0xa231c2b6, 0x891c9175, 0x9007a034,
6999 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38,
7000 0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c,
7001 0xf0794f05, 0xe9627e44, 0xc24f2d87, 0xdb541cc6,
7002 0x94158a01, 0x8d0ebb40, 0xa623e883, 0xbf38d9c2,
7003 0x38a0c50d, 0x21bbf44c, 0x0a96a78f, 0x138d96ce,
7004 0x5ccc0009, 0x45d73148, 0x6efa628b, 0x77e153ca,
7005 0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97,
7006 0xded79850, 0xc7cca911, 0xece1fad2, 0xf5facb93,
7007 0x7262d75c, 0x6b79e61d, 0x4054b5de, 0x594f849f,
7008 0x160e1258, 0x0f152319, 0x243870da, 0x3d23419b,
7009 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, 0x4ed03864,
7010 0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60,
7011 0xad24e1af, 0xb43fd0ee, 0x9f12832d, 0x8609b26c,
7012 0xc94824ab, 0xd05315ea, 0xfb7e4629, 0xe2657768,
7013 0x2f3f79f6, 0x362448b7, 0x1d091b74, 0x04122a35,
7014 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31,
7015 0xe7e6f3fe, 0xfefdc2bf, 0xd5d0917c, 0xcccba03d,
7016 0x838a36fa, 0x9a9107bb, 0xb1bc5478, 0xa8a76539,
7017 0x3b83984b, 0x2298a90a, 0x09b5fac9, 0x10aecb88,
7018 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd, 0x74c20e8c,
7019 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180,
7020 0x9736d747, 0x8e2de606, 0xa500b5c5, 0xbc1b8484,
7021 0x71418a1a, 0x685abb5b, 0x4377e898, 0x5a6cd9d9,
7022 0x152d4f1e, 0x0c367e5f, 0x271b2d9c, 0x3e001cdd,
7023 0xb9980012, 0xa0833153, 0x8bae6290, 0x92b553d1,
7024 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5,
7025 0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a,
7026 0xca6b79ed, 0xd37048ac, 0xf85d1b6f, 0xe1462a2e,
7027 0x66de36e1, 0x7fc507a0, 0x54e85463, 0x4df36522,
7028 0x02b2f3e5, 0x1ba9c2a4, 0x30849167, 0x299fa026,
7029 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b,
7030 0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f,
7031 0x2c1c24b0, 0x350715f1, 0x1e2a4632, 0x07317773,
7032 0x4870e1b4, 0x516bd0f5, 0x7a468336, 0x635db277,
7033 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc, 0xe0d7848d,
7034 0xaf96124a, 0xb68d230b, 0x9da070c8, 0x84bb4189,
7035 0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85,
7036 0x674f9842, 0x7e54a903, 0x5579fac0, 0x4c62cb81,
7037 0x8138c51f, 0x9823f45e, 0xb30ea79d, 0xaa1596dc,
7038 0xe554001b, 0xfc4f315a, 0xd7626299, 0xce7953d8,
7039 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4,
7040 0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0,
7041 0x5e7ef3ec, 0x4765c2ad, 0x6c48916e, 0x7553a02f,
7042 0x3a1236e8, 0x230907a9, 0x0824546a, 0x113f652b,
7043 0x96a779e4, 0x8fbc48a5, 0xa4911b66, 0xbd8a2a27,
7044 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23,
7045 0x14bce1bd, 0x0da7d0fc, 0x268a833f, 0x3f91b27e,
7046 0x70d024b9, 0x69cb15f8, 0x42e6463b, 0x5bfd777a,
7047 0xdc656bb5, 0xc57e5af4, 0xee530937, 0xf7483876,
7048 0xb809aeb1, 0xa1129ff0, 0x8a3fcc33, 0x9324fd72,
7049 0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59,
7050 0x0709a8dc, 0x06cbc2eb, 0x048d7cb2, 0x054f1685,
7051 0x0e1351b8, 0x0fd13b8f, 0x0d9785d6, 0x0c55efe1,
7052 0x091af964, 0x08d89353, 0x0a9e2d0a, 0x0b5c473d,
7053 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29,
7054 0x1b2f0bac, 0x1aed619b, 0x18abdfc2, 0x1969b5f5,
7055 0x1235f2c8, 0x13f798ff, 0x11b126a6, 0x10734c91,
7056 0x153c5a14, 0x14fe3023, 0x16b88e7a, 0x177ae44d,
7057 0x384d46e0, 0x398f2cd7, 0x3bc9928e, 0x3a0bf8b9,
7058 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065,
7059 0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901,
7060 0x3157bf84, 0x3095d5b3, 0x32d36bea, 0x331101dd,
7061 0x246be590, 0x25a98fa7, 0x27ef31fe, 0x262d5bc9,
7062 0x23624d4c, 0x22a0277b, 0x20e69922, 0x2124f315,
7063 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71,
7064 0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad,
7065 0x709a8dc0, 0x7158e7f7, 0x731e59ae, 0x72dc3399,
7066 0x7793251c, 0x76514f2b, 0x7417f172, 0x75d59b45,
7067 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816, 0x7ccf6221,
7068 0x798074a4, 0x78421e93, 0x7a04a0ca, 0x7bc6cafd,
7069 0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9,
7070 0x6bb5866c, 0x6a77ec5b, 0x68315202, 0x69f33835,
7071 0x62af7f08, 0x636d153f, 0x612bab66, 0x60e9c151,
7072 0x65a6d7d4, 0x6464bde3, 0x662203ba, 0x67e0698d,
7073 0x48d7cb20, 0x4915a117, 0x4b531f4e, 0x4a917579,
7074 0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5,
7075 0x46c49a98, 0x4706f0af, 0x45404ef6, 0x448224c1,
7076 0x41cd3244, 0x400f5873, 0x4249e62a, 0x438b8c1d,
7077 0x54f16850, 0x55330267, 0x5775bc3e, 0x56b7d609,
7078 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5,
7079 0x5ae239e8, 0x5b2053df, 0x5966ed86, 0x58a487b1,
7080 0x5deb9134, 0x5c29fb03, 0x5e6f455a, 0x5fad2f6d,
7081 0xe1351b80, 0xe0f771b7, 0xe2b1cfee, 0xe373a5d9,
7082 0xe63cb35c, 0xe7fed96b, 0xe5b86732, 0xe47a0d05,
7083 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461,
7084 0xe82fe2e4, 0xe9ed88d3, 0xebab368a, 0xea695cbd,
7085 0xfd13b8f0, 0xfcd1d2c7, 0xfe976c9e, 0xff5506a9,
7086 0xfa1a102c, 0xfbd87a1b, 0xf99ec442, 0xf85cae75,
7087 0xf300e948, 0xf2c2837f, 0xf0843d26, 0xf1465711,
7088 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd,
7089 0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339,
7090 0xde71f5bc, 0xdfb39f8b, 0xddf521d2, 0xdc374be5,
7091 0xd76b0cd8, 0xd6a966ef, 0xd4efd8b6, 0xd52db281,
7092 0xd062a404, 0xd1a0ce33, 0xd3e6706a, 0xd2241a5d,
7093 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049,
7094 0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895,
7095 0xcb4dafa8, 0xca8fc59f, 0xc8c97bc6, 0xc90b11f1,
7096 0xcc440774, 0xcd866d43, 0xcfc0d31a, 0xce02b92d,
7097 0x91af9640, 0x906dfc77, 0x922b422e, 0x93e92819,
7098 0x96a63e9c, 0x976454ab, 0x9522eaf2, 0x94e080c5,
7099 0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1,
7100 0x98b56f24, 0x99770513, 0x9b31bb4a, 0x9af3d17d,
7101 0x8d893530, 0x8c4b5f07, 0x8e0de15e, 0x8fcf8b69,
7102 0x8a809dec, 0x8b42f7db, 0x89044982, 0x88c623b5,
7103 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1,
7104 0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d,
7105 0xa9e2d0a0, 0xa820ba97, 0xaa6604ce, 0xaba46ef9,
7106 0xaeeb787c, 0xaf29124b, 0xad6fac12, 0xacadc625,
7107 0xa7f18118, 0xa633eb2f, 0xa4755576, 0xa5b73f41,
7108 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d,
7109 0xb5c473d0, 0xb40619e7, 0xb640a7be, 0xb782cd89,
7110 0xb2cddb0c, 0xb30fb13b, 0xb1490f62, 0xb08b6555,
7111 0xbbd72268, 0xba15485f, 0xb853f606, 0xb9919c31,
7112 0xbcde8ab4, 0xbd1ce083, 0xbf5a5eda, 0xbe9834ed,
7113 0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee,
7114 0x8f629757, 0x37def032, 0x256b5fdc, 0x9dd738b9,
7115 0xc5b428ef, 0x7d084f8a, 0x6fbde064, 0xd7018701,
7116 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733, 0x58631056,
7117 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871,
7118 0xdf7bc0c8, 0x67c7a7ad, 0x75720843, 0xcdce6f26,
7119 0x95ad7f70, 0x2d111815, 0x3fa4b7fb, 0x8718d09e,
7120 0x1acfe827, 0xa2738f42, 0xb0c620ac, 0x087a47c9,
7121 0xa032af3e, 0x188ec85b, 0x0a3b67b5, 0xb28700d0,
7122 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787,
7123 0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f,
7124 0xeae41086, 0x525877e3, 0x40edd80d, 0xf851bf68,
7125 0xf02bf8a1, 0x48979fc4, 0x5a22302a, 0xe29e574f,
7126 0x7f496ff6, 0xc7f50893, 0xd540a77d, 0x6dfcc018,
7127 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0,
7128 0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7,
7129 0x9b14583d, 0x23a83f58, 0x311d90b6, 0x89a1f7d3,
7130 0x1476cf6a, 0xaccaa80f, 0xbe7f07e1, 0x06c36084,
7131 0x5ea070d2, 0xe61c17b7, 0xf4a9b859, 0x4c15df3c,
7132 0xd1c2e785, 0x697e80e0, 0x7bcb2f0e, 0xc377486b,
7133 0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c,
7134 0x446f98f5, 0xfcd3ff90, 0xee66507e, 0x56da371b,
7135 0x0eb9274d, 0xb6054028, 0xa4b0efc6, 0x1c0c88a3,
7136 0x81dbb01a, 0x3967d77f, 0x2bd27891, 0x936e1ff4,
7137 0x3b26f703, 0x839a9066, 0x912f3f88, 0x299358ed,
7138 0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba,
7139 0xfe92dfec, 0x462eb889, 0x549b1767, 0xec277002,
7140 0x71f048bb, 0xc94c2fde, 0xdbf98030, 0x6345e755,
7141 0x6b3fa09c, 0xd383c7f9, 0xc1366817, 0x798a0f72,
7142 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825,
7143 0xae8b8873, 0x1637ef16, 0x048240f8, 0xbc3e279d,
7144 0x21e91f24, 0x99557841, 0x8be0d7af, 0x335cb0ca,
7145 0xed59b63b, 0x55e5d15e, 0x47507eb0, 0xffec19d5,
7146 0x623b216c, 0xda874609, 0xc832e9e7, 0x708e8e82,
7147 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a,
7148 0xa78f0983, 0x1f336ee6, 0x0d86c108, 0xb53aa66d,
7149 0xbd40e1a4, 0x05fc86c1, 0x1749292f, 0xaff54e4a,
7150 0x322276f3, 0x8a9e1196, 0x982bbe78, 0x2097d91d,
7151 0x78f4c94b, 0xc048ae2e, 0xd2fd01c0, 0x6a4166a5,
7152 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2,
7153 0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb,
7154 0xc2098e52, 0x7ab5e937, 0x680046d9, 0xd0bc21bc,
7155 0x88df31ea, 0x3063568f, 0x22d6f961, 0x9a6a9e04,
7156 0x07bda6bd, 0xbf01c1d8, 0xadb46e36, 0x15080953,
7157 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174,
7158 0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623,
7159 0xd8c66675, 0x607a0110, 0x72cfaefe, 0xca73c99b,
7160 0x57a4f122, 0xef189647, 0xfdad39a9, 0x45115ecc,
7161 0x764dee06, 0xcef18963, 0xdc44268d, 0x64f841e8,
7162 0xf92f7951, 0x41931e34, 0x5326b1da, 0xeb9ad6bf,
7163 0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907,
7164 0x3c9b51be, 0x842736db, 0x96929935, 0x2e2efe50,
7165 0x2654b999, 0x9ee8defc, 0x8c5d7112, 0x34e11677,
7166 0xa9362ece, 0x118a49ab, 0x033fe645, 0xbb838120,
7167 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98,
7168 0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf,
7169 0xd67f4138, 0x6ec3265d, 0x7c7689b3, 0xc4caeed6,
7170 0x591dd66f, 0xe1a1b10a, 0xf3141ee4, 0x4ba87981,
7171 0x13cb69d7, 0xab770eb2, 0xb9c2a15c, 0x017ec639,
7172 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e,
7173 0x866616a7, 0x3eda71c2, 0x2c6fde2c, 0x94d3b949,
7174 0x090481f0, 0xb1b8e695, 0xa30d497b, 0x1bb12e1e,
7175 0x43d23e48, 0xfb6e592d, 0xe9dbf6c3, 0x516791a6,
7176 0xccb0a91f, 0x740cce7a, 0x66b96194, 0xde0506f1,
7177 0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0,
7178 0xf580a6c0, 0xc8e08f70, 0x8f40f5a0, 0xb220dc10,
7179 0x30704bc1, 0x0d106271, 0x4ab018a1, 0x77d03111,
7180 0xc5f0ed01, 0xf890c4b1, 0xbf30be61, 0x825097d1,
7181 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52,
7182 0x95603142, 0xa80018f2, 0xefa06222, 0xd2c04b92,
7183 0x5090dc43, 0x6df0f5f3, 0x2a508f23, 0x1730a693,
7184 0xa5107a83, 0x98705333, 0xdfd029e3, 0xe2b00053,
7185 0xc1c12f04, 0xfca106b4, 0xbb017c64, 0x866155d4,
7186 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314,
7187 0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15,
7188 0x0431c205, 0x3951ebb5, 0x7ef19165, 0x4391b8d5,
7189 0xa121b886, 0x9c419136, 0xdbe1ebe6, 0xe681c256,
7190 0x54a11e46, 0x69c137f6, 0x2e614d26, 0x13016496,
7191 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997,
7192 0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57,
7193 0x58f35849, 0x659371f9, 0x22330b29, 0x1f532299,
7194 0xad73fe89, 0x9013d739, 0xd7b3ade9, 0xead38459,
7195 0x68831388, 0x55e33a38, 0x124340e8, 0x2f236958,
7196 0x9d03b548, 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98,
7197 0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b,
7198 0xcd93690b, 0xf0f340bb, 0xb7533a6b, 0x8a3313db,
7199 0x0863840a, 0x3503adba, 0x72a3d76a, 0x4fc3feda,
7200 0xfde322ca, 0xc0830b7a, 0x872371aa, 0xba43581a,
7201 0x9932774d, 0xa4525efd, 0xe3f2242d, 0xde920d9d,
7202 0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d,
7203 0xa9423c8c, 0x9422153c, 0xd3826fec, 0xeee2465c,
7204 0x5cc29a4c, 0x61a2b3fc, 0x2602c92c, 0x1b62e09c,
7205 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af, 0xbe729a1f,
7206 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf,
7207 0xc9a2ab0e, 0xf4c282be, 0xb362f86e, 0x8e02d1de,
7208 0x3c220dce, 0x0142247e, 0x46e25eae, 0x7b82771e,
7209 0xb1e6b092, 0x8c869922, 0xcb26e3f2, 0xf646ca42,
7210 0x44661652, 0x79063fe2, 0x3ea64532, 0x03c66c82,
7211 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183,
7212 0x74165d93, 0x49767423, 0x0ed60ef3, 0x33b62743,
7213 0xd1062710, 0xec660ea0, 0xabc67470, 0x96a65dc0,
7214 0x248681d0, 0x19e6a860, 0x5e46d2b0, 0x6326fb00,
7215 0xe1766cd1, 0xdc164561, 0x9bb63fb1, 0xa6d61601,
7216 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1,
7217 0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546,
7218 0x85a73956, 0xb8c710e6, 0xff676a36, 0xc2074386,
7219 0x4057d457, 0x7d37fde7, 0x3a978737, 0x07f7ae87,
7220 0xb5d77297, 0x88b75b27, 0xcf1721f7, 0xf2770847,
7221 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4,
7222 0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404,
7223 0x20b743d5, 0x1dd76a65, 0x5a7710b5, 0x67173905,
7224 0xd537e515, 0xe857cca5, 0xaff7b675, 0x92979fc5,
7225 0xe915e8db, 0xd475c16b, 0x93d5bbbb, 0xaeb5920b,
7226 0x1c954e1b, 0x21f567ab, 0x66551d7b, 0x5b3534cb,
7227 0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca,
7228 0x2ce505da, 0x11852c6a, 0x562556ba, 0x6b457f0a,
7229 0x89f57f59, 0xb49556e9, 0xf3352c39, 0xce550589,
7230 0x7c75d999, 0x4115f029, 0x06b58af9, 0x3bd5a349,
7231 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48,
7232 0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888,
7233 0x28d4c7df, 0x15b4ee6f, 0x521494bf, 0x6f74bd0f,
7234 0xdd54611f, 0xe03448af, 0xa794327f, 0x9af41bcf,
7235 0x18a48c1e, 0x25c4a5ae, 0x6264df7e, 0x5f04f6ce,
7236 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e,
7237 0x4834505d, 0x755479ed, 0x32f4033d, 0x0f942a8d,
7238 0xbdb4f69d, 0x80d4df2d, 0xc774a5fd, 0xfa148c4d,
7239 0x78441b9c, 0x4524322c, 0x028448fc, 0x3fe4614c,
7240 0x8dc4bd5c, 0xb0a494ec, 0xf704ee3c, 0xca64c78c,
7241 0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae,
7242 0x9b914216, 0x50cd91b3, 0xd659e31d, 0x1d0530b8,
7243 0xec53826d, 0x270f51c8, 0xa19b2366, 0x6ac7f0c3,
7244 0x77c2c07b, 0xbc9e13de, 0x3a0a6170, 0xf156b2d5,
7245 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035,
7246 0x9847408d, 0x531b9328, 0xd58fe186, 0x1ed33223,
7247 0xef8580f6, 0x24d95353, 0xa24d21fd, 0x6911f258,
7248 0x7414c2e0, 0xbf481145, 0x39dc63eb, 0xf280b04e,
7249 0x07ac0536, 0xccf0d693, 0x4a64a43d, 0x81387798,
7250 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e,
7251 0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5,
7252 0x706ec54d, 0xbb3216e8, 0x3da66446, 0xf6fab7e3,
7253 0x047a07ad, 0xcf26d408, 0x49b2a6a6, 0x82ee7503,
7254 0x9feb45bb, 0x54b7961e, 0xd223e4b0, 0x197f3715,
7255 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e,
7256 0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578,
7257 0x0f580a6c, 0xc404d9c9, 0x4290ab67, 0x89cc78c2,
7258 0x94c9487a, 0x5f959bdf, 0xd901e971, 0x125d3ad4,
7259 0xe30b8801, 0x28575ba4, 0xaec3290a, 0x659ffaaf,
7260 0x789aca17, 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9,
7261 0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59,
7262 0x971f4ae1, 0x5c439944, 0xdad7ebea, 0x118b384f,
7263 0xe0dd8a9a, 0x2b81593f, 0xad152b91, 0x6649f834,
7264 0x7b4cc88c, 0xb0101b29, 0x36846987, 0xfdd8ba22,
7265 0x08f40f5a, 0xc3a8dcff, 0x453cae51, 0x8e607df4,
7266 0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2,
7267 0xe4a78d37, 0x2ffb5e92, 0xa96f2c3c, 0x6233ff99,
7268 0x7f36cf21, 0xb46a1c84, 0x32fe6e2a, 0xf9a2bd8f,
7269 0x0b220dc1, 0xc07ede64, 0x46eaacca, 0x8db67f6f,
7270 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79,
7271 0xe7718fac, 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02,
7272 0x7ce0cdba, 0xb7bc1e1f, 0x31286cb1, 0xfa74bf14,
7273 0x1eb014d8, 0xd5ecc77d, 0x5378b5d3, 0x98246676,
7274 0x852156ce, 0x4e7d856b, 0xc8e9f7c5, 0x03b52460,
7275 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b,
7276 0x6972d4a3, 0xa22e0706, 0x24ba75a8, 0xefe6a60d,
7277 0x1d661643, 0xd63ac5e6, 0x50aeb748, 0x9bf264ed,
7278 0x86f75455, 0x4dab87f0, 0xcb3ff55e, 0x006326fb,
7279 0xf135942e, 0x3a69478b, 0xbcfd3525, 0x77a1e680,
7280 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496,
7281 0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340,
7282 0x828d53f8, 0x49d1805d, 0xcf45f2f3, 0x04192156,
7283 0xf54f9383, 0x3e134026, 0xb8873288, 0x73dbe12d,
7284 0x6eded195, 0xa5820230, 0x2316709e, 0xe84aa33b,
7285 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db,
7286 0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd,
7287 0xf6999118, 0x3dc542bd, 0xbb513013, 0x700de3b6,
7288 0x6d08d30e, 0xa65400ab, 0x20c07205, 0xeb9ca1a0,
7289 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf, 0x977c6c1a,
7290 0x8a795ca2, 0x41258f07, 0xc7b1fda9, 0x0ced2e0c,
7291 0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77,
7292 0x662adecf, 0xad760d6a, 0x2be27fc4, 0xe0beac61,
7293 0x123e1c2f, 0xd962cf8a, 0x5ff6bd24, 0x94aa6e81,
7294 0x89af5e39, 0x42f38d9c, 0xc467ff32, 0x0f3b2c97,
7295 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec,
7296 0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa,
7297 0x16441b82, 0xdd18c827, 0x5b8cba89, 0x90d0692c,
7298 0x8dd55994, 0x46898a31, 0xc01df89f, 0x0b412b3a,
7299 0xfa1799ef, 0x314b4a4a, 0xb7df38e4, 0x7c83eb41,
7300 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957,
7301 0x15921919, 0xdececabc, 0x585ab812, 0x93066bb7,
7302 0x8e035b0f, 0x455f88aa, 0xc3cbfa04, 0x089729a1,
7303 0xf9c19b74, 0x329d48d1, 0xb4093a7f, 0x7f55e9da,
7304 0x6250d962, 0xa90c0ac7, 0x2f987869, 0xe4c4abcc,
7305 0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d,
7306 0xf44f2413, 0x52382fa7, 0x63d0353a, 0xc5a73e8e,
7307 0x33ef4e67, 0x959845d3, 0xa4705f4e, 0x020754fa,
7308 0xc7a06a74, 0x61d761c0, 0x503f7b5d, 0xf64870e9,
7309 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653,
7310 0x9391b8dd, 0x35e6b369, 0x040ea9f4, 0xa279a240,
7311 0x5431d2a9, 0xf246d91d, 0xc3aec380, 0x65d9c834,
7312 0xa07ef6ba, 0x0609fd0e, 0x37e1e793, 0x9196ec27,
7313 0xcfbd399c, 0x69ca3228, 0x582228b5, 0xfe552301,
7314 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712,
7315 0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66,
7316 0x081d53e8, 0xae6a585c, 0x9f8242c1, 0x39f54975,
7317 0xa863a552, 0x0e14aee6, 0x3ffcb47b, 0x998bbfcf,
7318 0x5c2c8141, 0xfa5b8af5, 0xcbb39068, 0x6dc49bdc,
7319 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8,
7320 0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb,
7321 0x440b7579, 0xe27c7ecd, 0xd3946450, 0x75e36fe4,
7322 0xb044516a, 0x16335ade, 0x27db4043, 0x81ac4bf7,
7323 0x77e43b1e, 0xd19330aa, 0xe07b2a37, 0x460c2183,
7324 0x83ab1f0d, 0x25dc14b9, 0x14340e24, 0xb2430590,
7325 0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a,
7326 0xd79acda4, 0x71edc610, 0x4005dc8d, 0xe672d739,
7327 0x103aa7d0, 0xb64dac64, 0x87a5b6f9, 0x21d2bd4d,
7328 0xe47583c3, 0x42028877, 0x73ea92ea, 0xd59d995e,
7329 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, 0xba5e5678,
7330 0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b,
7331 0xb8590282, 0x1e2e0936, 0x2fc613ab, 0x89b1181f,
7332 0x4c162691, 0xea612d25, 0xdb8937b8, 0x7dfe3c0c,
7333 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102, 0xdd80cab6,
7334 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5,
7335 0xdf879e4c, 0x79f095f8, 0x48188f65, 0xee6f84d1,
7336 0x2bc8ba5f, 0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2,
7337 0x8816eaf2, 0x2e61e146, 0x1f89fbdb, 0xb9fef06f,
7338 0x7c59cee1, 0xda2ec555, 0xebc6dfc8, 0x4db1d47c,
7339 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08,
7340 0x4fb68086, 0xe9c18b32, 0xd82991af, 0x7e5e9a1b,
7341 0xefc8763c, 0x49bf7d88, 0x78576715, 0xde206ca1,
7342 0x1b87522f, 0xbdf0599b, 0x8c184306, 0x2a6f48b2,
7343 0xdc27385b, 0x7a5033ef, 0x4bb82972, 0xedcf22c6,
7344 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5,
7345 0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3,
7346 0xb3e4f77d, 0x1593fcc9, 0x247be654, 0x820cede0,
7347 0x74449d09, 0xd23396bd, 0xe3db8c20, 0x45ac8794,
7348 0x800bb91a, 0x267cb2ae, 0x1794a833, 0xb1e3a387,
7349 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d,
7350 0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e,
7351 0x139a01c7, 0xb5ed0a73, 0x840510ee, 0x22721b5a,
7352 0xe7d525d4, 0x41a22e60, 0x704a34fd, 0xd63d3f49,
7353 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2, 0xfdf58516,
7354 0x3852bb98, 0x9e25b02c, 0xafcdaab1, 0x09baa105,
7355 0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71,
7356 0x0bbdf5ff, 0xadcafe4b, 0x9c22e4d6, 0x3a55ef62,
7357 0xabc30345, 0x0db408f1, 0x3c5c126c, 0x9a2b19d8,
7358 0x5f8c2756, 0xf9fb2ce2, 0xc813367f, 0x6e643dcb,
7359 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf,
7360 0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac,
7361 0x03a0a617, 0xa5d7ada3, 0x943fb73e, 0x3248bc8a,
7362 0xf7ef8204, 0x519889b0, 0x6070932d, 0xc6079899,
7363 0x304fe870, 0x9638e3c4, 0xa7d0f959, 0x01a7f2ed,
7364 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe,
7365 0x647e3ad9, 0xc209316d, 0xf3e12bf0, 0x55962044,
7366 0x90311eca, 0x3646157e, 0x07ae0fe3, 0xa1d90457,
7367 0x579174be, 0xf1e67f0a, 0xc00e6597, 0x66796e23,
7368 0xa3de50ad, 0x05a95b19, 0x34414184, 0x92364a30,
7369 0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3,
7370 0x844a0efa, 0x48e00e64, 0xc66f0987, 0x0ac50919,
7371 0xd3e51bb5, 0x1f4f1b2b, 0x91c01cc8, 0x5d6a1c56,
7372 0x57af154f, 0x9b0515d1, 0x158a1232, 0xd92012ac,
7373 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8,
7374 0xf8f13fd1, 0x345b3f4f, 0xbad438ac, 0x767e3832,
7375 0xaf5e2a9e, 0x63f42a00, 0xed7b2de3, 0x21d12d7d,
7376 0x2b142464, 0xe7be24fa, 0x69312319, 0xa59b2387,
7377 0xf9766256, 0x35dc62c8, 0xbb53652b, 0x77f965b5,
7378 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f,
7379 0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00,
7380 0xaed97719, 0x62737787, 0xecfc7064, 0x205670fa,
7381 0x85cd537d, 0x496753e3, 0xc7e85400, 0x0b42549e,
7382 0x01875d87, 0xcd2d5d19, 0x43a25afa, 0x8f085a64,
7383 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b,
7384 0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1,
7385 0x299dc2ed, 0xe537c273, 0x6bb8c590, 0xa712c50e,
7386 0xadd7cc17, 0x617dcc89, 0xeff2cb6a, 0x2358cbf4,
7387 0xfa78d958, 0x36d2d9c6, 0xb85dde25, 0x74f7debb,
7388 0x7e32d7a2, 0xb298d73c, 0x3c17d0df, 0xf0bdd041,
7389 0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425,
7390 0xd16cfd3c, 0x1dc6fda2, 0x9349fa41, 0x5fe3fadf,
7391 0x86c3e873, 0x4a69e8ed, 0xc4e6ef0e, 0x084cef90,
7392 0x0289e689, 0xce23e617, 0x40ace1f4, 0x8c06e16a,
7393 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, 0x5e64a758,
7394 0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2,
7395 0x030ebb0e, 0xcfa4bb90, 0x412bbc73, 0x8d81bced,
7396 0x8744b5f4, 0x4beeb56a, 0xc561b289, 0x09cbb217,
7397 0xac509190, 0x60fa910e, 0xee7596ed, 0x22df9673,
7398 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889,
7399 0x7fb58a25, 0xb31f8abb, 0x3d908d58, 0xf13a8dc6,
7400 0xfbff84df, 0x37558441, 0xb9da83a2, 0x7570833c,
7401 0x533b85da, 0x9f918544, 0x111e82a7, 0xddb48239,
7402 0xd7718b20, 0x1bdb8bbe, 0x95548c5d, 0x59fe8cc3,
7403 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c,
7404 0x04949095, 0xc83e900b, 0x46b197e8, 0x8a1b9776,
7405 0x2f80b4f1, 0xe32ab46f, 0x6da5b38c, 0xa10fb312,
7406 0xabcaba0b, 0x6760ba95, 0xe9efbd76, 0x2545bde8,
7407 0xfc65af44, 0x30cfafda, 0xbe40a839, 0x72eaa8a7,
7408 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d,
7409 0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f,
7410 0x2e07e976, 0xe2ade9e8, 0x6c22ee0b, 0xa088ee95,
7411 0x79a8fc39, 0xb502fca7, 0x3b8dfb44, 0xf727fbda,
7412 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be, 0x736df520,
7413 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144,
7414 0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe,
7415 0x0513cd12, 0xc9b9cd8c, 0x4736ca6f, 0x8b9ccaf1,
7416 0x8159c3e8, 0x4df3c376, 0xc37cc495, 0x0fd6c40b,
7417 0x7aa64737, 0xb60c47a9, 0x3883404a, 0xf42940d4,
7418 0xfeec49cd, 0x32464953, 0xbcc94eb0, 0x70634e2e,
7419 0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61,
7420 0x2d095278, 0xe1a352e6, 0x6f2c5505, 0xa386559b,
7421 0x061d761c, 0xcab77682, 0x44387161, 0x889271ff,
7422 0x825778e6, 0x4efd7878, 0xc0727f9b, 0x0cd87f05,
7423 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a,
7424 0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0,
7425 0x83d02561, 0x4f7a25ff, 0xc1f5221c, 0x0d5f2282,
7426 0x079a2b9b, 0xcb302b05, 0x45bf2ce6, 0x89152c78,
7427 0x50353ed4, 0x9c9f3e4a, 0x121039a9, 0xdeba3937,
7428 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd,
7429 0xff6b144a, 0x33c114d4, 0xbd4e1337, 0x71e413a9,
7430 0x7b211ab0, 0xb78b1a2e, 0x39041dcd, 0xf5ae1d53,
7431 0x2c8e0fff, 0xe0240f61, 0x6eab0882, 0xa201081c,
7432 0xa8c40105, 0x646e019b, 0xeae10678, 0x264b06e6,
7433 };
7434  
7435  
7436  
7437 static u32 MAYBE_UNUSED
7438 0 crc32_slice8(u32 crc, const u8 *p, size_t len)
7439 {
7440 0 const u8 * const end = p + len;
7441 const u8 *end64;
7442  
7443 0 ▶ for (; ((uintptr_t)p & 7) && p != end; p++)
Branch: 2 branches
BranchTrueFalseTotal
for (; ((uintptr_t)p & 7) && p != end; p++) 0 0 0
for (; ((uintptr_t)p & 7) && p != end; p++) 0 0 0
7444 0 crc = (crc >> 8) ^ crc32_slice8_table[(u8)crc ^ *p];
7445  
7446 0 end64 = p + ((end - p) & ~7);
7447 0 ▶ for (; p != end64; p += 8) {
Branch: 1 branch
BranchTrueFalseTotal
for (; p != end64; p += 8) { 0 0 0
7448 0 u32 v1 = le32_bswap(*(const u32 *)(p + 0));
7449 0 u32 v2 = le32_bswap(*(const u32 *)(p + 4));
7450  
7451 0 crc = crc32_slice8_table[0x700 + (u8)((crc ^ v1) >> 0)] ^
7452 0 crc32_slice8_table[0x600 + (u8)((crc ^ v1) >> 8)] ^
7453 0 crc32_slice8_table[0x500 + (u8)((crc ^ v1) >> 16)] ^
7454 0 crc32_slice8_table[0x400 + (u8)((crc ^ v1) >> 24)] ^
7455 0 crc32_slice8_table[0x300 + (u8)(v2 >> 0)] ^
7456 0 crc32_slice8_table[0x200 + (u8)(v2 >> 8)] ^
7457 0 crc32_slice8_table[0x100 + (u8)(v2 >> 16)] ^
7458 0 crc32_slice8_table[0x000 + (u8)(v2 >> 24)];
7459 }
7460  
7461 0 ▶ for (; p != end; p++)
Branch: 1 branch
BranchTrueFalseTotal
for (; p != end; p++) 0 0 0
7462 0 crc = (crc >> 8) ^ crc32_slice8_table[(u8)crc ^ *p];
7463  
7464 0 return crc;
7465 }
7466  
7467  
7468 static forceinline u32 MAYBE_UNUSED
7469 crc32_slice1(u32 crc, const u8 *p, size_t len)
7470 {
7471 size_t i;
7472  
7473 0 ▶ for (i = 0; i < len; i++)
Branch: 5 branches
BranchTrueFalseTotal
for (i = 0; i < len; i++) 0 0 0
for (i = 0; i < len; i++) 0 0 0
for (i = 0; i < len; i++) 0 0 0
for (i = 0; i < len; i++) 0 0 0
for (i = 0; i < len; i++) 0 0 0
7474 0 crc = (crc >> 8) ^ crc32_slice1_table[(u8)crc ^ p[i]];
7475 0 return crc;
7476 }
7477  
7478  
7479 #undef DEFAULT_IMPL
7480 #undef arch_select_crc32_func
7481 typedef u32 (*crc32_func_t)(u32 crc, const u8 *p, size_t len);
7482 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
7483 /* # include "arm/crc32_impl.h" */
7484  
7485  
7486 #ifndef LIB_ARM_CRC32_IMPL_H
7487 #define LIB_ARM_CRC32_IMPL_H
7488  
7489 /* #include "arm-cpu_features.h" */
7490  
7491  
7492 #ifndef LIB_ARM_CPU_FEATURES_H
7493 #define LIB_ARM_CPU_FEATURES_H
7494  
7495 /* #include "lib_common.h" */
7496  
7497  
7498 #ifndef LIB_LIB_COMMON_H
7499 #define LIB_LIB_COMMON_H
7500  
7501 #ifdef LIBDEFLATE_H
7502
7503 # error "lib_common.h must always be included before libdeflate.h"
7504 #endif
7505  
7506 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
7507 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
7508 #elif defined(__GNUC__)
7509 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
7510 #else
7511 # define LIBDEFLATE_EXPORT_SYM
7512 #endif
7513  
7514  
7515 #if defined(__GNUC__) && defined(__i386__)
7516 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
7517 #else
7518 # define LIBDEFLATE_ALIGN_STACK
7519 #endif
7520  
7521 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
7522  
7523 /* #include "../common_defs.h" */
7524  
7525  
7526 #ifndef COMMON_DEFS_H
7527 #define COMMON_DEFS_H
7528  
7529 /* #include "libdeflate.h" */
7530  
7531  
7532 #ifndef LIBDEFLATE_H
7533 #define LIBDEFLATE_H
7534  
7535 #include <stddef.h>
7536 #include <stdint.h>
7537  
7538 #ifdef __cplusplus
7539 extern "C" {
7540 #endif
7541  
7542 #define LIBDEFLATE_VERSION_MAJOR 1
7543 #define LIBDEFLATE_VERSION_MINOR 26
7544 #define LIBDEFLATE_VERSION_STRING "1.26"
7545  
7546  
7547 #ifndef LIBDEFLATEAPI
7548 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
7549 # define LIBDEFLATEAPI __declspec(dllimport)
7550 # else
7551 # define LIBDEFLATEAPI
7552 # endif
7553 #endif
7554  
7555  
7556  
7557  
7558  
7559 struct libdeflate_compressor;
7560 struct libdeflate_options;
7561  
7562  
7563 LIBDEFLATEAPI struct libdeflate_compressor *
7564 libdeflate_alloc_compressor(int compression_level);
7565  
7566  
7567 LIBDEFLATEAPI struct libdeflate_compressor *
7568 libdeflate_alloc_compressor_ex(int compression_level,
7569 const struct libdeflate_options *options);
7570  
7571  
7572 LIBDEFLATEAPI size_t
7573 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
7574 const void *in, size_t in_nbytes,
7575 void *out, size_t out_nbytes_avail);
7576  
7577  
7578 LIBDEFLATEAPI size_t
7579 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
7580 size_t in_nbytes);
7581  
7582  
7583 LIBDEFLATEAPI size_t
7584 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
7585 const void *in, size_t in_nbytes,
7586 void *out, size_t out_nbytes_avail);
7587  
7588  
7589 LIBDEFLATEAPI size_t
7590 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
7591 size_t in_nbytes);
7592  
7593  
7594 LIBDEFLATEAPI size_t
7595 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
7596 const void *in, size_t in_nbytes,
7597 void *out, size_t out_nbytes_avail);
7598  
7599  
7600 LIBDEFLATEAPI size_t
7601 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
7602 size_t in_nbytes);
7603  
7604  
7605 LIBDEFLATEAPI void
7606 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
7607  
7608  
7609  
7610  
7611  
7612 struct libdeflate_decompressor;
7613 struct libdeflate_options;
7614  
7615  
7616 LIBDEFLATEAPI struct libdeflate_decompressor *
7617 libdeflate_alloc_decompressor(void);
7618  
7619  
7620 LIBDEFLATEAPI struct libdeflate_decompressor *
7621 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
7622  
7623  
7624 enum libdeflate_result {
7625
7626 LIBDEFLATE_SUCCESS = 0,
7627  
7628
7629 LIBDEFLATE_BAD_DATA = 1,
7630  
7631
7632 LIBDEFLATE_SHORT_OUTPUT = 2,
7633  
7634
7635 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
7636 };
7637  
7638  
7639 LIBDEFLATEAPI enum libdeflate_result
7640 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
7641 const void *in, size_t in_nbytes,
7642 void *out, size_t out_nbytes_avail,
7643 size_t *actual_out_nbytes_ret);
7644  
7645  
7646 LIBDEFLATEAPI enum libdeflate_result
7647 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
7648 const void *in, size_t in_nbytes,
7649 void *out, size_t out_nbytes_avail,
7650 size_t *actual_in_nbytes_ret,
7651 size_t *actual_out_nbytes_ret);
7652  
7653  
7654 LIBDEFLATEAPI enum libdeflate_result
7655 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
7656 const void *in, size_t in_nbytes,
7657 void *out, size_t out_nbytes_avail,
7658 size_t *actual_out_nbytes_ret);
7659  
7660  
7661 LIBDEFLATEAPI enum libdeflate_result
7662 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
7663 const void *in, size_t in_nbytes,
7664 void *out, size_t out_nbytes_avail,
7665 size_t *actual_in_nbytes_ret,
7666 size_t *actual_out_nbytes_ret);
7667  
7668  
7669 LIBDEFLATEAPI enum libdeflate_result
7670 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
7671 const void *in, size_t in_nbytes,
7672 void *out, size_t out_nbytes_avail,
7673 size_t *actual_out_nbytes_ret);
7674  
7675  
7676 LIBDEFLATEAPI enum libdeflate_result
7677 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
7678 const void *in, size_t in_nbytes,
7679 void *out, size_t out_nbytes_avail,
7680 size_t *actual_in_nbytes_ret,
7681 size_t *actual_out_nbytes_ret);
7682  
7683  
7684 LIBDEFLATEAPI void
7685 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
7686  
7687  
7688  
7689  
7690  
7691  
7692 LIBDEFLATEAPI uint32_t
7693 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
7694  
7695  
7696  
7697 LIBDEFLATEAPI uint32_t
7698 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
7699  
7700  
7701  
7702  
7703  
7704  
7705 LIBDEFLATEAPI void
7706 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
7707 void (*free_func)(void *));
7708  
7709  
7710 struct libdeflate_options {
7711  
7712
7713 size_t sizeof_options;
7714  
7715
7716 void *(*malloc_func)(size_t);
7717 void (*free_func)(void *);
7718 };
7719  
7720 #ifdef __cplusplus
7721 }
7722 #endif
7723  
7724 #endif
7725  
7726  
7727 #include <stdbool.h>
7728 #include <stddef.h>
7729 #include <stdint.h>
7730 #ifdef _MSC_VER
7731 # include <intrin.h>
7732 # include <stdlib.h>
7733
7734
7735 # pragma warning(disable : 4146)
7736
7737 # pragma warning(disable : 4018)
7738 # pragma warning(disable : 4244)
7739 # pragma warning(disable : 4267)
7740 # pragma warning(disable : 4310)
7741
7742 # pragma warning(disable : 4100)
7743 # pragma warning(disable : 4127)
7744 # pragma warning(disable : 4189)
7745 # pragma warning(disable : 4232)
7746 # pragma warning(disable : 4245)
7747 # pragma warning(disable : 4295)
7748 #endif
7749 #ifndef FREESTANDING
7750 # include <string.h>
7751 #endif
7752  
7753  
7754  
7755  
7756  
7757  
7758 #undef ARCH_X86_64
7759 #undef ARCH_X86_32
7760 #undef ARCH_ARM64
7761 #undef ARCH_ARM32
7762 #undef ARCH_RISCV
7763 #ifdef _MSC_VER
7764
7765 # if defined(_M_X64) && !defined(_M_ARM64EC)
7766 # define ARCH_X86_64
7767 # elif defined(_M_IX86)
7768 # define ARCH_X86_32
7769 # elif defined(_M_ARM64)
7770 # define ARCH_ARM64
7771 # elif defined(_M_ARM)
7772 # define ARCH_ARM32
7773 # endif
7774 #else
7775 # if defined(__x86_64__)
7776 # define ARCH_X86_64
7777 # elif defined(__i386__)
7778 # define ARCH_X86_32
7779 # elif defined(__aarch64__)
7780 # define ARCH_ARM64
7781 # elif defined(__arm__)
7782 # define ARCH_ARM32
7783 # elif defined(__riscv)
7784 # define ARCH_RISCV
7785 # endif
7786 #endif
7787  
7788  
7789  
7790  
7791  
7792  
7793 typedef uint8_t u8;
7794 typedef uint16_t u16;
7795 typedef uint32_t u32;
7796 typedef uint64_t u64;
7797 typedef int8_t s8;
7798 typedef int16_t s16;
7799 typedef int32_t s32;
7800 typedef int64_t s64;
7801  
7802  
7803 #ifdef _MSC_VER
7804 # ifdef _WIN64
7805 typedef long long ssize_t;
7806 # else
7807 typedef long ssize_t;
7808 # endif
7809 #endif
7810  
7811  
7812 typedef size_t machine_word_t;
7813  
7814  
7815 #define WORDBYTES ((int)sizeof(machine_word_t))
7816  
7817  
7818 #define WORDBITS (8 * WORDBYTES)
7819  
7820  
7821  
7822  
7823  
7824  
7825 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
7826 # define GCC_PREREQ(major, minor) \
7827 (__GNUC__ > (major) || \
7828 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
7829 # if !GCC_PREREQ(4, 9)
7830 # error "gcc versions older than 4.9 are no longer supported"
7831 # endif
7832 #else
7833 # define GCC_PREREQ(major, minor) 0
7834 #endif
7835 #ifdef __clang__
7836 # ifdef __apple_build_version__
7837 # define CLANG_PREREQ(major, minor, apple_version) \
7838 (__apple_build_version__ >= (apple_version))
7839 # else
7840 # define CLANG_PREREQ(major, minor, apple_version) \
7841 (__clang_major__ > (major) || \
7842 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
7843 # endif
7844 # if !CLANG_PREREQ(3, 9, 8000000)
7845 # error "clang versions older than 3.9 are no longer supported"
7846 # endif
7847 #else
7848 # define CLANG_PREREQ(major, minor, apple_version) 0
7849 #endif
7850 #ifdef _MSC_VER
7851 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
7852 # if !MSVC_PREREQ(1928)
7853 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
7854 # endif
7855 #else
7856 # define MSVC_PREREQ(version) 0
7857 #endif
7858  
7859  
7860 #ifndef __has_attribute
7861 # define __has_attribute(attribute) 0
7862 #endif
7863  
7864  
7865 #ifndef __has_builtin
7866 # define __has_builtin(builtin) 0
7867 #endif
7868  
7869  
7870 #ifdef _MSC_VER
7871 # define inline __inline
7872 #endif
7873  
7874  
7875 #if defined(__GNUC__) || __has_attribute(always_inline)
7876 # define forceinline inline __attribute__((always_inline))
7877 #elif defined(_MSC_VER)
7878 # define forceinline __forceinline
7879 #else
7880 # define forceinline inline
7881 #endif
7882  
7883  
7884 #if defined(__GNUC__) || __has_attribute(unused)
7885 # define MAYBE_UNUSED __attribute__((unused))
7886 #else
7887 # define MAYBE_UNUSED
7888 #endif
7889  
7890  
7891 #if defined(__GNUC__) || __has_attribute(noreturn)
7892 # define NORETURN __attribute__((noreturn))
7893 #else
7894 # define NORETURN
7895 #endif
7896  
7897  
7898 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
7899 # define likely(expr) __builtin_expect(!!(expr), 1)
7900 #else
7901 # define likely(expr) (expr)
7902 #endif
7903  
7904  
7905 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
7906 # define unlikely(expr) __builtin_expect(!!(expr), 0)
7907 #else
7908 # define unlikely(expr) (expr)
7909 #endif
7910  
7911  
7912 #undef prefetchr
7913 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
7914 # define prefetchr(addr) __builtin_prefetch((addr), 0)
7915 #elif defined(_MSC_VER)
7916 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
7917 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
7918 # elif defined(ARCH_ARM64)
7919 # define prefetchr(addr) __prefetch2((addr), 0x00 )
7920 # elif defined(ARCH_ARM32)
7921 # define prefetchr(addr) __prefetch(addr)
7922 # endif
7923 #endif
7924 #ifndef prefetchr
7925 # define prefetchr(addr)
7926 #endif
7927  
7928  
7929 #undef prefetchw
7930 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
7931 # define prefetchw(addr) __builtin_prefetch((addr), 1)
7932 #elif defined(_MSC_VER)
7933 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
7934 # define prefetchw(addr) _m_prefetchw(addr)
7935 # elif defined(ARCH_ARM64)
7936 # define prefetchw(addr) __prefetch2((addr), 0x10 )
7937 # elif defined(ARCH_ARM32)
7938 # define prefetchw(addr) __prefetchw(addr)
7939 # endif
7940 #endif
7941 #ifndef prefetchw
7942 # define prefetchw(addr)
7943 #endif
7944  
7945  
7946 #undef _aligned_attribute
7947 #if defined(__GNUC__) || __has_attribute(aligned)
7948 # define _aligned_attribute(n) __attribute__((aligned(n)))
7949 #elif defined(_MSC_VER)
7950 # define _aligned_attribute(n) __declspec(align(n))
7951 #endif
7952  
7953  
7954 #if defined(__GNUC__) || __has_attribute(target)
7955 # define _target_attribute(attrs) __attribute__((target(attrs)))
7956 #else
7957 # define _target_attribute(attrs)
7958 #endif
7959  
7960  
7961  
7962  
7963  
7964 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
7965 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
7966 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
7967 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
7968 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
7969 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
7970 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
7971  
7972  
7973  
7974  
7975  
7976  
7977 #if defined(__BYTE_ORDER__)
7978 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
7979 #elif defined(_MSC_VER)
7980 # define CPU_IS_LITTLE_ENDIAN() true
7981 #else
7982 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
7983 {
7984 union {
7985 u32 w;
7986 u8 b;
7987 } u;
7988  
7989 u.w = 1;
7990 return u.b;
7991 }
7992 #endif
7993  
7994  
7995 static forceinline u16 bswap16(u16 v)
7996 {
7997 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
7998 return __builtin_bswap16(v);
7999 #elif defined(_MSC_VER)
8000 return _byteswap_ushort(v);
8001 #else
8002 return (v << 8) | (v >> 8);
8003 #endif
8004 }
8005  
8006  
8007 static forceinline u32 bswap32(u32 v)
8008 {
8009 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
8010 return __builtin_bswap32(v);
8011 #elif defined(_MSC_VER)
8012 return _byteswap_ulong(v);
8013 #else
8014 return ((v & 0x000000FF) << 24) |
8015 ((v & 0x0000FF00) << 8) |
8016 ((v & 0x00FF0000) >> 8) |
8017 ((v & 0xFF000000) >> 24);
8018 #endif
8019 }
8020  
8021  
8022 static forceinline u64 bswap64(u64 v)
8023 {
8024 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
8025 return __builtin_bswap64(v);
8026 #elif defined(_MSC_VER)
8027 return _byteswap_uint64(v);
8028 #else
8029 return ((v & 0x00000000000000FF) << 56) |
8030 ((v & 0x000000000000FF00) << 40) |
8031 ((v & 0x0000000000FF0000) << 24) |
8032 ((v & 0x00000000FF000000) << 8) |
8033 ((v & 0x000000FF00000000) >> 8) |
8034 ((v & 0x0000FF0000000000) >> 24) |
8035 ((v & 0x00FF000000000000) >> 40) |
8036 ((v & 0xFF00000000000000) >> 56);
8037 #endif
8038 }
8039  
8040 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
8041 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
8042 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
8043 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
8044 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
8045 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
8046  
8047  
8048  
8049  
8050  
8051  
8052 #if (defined(__GNUC__) || defined(__clang__)) && \
8053 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
8054 defined(__ARM_FEATURE_UNALIGNED) || \
8055 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
8056 defined(__riscv_misaligned_fast) || \
8057 defined(__wasm__))
8058 # define UNALIGNED_ACCESS_IS_FAST 1
8059 #elif defined(_MSC_VER)
8060 # define UNALIGNED_ACCESS_IS_FAST 1
8061 #else
8062 # define UNALIGNED_ACCESS_IS_FAST 0
8063 #endif
8064  
8065  
8066  
8067 #ifdef FREESTANDING
8068 # define MEMCOPY __builtin_memcpy
8069 #else
8070 # define MEMCOPY memcpy
8071 #endif
8072  
8073  
8074  
8075 #define DEFINE_UNALIGNED_TYPE(type) \
8076 static forceinline type \
8077 load_##type##_unaligned(const void *p) \
8078 { \
8079 type v; \
8080 \
8081 MEMCOPY(&v, p, sizeof(v)); \
8082 return v; \
8083 } \
8084 \
8085 static forceinline void \
8086 store_##type##_unaligned(type v, void *p) \
8087 { \
8088 MEMCOPY(p, &v, sizeof(v)); \
8089 }
8090  
8091 DEFINE_UNALIGNED_TYPE(u16)
8092 DEFINE_UNALIGNED_TYPE(u32)
8093 DEFINE_UNALIGNED_TYPE(u64)
8094 DEFINE_UNALIGNED_TYPE(machine_word_t)
8095  
8096 #undef MEMCOPY
8097  
8098 #define load_word_unaligned load_machine_word_t_unaligned
8099 #define store_word_unaligned store_machine_word_t_unaligned
8100  
8101  
8102  
8103 static forceinline u16
8104 get_unaligned_le16(const u8 *p)
8105 {
8106 if (UNALIGNED_ACCESS_IS_FAST)
8107 return le16_bswap(load_u16_unaligned(p));
8108 else
8109 return ((u16)p[1] << 8) | p[0];
8110 }
8111  
8112 static forceinline u16
8113 get_unaligned_be16(const u8 *p)
8114 {
8115 if (UNALIGNED_ACCESS_IS_FAST)
8116 return be16_bswap(load_u16_unaligned(p));
8117 else
8118 return ((u16)p[0] << 8) | p[1];
8119 }
8120  
8121 static forceinline u32
8122 get_unaligned_le32(const u8 *p)
8123 {
8124 if (UNALIGNED_ACCESS_IS_FAST)
8125 return le32_bswap(load_u32_unaligned(p));
8126 else
8127 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
8128 ((u32)p[1] << 8) | p[0];
8129 }
8130  
8131 static forceinline u32
8132 get_unaligned_be32(const u8 *p)
8133 {
8134 if (UNALIGNED_ACCESS_IS_FAST)
8135 return be32_bswap(load_u32_unaligned(p));
8136 else
8137 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
8138 ((u32)p[2] << 8) | p[3];
8139 }
8140  
8141 static forceinline u64
8142 get_unaligned_le64(const u8 *p)
8143 {
8144 if (UNALIGNED_ACCESS_IS_FAST)
8145 return le64_bswap(load_u64_unaligned(p));
8146 else
8147 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
8148 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
8149 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
8150 ((u64)p[1] << 8) | p[0];
8151 }
8152  
8153 static forceinline machine_word_t
8154 get_unaligned_leword(const u8 *p)
8155 {
8156 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
8157 if (WORDBITS == 32)
8158 return get_unaligned_le32(p);
8159 else
8160 return get_unaligned_le64(p);
8161 }
8162  
8163  
8164  
8165 static forceinline void
8166 put_unaligned_le16(u16 v, u8 *p)
8167 {
8168 if (UNALIGNED_ACCESS_IS_FAST) {
8169 store_u16_unaligned(le16_bswap(v), p);
8170 } else {
8171 p[0] = (u8)(v >> 0);
8172 p[1] = (u8)(v >> 8);
8173 }
8174 }
8175  
8176 static forceinline void
8177 put_unaligned_be16(u16 v, u8 *p)
8178 {
8179 if (UNALIGNED_ACCESS_IS_FAST) {
8180 store_u16_unaligned(be16_bswap(v), p);
8181 } else {
8182 p[0] = (u8)(v >> 8);
8183 p[1] = (u8)(v >> 0);
8184 }
8185 }
8186  
8187 static forceinline void
8188 put_unaligned_le32(u32 v, u8 *p)
8189 {
8190 if (UNALIGNED_ACCESS_IS_FAST) {
8191 store_u32_unaligned(le32_bswap(v), p);
8192 } else {
8193 p[0] = (u8)(v >> 0);
8194 p[1] = (u8)(v >> 8);
8195 p[2] = (u8)(v >> 16);
8196 p[3] = (u8)(v >> 24);
8197 }
8198 }
8199  
8200 static forceinline void
8201 put_unaligned_be32(u32 v, u8 *p)
8202 {
8203 if (UNALIGNED_ACCESS_IS_FAST) {
8204 store_u32_unaligned(be32_bswap(v), p);
8205 } else {
8206 p[0] = (u8)(v >> 24);
8207 p[1] = (u8)(v >> 16);
8208 p[2] = (u8)(v >> 8);
8209 p[3] = (u8)(v >> 0);
8210 }
8211 }
8212  
8213 static forceinline void
8214 put_unaligned_le64(u64 v, u8 *p)
8215 {
8216 if (UNALIGNED_ACCESS_IS_FAST) {
8217 store_u64_unaligned(le64_bswap(v), p);
8218 } else {
8219 p[0] = (u8)(v >> 0);
8220 p[1] = (u8)(v >> 8);
8221 p[2] = (u8)(v >> 16);
8222 p[3] = (u8)(v >> 24);
8223 p[4] = (u8)(v >> 32);
8224 p[5] = (u8)(v >> 40);
8225 p[6] = (u8)(v >> 48);
8226 p[7] = (u8)(v >> 56);
8227 }
8228 }
8229  
8230 static forceinline void
8231 put_unaligned_leword(machine_word_t v, u8 *p)
8232 {
8233 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
8234 if (WORDBITS == 32)
8235 put_unaligned_le32(v, p);
8236 else
8237 put_unaligned_le64(v, p);
8238 }
8239  
8240  
8241  
8242  
8243  
8244  
8245  
8246 static forceinline unsigned
8247 bsr32(u32 v)
8248 {
8249 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
8250 return 31 - __builtin_clz(v);
8251 #elif defined(_MSC_VER)
8252 unsigned long i;
8253  
8254 _BitScanReverse(&i, v);
8255 return i;
8256 #else
8257 unsigned i = 0;
8258  
8259 while ((v >>= 1) != 0)
8260 i++;
8261 return i;
8262 #endif
8263 }
8264  
8265 static forceinline unsigned
8266 bsr64(u64 v)
8267 {
8268 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
8269 return 63 - __builtin_clzll(v);
8270 #elif defined(_MSC_VER) && defined(_WIN64)
8271 unsigned long i;
8272  
8273 _BitScanReverse64(&i, v);
8274 return i;
8275 #else
8276 unsigned i = 0;
8277  
8278 while ((v >>= 1) != 0)
8279 i++;
8280 return i;
8281 #endif
8282 }
8283  
8284 static forceinline unsigned
8285 bsrw(machine_word_t v)
8286 {
8287 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
8288 if (WORDBITS == 32)
8289 return bsr32(v);
8290 else
8291 return bsr64(v);
8292 }
8293  
8294  
8295  
8296 static forceinline unsigned
8297 bsf32(u32 v)
8298 {
8299 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
8300 return __builtin_ctz(v);
8301 #elif defined(_MSC_VER)
8302 unsigned long i;
8303  
8304 _BitScanForward(&i, v);
8305 return i;
8306 #else
8307 unsigned i = 0;
8308  
8309 for (; (v & 1) == 0; v >>= 1)
8310 i++;
8311 return i;
8312 #endif
8313 }
8314  
8315 static forceinline unsigned
8316 bsf64(u64 v)
8317 {
8318 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
8319 return __builtin_ctzll(v);
8320 #elif defined(_MSC_VER) && defined(_WIN64)
8321 unsigned long i;
8322  
8323 _BitScanForward64(&i, v);
8324 return i;
8325 #else
8326 unsigned i = 0;
8327  
8328 for (; (v & 1) == 0; v >>= 1)
8329 i++;
8330 return i;
8331 #endif
8332 }
8333  
8334 static forceinline unsigned
8335 bsfw(machine_word_t v)
8336 {
8337 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
8338 if (WORDBITS == 32)
8339 return bsf32(v);
8340 else
8341 return bsf64(v);
8342 }
8343  
8344  
8345 #undef rbit32
8346 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
8347 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
8348 static forceinline u32
8349 rbit32(u32 v)
8350 {
8351 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
8352 return v;
8353 }
8354 #define rbit32 rbit32
8355 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
8356 static forceinline u32
8357 rbit32(u32 v)
8358 {
8359 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
8360 return v;
8361 }
8362 #define rbit32 rbit32
8363 #endif
8364  
8365 #endif
8366  
8367  
8368 typedef void *(*malloc_func_t)(size_t);
8369 typedef void (*free_func_t)(void *);
8370  
8371 extern malloc_func_t libdeflate_default_malloc_func;
8372 extern free_func_t libdeflate_default_free_func;
8373  
8374 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
8375 size_t alignment, size_t size);
8376 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
8377  
8378 #ifdef FREESTANDING
8379  
8380 void *memset(void *s, int c, size_t n);
8381 #define memset(s, c, n) __builtin_memset((s), (c), (n))
8382  
8383 void *memcpy(void *dest, const void *src, size_t n);
8384 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
8385  
8386 void *memmove(void *dest, const void *src, size_t n);
8387 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
8388  
8389 int memcmp(const void *s1, const void *s2, size_t n);
8390 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
8391  
8392 #undef LIBDEFLATE_ENABLE_ASSERTIONS
8393 #else
8394 # include <string.h>
8395
8396 # ifdef __clang_analyzer__
8397 # define LIBDEFLATE_ENABLE_ASSERTIONS
8398 # endif
8399 #endif
8400  
8401  
8402 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
8403 NORETURN void
8404 libdeflate_assertion_failed(const char *expr, const char *file, int line);
8405 #define ASSERT(expr) { if (unlikely(!(expr))) \
8406 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
8407 #else
8408 #define ASSERT(expr) (void)(expr)
8409 #endif
8410  
8411 #define CONCAT_IMPL(a, b) a##b
8412 #define CONCAT(a, b) CONCAT_IMPL(a, b)
8413 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
8414  
8415 #endif
8416  
8417  
8418 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
8419  
8420 #define ARM_CPU_FEATURE_NEON (1 << 0)
8421 #define ARM_CPU_FEATURE_PMULL (1 << 1)
8422  
8423 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
8424 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
8425 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
8426 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
8427  
8428 #if !defined(FREESTANDING) && \
8429 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
8430 (defined(__linux__) || \
8431 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
8432 (defined(_WIN32) && defined(ARCH_ARM64)))
8433  
8434 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
8435 extern volatile u32 libdeflate_arm_cpu_features;
8436  
8437 void libdeflate_init_arm_cpu_features(void);
8438  
8439 static inline u32 get_arm_cpu_features(void)
8440 {
8441 if (libdeflate_arm_cpu_features == 0)
8442 libdeflate_init_arm_cpu_features();
8443 return libdeflate_arm_cpu_features;
8444 }
8445 #else
8446 static inline u32 get_arm_cpu_features(void) { return 0; }
8447 #endif
8448  
8449  
8450 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
8451 # define HAVE_NEON(features) 1
8452 # define HAVE_NEON_NATIVE 1
8453 #else
8454 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
8455 # define HAVE_NEON_NATIVE 0
8456 #endif
8457  
8458 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
8459 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
8460 # define HAVE_NEON_INTRIN 1
8461 # include <arm_neon.h>
8462 #else
8463 # define HAVE_NEON_INTRIN 0
8464 #endif
8465  
8466  
8467 #ifdef __ARM_FEATURE_CRYPTO
8468 # define HAVE_PMULL(features) 1
8469 #else
8470 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
8471 #endif
8472 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
8473 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
8474 CPU_IS_LITTLE_ENDIAN()
8475 # define HAVE_PMULL_INTRIN 1
8476
8477 # ifdef _MSC_VER
8478 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
8479 # else
8480 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
8481 # endif
8482 #else
8483 # define HAVE_PMULL_INTRIN 0
8484 #endif
8485  
8486  
8487 #ifdef __ARM_FEATURE_CRC32
8488 # define HAVE_CRC32(features) 1
8489 #else
8490 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
8491 #endif
8492 #if defined(ARCH_ARM64) && \
8493 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
8494 # define HAVE_CRC32_INTRIN 1
8495 # if defined(__GNUC__) || defined(__clang__)
8496 # include <arm_acle.h>
8497 # endif
8498
8499 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
8500 !defined(__ARM_FEATURE_CRC32)
8501 # undef __crc32b
8502 # define __crc32b(a, b) \
8503 ({ uint32_t res; \
8504 __asm__("crc32b %w0, %w1, %w2" \
8505 : "=r" (res) : "r" (a), "r" (b)); \
8506 res; })
8507 # undef __crc32h
8508 # define __crc32h(a, b) \
8509 ({ uint32_t res; \
8510 __asm__("crc32h %w0, %w1, %w2" \
8511 : "=r" (res) : "r" (a), "r" (b)); \
8512 res; })
8513 # undef __crc32w
8514 # define __crc32w(a, b) \
8515 ({ uint32_t res; \
8516 __asm__("crc32w %w0, %w1, %w2" \
8517 : "=r" (res) : "r" (a), "r" (b)); \
8518 res; })
8519 # undef __crc32d
8520 # define __crc32d(a, b) \
8521 ({ uint32_t res; \
8522 __asm__("crc32x %w0, %w1, %2" \
8523 : "=r" (res) : "r" (a), "r" (b)); \
8524 res; })
8525 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
8526 # endif
8527 #else
8528 # define HAVE_CRC32_INTRIN 0
8529 #endif
8530  
8531  
8532 #ifdef __ARM_FEATURE_SHA3
8533 # define HAVE_SHA3(features) 1
8534 #else
8535 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
8536 #endif
8537 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
8538 (GCC_PREREQ(9, 1) || \
8539 CLANG_PREREQ(7, 0, 10010463) )
8540 # define HAVE_SHA3_INTRIN 1
8541
8542 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
8543 !defined(__ARM_FEATURE_SHA3)
8544 # undef veor3q_u8
8545 # define veor3q_u8(a, b, c) \
8546 ({ uint8x16_t res; \
8547 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
8548 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
8549 res; })
8550 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
8551 # endif
8552 #else
8553 # define HAVE_SHA3_INTRIN 0
8554 #endif
8555  
8556  
8557 #ifdef __ARM_FEATURE_DOTPROD
8558 # define HAVE_DOTPROD(features) 1
8559 #else
8560 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
8561 #endif
8562 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
8563 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
8564 # define HAVE_DOTPROD_INTRIN 1
8565
8566 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
8567 !defined(__ARM_FEATURE_DOTPROD)
8568 # undef vdotq_u32
8569 # define vdotq_u32(a, b, c) \
8570 ({ uint32x4_t res = (a); \
8571 __asm__("udot %0.4s, %1.16b, %2.16b" \
8572 : "+w" (res) : "w" (b), "w" (c)); \
8573 res; })
8574 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
8575 # endif
8576 #else
8577 # define HAVE_DOTPROD_INTRIN 0
8578 #endif
8579  
8580 #endif
8581  
8582 #endif
8583  
8584  
8585  
8586 #if HAVE_CRC32_INTRIN
8587 # ifdef __clang__
8588 # define ATTRIBUTES _target_attribute("crc")
8589 # else
8590 # define ATTRIBUTES _target_attribute("+crc")
8591 # endif
8592  
8593  
8594 static forceinline ATTRIBUTES u32
8595 combine_crcs_slow(u32 crc0, u32 crc1, u32 crc2, u32 crc3)
8596 {
8597 u64 res0 = 0, res1 = 0, res2 = 0;
8598 int i;
8599  
8600
8601 for (i = 0; i < 32; i++) {
8602 if (CRC32_FIXED_CHUNK_MULT_3 & (1U << i))
8603 res0 ^= (u64)crc0 << i;
8604 if (CRC32_FIXED_CHUNK_MULT_2 & (1U << i))
8605 res1 ^= (u64)crc1 << i;
8606 if (CRC32_FIXED_CHUNK_MULT_1 & (1U << i))
8607 res2 ^= (u64)crc2 << i;
8608 }
8609
8610 return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
8611 }
8612  
8613 #define crc32_arm_crc crc32_arm_crc
8614 static ATTRIBUTES u32
8615 crc32_arm_crc(u32 crc, const u8 *p, size_t len)
8616 {
8617 if (len >= 64) {
8618 const size_t align = -(uintptr_t)p & 7;
8619  
8620
8621 if (align) {
8622 if (align & 1)
8623 crc = __crc32b(crc, *p++);
8624 if (align & 2) {
8625 crc = __crc32h(crc, le16_bswap(*(u16 *)p));
8626 p += 2;
8627 }
8628 if (align & 4) {
8629 crc = __crc32w(crc, le32_bswap(*(u32 *)p));
8630 p += 4;
8631 }
8632 len -= align;
8633 }
8634
8635 while (len >= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN) {
8636 const u64 *wp0 = (const u64 *)p;
8637 const u64 * const wp0_end =
8638 (const u64 *)(p + CRC32_FIXED_CHUNK_LEN);
8639 u32 crc1 = 0, crc2 = 0, crc3 = 0;
8640  
8641 STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
8642 STATIC_ASSERT(CRC32_FIXED_CHUNK_LEN % (4 * 8) == 0);
8643 do {
8644 prefetchr(&wp0[64 + 0*CRC32_FIXED_CHUNK_LEN/8]);
8645 prefetchr(&wp0[64 + 1*CRC32_FIXED_CHUNK_LEN/8]);
8646 prefetchr(&wp0[64 + 2*CRC32_FIXED_CHUNK_LEN/8]);
8647 prefetchr(&wp0[64 + 3*CRC32_FIXED_CHUNK_LEN/8]);
8648 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
8649 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
8650 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
8651 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
8652 wp0++;
8653 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
8654 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
8655 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
8656 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
8657 wp0++;
8658 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
8659 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
8660 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
8661 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
8662 wp0++;
8663 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
8664 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
8665 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
8666 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
8667 wp0++;
8668 } while (wp0 != wp0_end);
8669 crc = combine_crcs_slow(crc, crc1, crc2, crc3);
8670 p += CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
8671 len -= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
8672 }
8673
8674 while (len >= 64) {
8675 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
8676 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
8677 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
8678 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
8679 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 32)));
8680 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 40)));
8681 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 48)));
8682 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 56)));
8683 p += 64;
8684 len -= 64;
8685 }
8686 }
8687 if (len & 32) {
8688 crc = __crc32d(crc, get_unaligned_le64(p + 0));
8689 crc = __crc32d(crc, get_unaligned_le64(p + 8));
8690 crc = __crc32d(crc, get_unaligned_le64(p + 16));
8691 crc = __crc32d(crc, get_unaligned_le64(p + 24));
8692 p += 32;
8693 }
8694 if (len & 16) {
8695 crc = __crc32d(crc, get_unaligned_le64(p + 0));
8696 crc = __crc32d(crc, get_unaligned_le64(p + 8));
8697 p += 16;
8698 }
8699 if (len & 8) {
8700 crc = __crc32d(crc, get_unaligned_le64(p));
8701 p += 8;
8702 }
8703 if (len & 4) {
8704 crc = __crc32w(crc, get_unaligned_le32(p));
8705 p += 4;
8706 }
8707 if (len & 2) {
8708 crc = __crc32h(crc, get_unaligned_le16(p));
8709 p += 2;
8710 }
8711 if (len & 1)
8712 crc = __crc32b(crc, *p);
8713 return crc;
8714 }
8715 #undef ATTRIBUTES
8716 #endif
8717  
8718  
8719 #if HAVE_CRC32_INTRIN && HAVE_PMULL_INTRIN
8720 # ifdef __clang__
8721 # define ATTRIBUTES _target_attribute("crc,aes")
8722 # else
8723 # define ATTRIBUTES _target_attribute("+crc,+crypto")
8724 # endif
8725  
8726  
8727 static forceinline ATTRIBUTES u64
8728 clmul_u32(u32 a, u32 b)
8729 {
8730 uint64x2_t res = vreinterpretq_u64_p128(
8731 compat_vmull_p64((poly64_t)a, (poly64_t)b));
8732  
8733 return vgetq_lane_u64(res, 0);
8734 }
8735  
8736  
8737 static forceinline ATTRIBUTES u32
8738 combine_crcs_fast(u32 crc0, u32 crc1, u32 crc2, u32 crc3, size_t i)
8739 {
8740 u64 res0 = clmul_u32(crc0, crc32_mults_for_chunklen[i][0]);
8741 u64 res1 = clmul_u32(crc1, crc32_mults_for_chunklen[i][1]);
8742 u64 res2 = clmul_u32(crc2, crc32_mults_for_chunklen[i][2]);
8743  
8744 return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
8745 }
8746  
8747 #define crc32_arm_crc_pmullcombine crc32_arm_crc_pmullcombine
8748 static ATTRIBUTES u32
8749 crc32_arm_crc_pmullcombine(u32 crc, const u8 *p, size_t len)
8750 {
8751 const size_t align = -(uintptr_t)p & 7;
8752  
8753 if (len >= align + CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
8754
8755 if (align) {
8756 if (align & 1)
8757 crc = __crc32b(crc, *p++);
8758 if (align & 2) {
8759 crc = __crc32h(crc, le16_bswap(*(u16 *)p));
8760 p += 2;
8761 }
8762 if (align & 4) {
8763 crc = __crc32w(crc, le32_bswap(*(u32 *)p));
8764 p += 4;
8765 }
8766 len -= align;
8767 }
8768
8769 while (len >= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN) {
8770 const u64 *wp0 = (const u64 *)p;
8771 const u64 * const wp0_end =
8772 (const u64 *)(p + CRC32_MAX_VARIABLE_CHUNK_LEN);
8773 u32 crc1 = 0, crc2 = 0, crc3 = 0;
8774  
8775 STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
8776 STATIC_ASSERT(CRC32_MAX_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
8777 do {
8778 prefetchr(&wp0[64 + 0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
8779 prefetchr(&wp0[64 + 1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
8780 prefetchr(&wp0[64 + 2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
8781 prefetchr(&wp0[64 + 3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
8782 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8783 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8784 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8785 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8786 wp0++;
8787 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8788 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8789 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8790 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8791 wp0++;
8792 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8793 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8794 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8795 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8796 wp0++;
8797 crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8798 crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8799 crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8800 crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
8801 wp0++;
8802 } while (wp0 != wp0_end);
8803 crc = combine_crcs_fast(crc, crc1, crc2, crc3,
8804 ARRAY_LEN(crc32_mults_for_chunklen) - 1);
8805 p += CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
8806 len -= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
8807 }
8808
8809 if (len >= CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
8810 const size_t i = len / (CRC32_NUM_CHUNKS *
8811 CRC32_MIN_VARIABLE_CHUNK_LEN);
8812 const size_t chunk_len =
8813 i * CRC32_MIN_VARIABLE_CHUNK_LEN;
8814 const u64 *wp0 = (const u64 *)(p + 0*chunk_len);
8815 const u64 *wp1 = (const u64 *)(p + 1*chunk_len);
8816 const u64 *wp2 = (const u64 *)(p + 2*chunk_len);
8817 const u64 *wp3 = (const u64 *)(p + 3*chunk_len);
8818 const u64 * const wp0_end = wp1;
8819 u32 crc1 = 0, crc2 = 0, crc3 = 0;
8820  
8821 STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
8822 STATIC_ASSERT(CRC32_MIN_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
8823 do {
8824 prefetchr(wp0 + 64);
8825 prefetchr(wp1 + 64);
8826 prefetchr(wp2 + 64);
8827 prefetchr(wp3 + 64);
8828 crc = __crc32d(crc, le64_bswap(*wp0++));
8829 crc1 = __crc32d(crc1, le64_bswap(*wp1++));
8830 crc2 = __crc32d(crc2, le64_bswap(*wp2++));
8831 crc3 = __crc32d(crc3, le64_bswap(*wp3++));
8832 crc = __crc32d(crc, le64_bswap(*wp0++));
8833 crc1 = __crc32d(crc1, le64_bswap(*wp1++));
8834 crc2 = __crc32d(crc2, le64_bswap(*wp2++));
8835 crc3 = __crc32d(crc3, le64_bswap(*wp3++));
8836 crc = __crc32d(crc, le64_bswap(*wp0++));
8837 crc1 = __crc32d(crc1, le64_bswap(*wp1++));
8838 crc2 = __crc32d(crc2, le64_bswap(*wp2++));
8839 crc3 = __crc32d(crc3, le64_bswap(*wp3++));
8840 crc = __crc32d(crc, le64_bswap(*wp0++));
8841 crc1 = __crc32d(crc1, le64_bswap(*wp1++));
8842 crc2 = __crc32d(crc2, le64_bswap(*wp2++));
8843 crc3 = __crc32d(crc3, le64_bswap(*wp3++));
8844 } while (wp0 != wp0_end);
8845 crc = combine_crcs_fast(crc, crc1, crc2, crc3, i);
8846 p += CRC32_NUM_CHUNKS * chunk_len;
8847 len -= CRC32_NUM_CHUNKS * chunk_len;
8848 }
8849  
8850 while (len >= 32) {
8851 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
8852 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
8853 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
8854 crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
8855 p += 32;
8856 len -= 32;
8857 }
8858 } else {
8859 while (len >= 32) {
8860 crc = __crc32d(crc, get_unaligned_le64(p + 0));
8861 crc = __crc32d(crc, get_unaligned_le64(p + 8));
8862 crc = __crc32d(crc, get_unaligned_le64(p + 16));
8863 crc = __crc32d(crc, get_unaligned_le64(p + 24));
8864 p += 32;
8865 len -= 32;
8866 }
8867 }
8868 if (len & 16) {
8869 crc = __crc32d(crc, get_unaligned_le64(p + 0));
8870 crc = __crc32d(crc, get_unaligned_le64(p + 8));
8871 p += 16;
8872 }
8873 if (len & 8) {
8874 crc = __crc32d(crc, get_unaligned_le64(p));
8875 p += 8;
8876 }
8877 if (len & 4) {
8878 crc = __crc32w(crc, get_unaligned_le32(p));
8879 p += 4;
8880 }
8881 if (len & 2) {
8882 crc = __crc32h(crc, get_unaligned_le16(p));
8883 p += 2;
8884 }
8885 if (len & 1)
8886 crc = __crc32b(crc, *p);
8887 return crc;
8888 }
8889 #undef ATTRIBUTES
8890 #endif
8891  
8892  
8893 #if HAVE_PMULL_INTRIN
8894 # define crc32_arm_pmullx4 crc32_arm_pmullx4
8895 # define SUFFIX _pmullx4
8896 # ifdef __clang__
8897
8898 # define ATTRIBUTES _target_attribute("aes")
8899 # else
8900
8901 # define ATTRIBUTES _target_attribute("+crypto")
8902 # endif
8903 # define ENABLE_EOR3 0
8904 /* #include "arm-crc32_pmull_helpers.h" */
8905  
8906  
8907  
8908  
8909  
8910 #undef u32_to_bytevec
8911 static forceinline ATTRIBUTES uint8x16_t
8912 ADD_SUFFIX(u32_to_bytevec)(u32 a)
8913 {
8914 return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));
8915 }
8916 #define u32_to_bytevec ADD_SUFFIX(u32_to_bytevec)
8917  
8918  
8919 #undef load_multipliers
8920 static forceinline ATTRIBUTES poly64x2_t
8921 ADD_SUFFIX(load_multipliers)(const u64 p[2])
8922 {
8923 return vreinterpretq_p64_u64(vld1q_u64(p));
8924 }
8925 #define load_multipliers ADD_SUFFIX(load_multipliers)
8926  
8927  
8928 #undef clmul_low
8929 static forceinline ATTRIBUTES uint8x16_t
8930 ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)
8931 {
8932 return vreinterpretq_u8_p128(
8933 compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),
8934 vgetq_lane_p64(b, 0)));
8935 }
8936 #define clmul_low ADD_SUFFIX(clmul_low)
8937  
8938  
8939 #undef clmul_high
8940 static forceinline ATTRIBUTES uint8x16_t
8941 ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)
8942 {
8943 #ifdef __clang__
8944
8945 uint8x16_t res;
8946  
8947 __asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));
8948 return res;
8949 #else
8950 return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));
8951 #endif
8952 }
8953 #define clmul_high ADD_SUFFIX(clmul_high)
8954  
8955 #undef eor3
8956 static forceinline ATTRIBUTES uint8x16_t
8957 ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)
8958 {
8959 #if ENABLE_EOR3
8960 return veor3q_u8(a, b, c);
8961 #else
8962 return veorq_u8(veorq_u8(a, b), c);
8963 #endif
8964 }
8965 #define eor3 ADD_SUFFIX(eor3)
8966  
8967 #undef fold_vec
8968 static forceinline ATTRIBUTES uint8x16_t
8969 ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)
8970 {
8971 uint8x16_t a = clmul_low(src, multipliers);
8972 uint8x16_t b = clmul_high(src, multipliers);
8973  
8974 return eor3(a, b, dst);
8975 }
8976 #define fold_vec ADD_SUFFIX(fold_vec)
8977  
8978  
8979 #undef fold_partial_vec
8980 static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t
8981 ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,
8982 poly64x2_t multipliers_1)
8983 {
8984
8985 static const u8 shift_tab[48] = {
8986 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
8987 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
8988 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
8989 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
8990 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
8991 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
8992 };
8993 const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);
8994 const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);
8995 uint8x16_t x0, x1, bsl_mask;
8996  
8997
8998 x0 = vqtbl1q_u8(v, lshift);
8999  
9000
9001 bsl_mask = vreinterpretq_u8_s8(
9002 vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));
9003  
9004
9005 x1 = vbslq_u8(bsl_mask ,
9006 vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));
9007  
9008 return fold_vec(x0, x1, multipliers_1);
9009 }
9010 #define fold_partial_vec ADD_SUFFIX(fold_partial_vec)
9011  
9012  
9013 static ATTRIBUTES u32
9014 crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
9015 {
9016 static const u64 _aligned_attribute(16) mults[3][2] = {
9017 { CRC32_X159_MODG, CRC32_X95_MODG },
9018 { CRC32_X543_MODG, CRC32_X479_MODG },
9019 { CRC32_X287_MODG, CRC32_X223_MODG },
9020 };
9021 static const u64 _aligned_attribute(16) barrett_consts[3][2] = {
9022 { CRC32_X95_MODG, },
9023 { CRC32_BARRETT_CONSTANT_1, },
9024 { CRC32_BARRETT_CONSTANT_2, },
9025 };
9026 const poly64x2_t multipliers_1 = load_multipliers(mults[0]);
9027 uint8x16_t v0, v1, v2, v3;
9028  
9029 if (len < 64 + 15) {
9030 if (len < 16)
9031 return crc32_slice1(crc, p, len);
9032 v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
9033 p += 16;
9034 len -= 16;
9035 while (len >= 16) {
9036 v0 = fold_vec(v0, vld1q_u8(p), multipliers_1);
9037 p += 16;
9038 len -= 16;
9039 }
9040 } else {
9041 const poly64x2_t multipliers_4 = load_multipliers(mults[1]);
9042 const poly64x2_t multipliers_2 = load_multipliers(mults[2]);
9043 const size_t align = -(uintptr_t)p & 15;
9044 const uint8x16_t *vp;
9045  
9046 v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
9047 p += 16;
9048
9049 if (align) {
9050 v0 = fold_partial_vec(v0, p, align, multipliers_1);
9051 p += align;
9052 len -= align;
9053 }
9054 vp = (const uint8x16_t *)p;
9055 v1 = *vp++;
9056 v2 = *vp++;
9057 v3 = *vp++;
9058 while (len >= 64 + 64) {
9059 v0 = fold_vec(v0, *vp++, multipliers_4);
9060 v1 = fold_vec(v1, *vp++, multipliers_4);
9061 v2 = fold_vec(v2, *vp++, multipliers_4);
9062 v3 = fold_vec(v3, *vp++, multipliers_4);
9063 len -= 64;
9064 }
9065 v0 = fold_vec(v0, v2, multipliers_2);
9066 v1 = fold_vec(v1, v3, multipliers_2);
9067 if (len & 32) {
9068 v0 = fold_vec(v0, *vp++, multipliers_2);
9069 v1 = fold_vec(v1, *vp++, multipliers_2);
9070 }
9071 v0 = fold_vec(v0, v1, multipliers_1);
9072 if (len & 16)
9073 v0 = fold_vec(v0, *vp++, multipliers_1);
9074 p = (const u8 *)vp;
9075 len &= 15;
9076 }
9077  
9078
9079 if (len)
9080 v0 = fold_partial_vec(v0, p, len, multipliers_1);
9081  
9082
9083 v0 = veorq_u8(clmul_low(v0, load_multipliers(barrett_consts[0])),
9084 vextq_u8(v0, vdupq_n_u8(0), 8));
9085 v1 = clmul_low(v0, load_multipliers(barrett_consts[1]));
9086 v1 = clmul_low(v1, load_multipliers(barrett_consts[2]));
9087 v0 = veorq_u8(v0, v1);
9088 return vgetq_lane_u32(vreinterpretq_u32_u8(v0), 2);
9089 }
9090 #undef SUFFIX
9091 #undef ATTRIBUTES
9092 #undef ENABLE_EOR3
9093 #endif
9094  
9095  
9096 #if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN
9097 # define crc32_arm_pmullx12_crc crc32_arm_pmullx12_crc
9098 # define SUFFIX _pmullx12_crc
9099 # ifdef __clang__
9100 # define ATTRIBUTES _target_attribute("aes,crc")
9101 # else
9102 # define ATTRIBUTES _target_attribute("+crypto,+crc")
9103 # endif
9104 # define ENABLE_EOR3 0
9105 /* #include "arm-crc32_pmull_wide.h" */
9106  
9107  
9108  
9109  
9110 /* #include "arm-crc32_pmull_helpers.h" */
9111  
9112  
9113  
9114  
9115  
9116 #undef u32_to_bytevec
9117 static forceinline ATTRIBUTES uint8x16_t
9118 ADD_SUFFIX(u32_to_bytevec)(u32 a)
9119 {
9120 return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));
9121 }
9122 #define u32_to_bytevec ADD_SUFFIX(u32_to_bytevec)
9123  
9124  
9125 #undef load_multipliers
9126 static forceinline ATTRIBUTES poly64x2_t
9127 ADD_SUFFIX(load_multipliers)(const u64 p[2])
9128 {
9129 return vreinterpretq_p64_u64(vld1q_u64(p));
9130 }
9131 #define load_multipliers ADD_SUFFIX(load_multipliers)
9132  
9133  
9134 #undef clmul_low
9135 static forceinline ATTRIBUTES uint8x16_t
9136 ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)
9137 {
9138 return vreinterpretq_u8_p128(
9139 compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),
9140 vgetq_lane_p64(b, 0)));
9141 }
9142 #define clmul_low ADD_SUFFIX(clmul_low)
9143  
9144  
9145 #undef clmul_high
9146 static forceinline ATTRIBUTES uint8x16_t
9147 ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)
9148 {
9149 #ifdef __clang__
9150
9151 uint8x16_t res;
9152  
9153 __asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));
9154 return res;
9155 #else
9156 return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));
9157 #endif
9158 }
9159 #define clmul_high ADD_SUFFIX(clmul_high)
9160  
9161 #undef eor3
9162 static forceinline ATTRIBUTES uint8x16_t
9163 ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)
9164 {
9165 #if ENABLE_EOR3
9166 return veor3q_u8(a, b, c);
9167 #else
9168 return veorq_u8(veorq_u8(a, b), c);
9169 #endif
9170 }
9171 #define eor3 ADD_SUFFIX(eor3)
9172  
9173 #undef fold_vec
9174 static forceinline ATTRIBUTES uint8x16_t
9175 ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)
9176 {
9177 uint8x16_t a = clmul_low(src, multipliers);
9178 uint8x16_t b = clmul_high(src, multipliers);
9179  
9180 return eor3(a, b, dst);
9181 }
9182 #define fold_vec ADD_SUFFIX(fold_vec)
9183  
9184  
9185 #undef fold_partial_vec
9186 static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t
9187 ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,
9188 poly64x2_t multipliers_1)
9189 {
9190
9191 static const u8 shift_tab[48] = {
9192 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9193 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9194 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
9195 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
9196 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9197 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9198 };
9199 const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);
9200 const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);
9201 uint8x16_t x0, x1, bsl_mask;
9202  
9203
9204 x0 = vqtbl1q_u8(v, lshift);
9205  
9206
9207 bsl_mask = vreinterpretq_u8_s8(
9208 vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));
9209  
9210
9211 x1 = vbslq_u8(bsl_mask ,
9212 vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));
9213  
9214 return fold_vec(x0, x1, multipliers_1);
9215 }
9216 #define fold_partial_vec ADD_SUFFIX(fold_partial_vec)
9217  
9218  
9219 static ATTRIBUTES u32
9220 ADD_SUFFIX(crc32_arm)(u32 crc, const u8 *p, size_t len)
9221 {
9222 uint8x16_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11;
9223  
9224 if (len < 3 * 192) {
9225 static const u64 _aligned_attribute(16) mults[3][2] = {
9226 { CRC32_X543_MODG, CRC32_X479_MODG },
9227 { CRC32_X287_MODG, CRC32_X223_MODG },
9228 { CRC32_X159_MODG, CRC32_X95_MODG },
9229 };
9230 poly64x2_t multipliers_4, multipliers_2, multipliers_1;
9231  
9232 if (len < 64)
9233 goto tail;
9234 multipliers_4 = load_multipliers(mults[0]);
9235 multipliers_2 = load_multipliers(mults[1]);
9236 multipliers_1 = load_multipliers(mults[2]);
9237
9238 v0 = veorq_u8(vld1q_u8(p + 0), u32_to_bytevec(crc));
9239 v1 = vld1q_u8(p + 16);
9240 v2 = vld1q_u8(p + 32);
9241 v3 = vld1q_u8(p + 48);
9242 p += 64;
9243 len -= 64;
9244 while (len >= 64) {
9245 v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_4);
9246 v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_4);
9247 v2 = fold_vec(v2, vld1q_u8(p + 32), multipliers_4);
9248 v3 = fold_vec(v3, vld1q_u8(p + 48), multipliers_4);
9249 p += 64;
9250 len -= 64;
9251 }
9252 v0 = fold_vec(v0, v2, multipliers_2);
9253 v1 = fold_vec(v1, v3, multipliers_2);
9254 if (len >= 32) {
9255 v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_2);
9256 v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_2);
9257 p += 32;
9258 len -= 32;
9259 }
9260 v0 = fold_vec(v0, v1, multipliers_1);
9261 } else {
9262 static const u64 _aligned_attribute(16) mults[4][2] = {
9263 { CRC32_X1567_MODG, CRC32_X1503_MODG },
9264 { CRC32_X799_MODG, CRC32_X735_MODG },
9265 { CRC32_X415_MODG, CRC32_X351_MODG },
9266 { CRC32_X159_MODG, CRC32_X95_MODG },
9267 };
9268 const poly64x2_t multipliers_12 = load_multipliers(mults[0]);
9269 const poly64x2_t multipliers_6 = load_multipliers(mults[1]);
9270 const poly64x2_t multipliers_3 = load_multipliers(mults[2]);
9271 const poly64x2_t multipliers_1 = load_multipliers(mults[3]);
9272 const size_t align = -(uintptr_t)p & 15;
9273 const uint8x16_t *vp;
9274  
9275
9276 if (align) {
9277 if (align & 1)
9278 crc = __crc32b(crc, *p++);
9279 if (align & 2) {
9280 crc = __crc32h(crc, le16_bswap(*(u16 *)p));
9281 p += 2;
9282 }
9283 if (align & 4) {
9284 crc = __crc32w(crc, le32_bswap(*(u32 *)p));
9285 p += 4;
9286 }
9287 if (align & 8) {
9288 crc = __crc32d(crc, le64_bswap(*(u64 *)p));
9289 p += 8;
9290 }
9291 len -= align;
9292 }
9293 vp = (const uint8x16_t *)p;
9294 v0 = veorq_u8(*vp++, u32_to_bytevec(crc));
9295 v1 = *vp++;
9296 v2 = *vp++;
9297 v3 = *vp++;
9298 v4 = *vp++;
9299 v5 = *vp++;
9300 v6 = *vp++;
9301 v7 = *vp++;
9302 v8 = *vp++;
9303 v9 = *vp++;
9304 v10 = *vp++;
9305 v11 = *vp++;
9306 len -= 192;
9307
9308 do {
9309 v0 = fold_vec(v0, *vp++, multipliers_12);
9310 v1 = fold_vec(v1, *vp++, multipliers_12);
9311 v2 = fold_vec(v2, *vp++, multipliers_12);
9312 v3 = fold_vec(v3, *vp++, multipliers_12);
9313 v4 = fold_vec(v4, *vp++, multipliers_12);
9314 v5 = fold_vec(v5, *vp++, multipliers_12);
9315 v6 = fold_vec(v6, *vp++, multipliers_12);
9316 v7 = fold_vec(v7, *vp++, multipliers_12);
9317 v8 = fold_vec(v8, *vp++, multipliers_12);
9318 v9 = fold_vec(v9, *vp++, multipliers_12);
9319 v10 = fold_vec(v10, *vp++, multipliers_12);
9320 v11 = fold_vec(v11, *vp++, multipliers_12);
9321 len -= 192;
9322 } while (len >= 192);
9323  
9324
9325 v0 = fold_vec(v0, v6, multipliers_6);
9326 v1 = fold_vec(v1, v7, multipliers_6);
9327 v2 = fold_vec(v2, v8, multipliers_6);
9328 v3 = fold_vec(v3, v9, multipliers_6);
9329 v4 = fold_vec(v4, v10, multipliers_6);
9330 v5 = fold_vec(v5, v11, multipliers_6);
9331 if (len >= 96) {
9332 v0 = fold_vec(v0, *vp++, multipliers_6);
9333 v1 = fold_vec(v1, *vp++, multipliers_6);
9334 v2 = fold_vec(v2, *vp++, multipliers_6);
9335 v3 = fold_vec(v3, *vp++, multipliers_6);
9336 v4 = fold_vec(v4, *vp++, multipliers_6);
9337 v5 = fold_vec(v5, *vp++, multipliers_6);
9338 len -= 96;
9339 }
9340 v0 = fold_vec(v0, v3, multipliers_3);
9341 v1 = fold_vec(v1, v4, multipliers_3);
9342 v2 = fold_vec(v2, v5, multipliers_3);
9343 if (len >= 48) {
9344 v0 = fold_vec(v0, *vp++, multipliers_3);
9345 v1 = fold_vec(v1, *vp++, multipliers_3);
9346 v2 = fold_vec(v2, *vp++, multipliers_3);
9347 len -= 48;
9348 }
9349 v0 = fold_vec(v0, v1, multipliers_1);
9350 v0 = fold_vec(v0, v2, multipliers_1);
9351 p = (const u8 *)vp;
9352 }
9353
9354 crc = __crc32d(0, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 0));
9355 crc = __crc32d(crc, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 1));
9356 tail:
9357
9358 if (len & 32) {
9359 crc = __crc32d(crc, get_unaligned_le64(p + 0));
9360 crc = __crc32d(crc, get_unaligned_le64(p + 8));
9361 crc = __crc32d(crc, get_unaligned_le64(p + 16));
9362 crc = __crc32d(crc, get_unaligned_le64(p + 24));
9363 p += 32;
9364 }
9365 if (len & 16) {
9366 crc = __crc32d(crc, get_unaligned_le64(p + 0));
9367 crc = __crc32d(crc, get_unaligned_le64(p + 8));
9368 p += 16;
9369 }
9370 if (len & 8) {
9371 crc = __crc32d(crc, get_unaligned_le64(p));
9372 p += 8;
9373 }
9374 if (len & 4) {
9375 crc = __crc32w(crc, get_unaligned_le32(p));
9376 p += 4;
9377 }
9378 if (len & 2) {
9379 crc = __crc32h(crc, get_unaligned_le16(p));
9380 p += 2;
9381 }
9382 if (len & 1)
9383 crc = __crc32b(crc, *p);
9384 return crc;
9385 }
9386  
9387 #undef SUFFIX
9388 #undef ATTRIBUTES
9389 #undef ENABLE_EOR3
9390  
9391 #endif
9392  
9393  
9394 #if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN && \
9395 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3)
9396 # define crc32_arm_pmullx12_crc_eor3 crc32_arm_pmullx12_crc_eor3
9397 # define SUFFIX _pmullx12_crc_eor3
9398 # ifdef __clang__
9399 # define ATTRIBUTES _target_attribute("aes,crc,sha3")
9400
9401 # elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
9402 || defined(__ARM_FEATURE_DOTPROD)
9403 # define ATTRIBUTES _target_attribute("+crypto,+crc,+sha3")
9404 # else
9405 # define ATTRIBUTES _target_attribute("arch=armv8.2-a+crypto+crc+sha3")
9406 # endif
9407 # define ENABLE_EOR3 1
9408 /* #include "arm-crc32_pmull_wide.h" */
9409  
9410  
9411  
9412  
9413 /* #include "arm-crc32_pmull_helpers.h" */
9414  
9415  
9416  
9417  
9418  
9419 #undef u32_to_bytevec
9420 static forceinline ATTRIBUTES uint8x16_t
9421 ADD_SUFFIX(u32_to_bytevec)(u32 a)
9422 {
9423 return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));
9424 }
9425 #define u32_to_bytevec ADD_SUFFIX(u32_to_bytevec)
9426  
9427  
9428 #undef load_multipliers
9429 static forceinline ATTRIBUTES poly64x2_t
9430 ADD_SUFFIX(load_multipliers)(const u64 p[2])
9431 {
9432 return vreinterpretq_p64_u64(vld1q_u64(p));
9433 }
9434 #define load_multipliers ADD_SUFFIX(load_multipliers)
9435  
9436  
9437 #undef clmul_low
9438 static forceinline ATTRIBUTES uint8x16_t
9439 ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)
9440 {
9441 return vreinterpretq_u8_p128(
9442 compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),
9443 vgetq_lane_p64(b, 0)));
9444 }
9445 #define clmul_low ADD_SUFFIX(clmul_low)
9446  
9447  
9448 #undef clmul_high
9449 static forceinline ATTRIBUTES uint8x16_t
9450 ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)
9451 {
9452 #ifdef __clang__
9453
9454 uint8x16_t res;
9455  
9456 __asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));
9457 return res;
9458 #else
9459 return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));
9460 #endif
9461 }
9462 #define clmul_high ADD_SUFFIX(clmul_high)
9463  
9464 #undef eor3
9465 static forceinline ATTRIBUTES uint8x16_t
9466 ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)
9467 {
9468 #if ENABLE_EOR3
9469 return veor3q_u8(a, b, c);
9470 #else
9471 return veorq_u8(veorq_u8(a, b), c);
9472 #endif
9473 }
9474 #define eor3 ADD_SUFFIX(eor3)
9475  
9476 #undef fold_vec
9477 static forceinline ATTRIBUTES uint8x16_t
9478 ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)
9479 {
9480 uint8x16_t a = clmul_low(src, multipliers);
9481 uint8x16_t b = clmul_high(src, multipliers);
9482  
9483 return eor3(a, b, dst);
9484 }
9485 #define fold_vec ADD_SUFFIX(fold_vec)
9486  
9487  
9488 #undef fold_partial_vec
9489 static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t
9490 ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,
9491 poly64x2_t multipliers_1)
9492 {
9493
9494 static const u8 shift_tab[48] = {
9495 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9496 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9497 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
9498 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
9499 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9500 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
9501 };
9502 const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);
9503 const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);
9504 uint8x16_t x0, x1, bsl_mask;
9505  
9506
9507 x0 = vqtbl1q_u8(v, lshift);
9508  
9509
9510 bsl_mask = vreinterpretq_u8_s8(
9511 vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));
9512  
9513
9514 x1 = vbslq_u8(bsl_mask ,
9515 vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));
9516  
9517 return fold_vec(x0, x1, multipliers_1);
9518 }
9519 #define fold_partial_vec ADD_SUFFIX(fold_partial_vec)
9520  
9521  
9522 static ATTRIBUTES u32
9523 ADD_SUFFIX(crc32_arm)(u32 crc, const u8 *p, size_t len)
9524 {
9525 uint8x16_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11;
9526  
9527 if (len < 3 * 192) {
9528 static const u64 _aligned_attribute(16) mults[3][2] = {
9529 { CRC32_X543_MODG, CRC32_X479_MODG },
9530 { CRC32_X287_MODG, CRC32_X223_MODG },
9531 { CRC32_X159_MODG, CRC32_X95_MODG },
9532 };
9533 poly64x2_t multipliers_4, multipliers_2, multipliers_1;
9534  
9535 if (len < 64)
9536 goto tail;
9537 multipliers_4 = load_multipliers(mults[0]);
9538 multipliers_2 = load_multipliers(mults[1]);
9539 multipliers_1 = load_multipliers(mults[2]);
9540
9541 v0 = veorq_u8(vld1q_u8(p + 0), u32_to_bytevec(crc));
9542 v1 = vld1q_u8(p + 16);
9543 v2 = vld1q_u8(p + 32);
9544 v3 = vld1q_u8(p + 48);
9545 p += 64;
9546 len -= 64;
9547 while (len >= 64) {
9548 v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_4);
9549 v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_4);
9550 v2 = fold_vec(v2, vld1q_u8(p + 32), multipliers_4);
9551 v3 = fold_vec(v3, vld1q_u8(p + 48), multipliers_4);
9552 p += 64;
9553 len -= 64;
9554 }
9555 v0 = fold_vec(v0, v2, multipliers_2);
9556 v1 = fold_vec(v1, v3, multipliers_2);
9557 if (len >= 32) {
9558 v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_2);
9559 v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_2);
9560 p += 32;
9561 len -= 32;
9562 }
9563 v0 = fold_vec(v0, v1, multipliers_1);
9564 } else {
9565 static const u64 _aligned_attribute(16) mults[4][2] = {
9566 { CRC32_X1567_MODG, CRC32_X1503_MODG },
9567 { CRC32_X799_MODG, CRC32_X735_MODG },
9568 { CRC32_X415_MODG, CRC32_X351_MODG },
9569 { CRC32_X159_MODG, CRC32_X95_MODG },
9570 };
9571 const poly64x2_t multipliers_12 = load_multipliers(mults[0]);
9572 const poly64x2_t multipliers_6 = load_multipliers(mults[1]);
9573 const poly64x2_t multipliers_3 = load_multipliers(mults[2]);
9574 const poly64x2_t multipliers_1 = load_multipliers(mults[3]);
9575 const size_t align = -(uintptr_t)p & 15;
9576 const uint8x16_t *vp;
9577  
9578
9579 if (align) {
9580 if (align & 1)
9581 crc = __crc32b(crc, *p++);
9582 if (align & 2) {
9583 crc = __crc32h(crc, le16_bswap(*(u16 *)p));
9584 p += 2;
9585 }
9586 if (align & 4) {
9587 crc = __crc32w(crc, le32_bswap(*(u32 *)p));
9588 p += 4;
9589 }
9590 if (align & 8) {
9591 crc = __crc32d(crc, le64_bswap(*(u64 *)p));
9592 p += 8;
9593 }
9594 len -= align;
9595 }
9596 vp = (const uint8x16_t *)p;
9597 v0 = veorq_u8(*vp++, u32_to_bytevec(crc));
9598 v1 = *vp++;
9599 v2 = *vp++;
9600 v3 = *vp++;
9601 v4 = *vp++;
9602 v5 = *vp++;
9603 v6 = *vp++;
9604 v7 = *vp++;
9605 v8 = *vp++;
9606 v9 = *vp++;
9607 v10 = *vp++;
9608 v11 = *vp++;
9609 len -= 192;
9610
9611 do {
9612 v0 = fold_vec(v0, *vp++, multipliers_12);
9613 v1 = fold_vec(v1, *vp++, multipliers_12);
9614 v2 = fold_vec(v2, *vp++, multipliers_12);
9615 v3 = fold_vec(v3, *vp++, multipliers_12);
9616 v4 = fold_vec(v4, *vp++, multipliers_12);
9617 v5 = fold_vec(v5, *vp++, multipliers_12);
9618 v6 = fold_vec(v6, *vp++, multipliers_12);
9619 v7 = fold_vec(v7, *vp++, multipliers_12);
9620 v8 = fold_vec(v8, *vp++, multipliers_12);
9621 v9 = fold_vec(v9, *vp++, multipliers_12);
9622 v10 = fold_vec(v10, *vp++, multipliers_12);
9623 v11 = fold_vec(v11, *vp++, multipliers_12);
9624 len -= 192;
9625 } while (len >= 192);
9626  
9627
9628 v0 = fold_vec(v0, v6, multipliers_6);
9629 v1 = fold_vec(v1, v7, multipliers_6);
9630 v2 = fold_vec(v2, v8, multipliers_6);
9631 v3 = fold_vec(v3, v9, multipliers_6);
9632 v4 = fold_vec(v4, v10, multipliers_6);
9633 v5 = fold_vec(v5, v11, multipliers_6);
9634 if (len >= 96) {
9635 v0 = fold_vec(v0, *vp++, multipliers_6);
9636 v1 = fold_vec(v1, *vp++, multipliers_6);
9637 v2 = fold_vec(v2, *vp++, multipliers_6);
9638 v3 = fold_vec(v3, *vp++, multipliers_6);
9639 v4 = fold_vec(v4, *vp++, multipliers_6);
9640 v5 = fold_vec(v5, *vp++, multipliers_6);
9641 len -= 96;
9642 }
9643 v0 = fold_vec(v0, v3, multipliers_3);
9644 v1 = fold_vec(v1, v4, multipliers_3);
9645 v2 = fold_vec(v2, v5, multipliers_3);
9646 if (len >= 48) {
9647 v0 = fold_vec(v0, *vp++, multipliers_3);
9648 v1 = fold_vec(v1, *vp++, multipliers_3);
9649 v2 = fold_vec(v2, *vp++, multipliers_3);
9650 len -= 48;
9651 }
9652 v0 = fold_vec(v0, v1, multipliers_1);
9653 v0 = fold_vec(v0, v2, multipliers_1);
9654 p = (const u8 *)vp;
9655 }
9656
9657 crc = __crc32d(0, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 0));
9658 crc = __crc32d(crc, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 1));
9659 tail:
9660
9661 if (len & 32) {
9662 crc = __crc32d(crc, get_unaligned_le64(p + 0));
9663 crc = __crc32d(crc, get_unaligned_le64(p + 8));
9664 crc = __crc32d(crc, get_unaligned_le64(p + 16));
9665 crc = __crc32d(crc, get_unaligned_le64(p + 24));
9666 p += 32;
9667 }
9668 if (len & 16) {
9669 crc = __crc32d(crc, get_unaligned_le64(p + 0));
9670 crc = __crc32d(crc, get_unaligned_le64(p + 8));
9671 p += 16;
9672 }
9673 if (len & 8) {
9674 crc = __crc32d(crc, get_unaligned_le64(p));
9675 p += 8;
9676 }
9677 if (len & 4) {
9678 crc = __crc32w(crc, get_unaligned_le32(p));
9679 p += 4;
9680 }
9681 if (len & 2) {
9682 crc = __crc32h(crc, get_unaligned_le16(p));
9683 p += 2;
9684 }
9685 if (len & 1)
9686 crc = __crc32b(crc, *p);
9687 return crc;
9688 }
9689  
9690 #undef SUFFIX
9691 #undef ATTRIBUTES
9692 #undef ENABLE_EOR3
9693  
9694 #endif
9695  
9696 static inline crc32_func_t
9697 arch_select_crc32_func(void)
9698 {
9699 const u32 features MAYBE_UNUSED = get_arm_cpu_features();
9700  
9701 #ifdef crc32_arm_pmullx12_crc_eor3
9702 if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
9703 HAVE_PMULL(features) && HAVE_CRC32(features) && HAVE_SHA3(features))
9704 return crc32_arm_pmullx12_crc_eor3;
9705 #endif
9706 #ifdef crc32_arm_pmullx12_crc
9707 if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
9708 HAVE_PMULL(features) && HAVE_CRC32(features))
9709 return crc32_arm_pmullx12_crc;
9710 #endif
9711 #ifdef crc32_arm_crc_pmullcombine
9712 if (HAVE_CRC32(features) && HAVE_PMULL(features))
9713 return crc32_arm_crc_pmullcombine;
9714 #endif
9715 #ifdef crc32_arm_crc
9716 if (HAVE_CRC32(features))
9717 return crc32_arm_crc;
9718 #endif
9719 #ifdef crc32_arm_pmullx4
9720 if (HAVE_PMULL(features))
9721 return crc32_arm_pmullx4;
9722 #endif
9723 return NULL;
9724 }
9725 #define arch_select_crc32_func arch_select_crc32_func
9726  
9727 #endif
9728  
9729 #elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
9730 /* # include "x86/crc32_impl.h" */
9731  
9732  
9733 #ifndef LIB_X86_CRC32_IMPL_H
9734 #define LIB_X86_CRC32_IMPL_H
9735  
9736 /* #include "x86-cpu_features.h" */
9737  
9738  
9739 #ifndef LIB_X86_CPU_FEATURES_H
9740 #define LIB_X86_CPU_FEATURES_H
9741  
9742 /* #include "lib_common.h" */
9743  
9744  
9745 #ifndef LIB_LIB_COMMON_H
9746 #define LIB_LIB_COMMON_H
9747  
9748 #ifdef LIBDEFLATE_H
9749
9750 # error "lib_common.h must always be included before libdeflate.h"
9751 #endif
9752  
9753 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
9754 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
9755 #elif defined(__GNUC__)
9756 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
9757 #else
9758 # define LIBDEFLATE_EXPORT_SYM
9759 #endif
9760  
9761  
9762 #if defined(__GNUC__) && defined(__i386__)
9763 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
9764 #else
9765 # define LIBDEFLATE_ALIGN_STACK
9766 #endif
9767  
9768 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
9769  
9770 /* #include "../common_defs.h" */
9771  
9772  
9773 #ifndef COMMON_DEFS_H
9774 #define COMMON_DEFS_H
9775  
9776 /* #include "libdeflate.h" */
9777  
9778  
9779 #ifndef LIBDEFLATE_H
9780 #define LIBDEFLATE_H
9781  
9782 #include <stddef.h>
9783 #include <stdint.h>
9784  
9785 #ifdef __cplusplus
9786 extern "C" {
9787 #endif
9788  
9789 #define LIBDEFLATE_VERSION_MAJOR 1
9790 #define LIBDEFLATE_VERSION_MINOR 26
9791 #define LIBDEFLATE_VERSION_STRING "1.26"
9792  
9793  
9794 #ifndef LIBDEFLATEAPI
9795 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
9796 # define LIBDEFLATEAPI __declspec(dllimport)
9797 # else
9798 # define LIBDEFLATEAPI
9799 # endif
9800 #endif
9801  
9802  
9803  
9804  
9805  
9806 struct libdeflate_compressor;
9807 struct libdeflate_options;
9808  
9809  
9810 LIBDEFLATEAPI struct libdeflate_compressor *
9811 libdeflate_alloc_compressor(int compression_level);
9812  
9813  
9814 LIBDEFLATEAPI struct libdeflate_compressor *
9815 libdeflate_alloc_compressor_ex(int compression_level,
9816 const struct libdeflate_options *options);
9817  
9818  
9819 LIBDEFLATEAPI size_t
9820 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
9821 const void *in, size_t in_nbytes,
9822 void *out, size_t out_nbytes_avail);
9823  
9824  
9825 LIBDEFLATEAPI size_t
9826 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
9827 size_t in_nbytes);
9828  
9829  
9830 LIBDEFLATEAPI size_t
9831 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
9832 const void *in, size_t in_nbytes,
9833 void *out, size_t out_nbytes_avail);
9834  
9835  
9836 LIBDEFLATEAPI size_t
9837 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
9838 size_t in_nbytes);
9839  
9840  
9841 LIBDEFLATEAPI size_t
9842 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
9843 const void *in, size_t in_nbytes,
9844 void *out, size_t out_nbytes_avail);
9845  
9846  
9847 LIBDEFLATEAPI size_t
9848 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
9849 size_t in_nbytes);
9850  
9851  
9852 LIBDEFLATEAPI void
9853 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
9854  
9855  
9856  
9857  
9858  
9859 struct libdeflate_decompressor;
9860 struct libdeflate_options;
9861  
9862  
9863 LIBDEFLATEAPI struct libdeflate_decompressor *
9864 libdeflate_alloc_decompressor(void);
9865  
9866  
9867 LIBDEFLATEAPI struct libdeflate_decompressor *
9868 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
9869  
9870  
9871 enum libdeflate_result {
9872
9873 LIBDEFLATE_SUCCESS = 0,
9874  
9875
9876 LIBDEFLATE_BAD_DATA = 1,
9877  
9878
9879 LIBDEFLATE_SHORT_OUTPUT = 2,
9880  
9881
9882 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
9883 };
9884  
9885  
9886 LIBDEFLATEAPI enum libdeflate_result
9887 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
9888 const void *in, size_t in_nbytes,
9889 void *out, size_t out_nbytes_avail,
9890 size_t *actual_out_nbytes_ret);
9891  
9892  
9893 LIBDEFLATEAPI enum libdeflate_result
9894 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
9895 const void *in, size_t in_nbytes,
9896 void *out, size_t out_nbytes_avail,
9897 size_t *actual_in_nbytes_ret,
9898 size_t *actual_out_nbytes_ret);
9899  
9900  
9901 LIBDEFLATEAPI enum libdeflate_result
9902 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
9903 const void *in, size_t in_nbytes,
9904 void *out, size_t out_nbytes_avail,
9905 size_t *actual_out_nbytes_ret);
9906  
9907  
9908 LIBDEFLATEAPI enum libdeflate_result
9909 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
9910 const void *in, size_t in_nbytes,
9911 void *out, size_t out_nbytes_avail,
9912 size_t *actual_in_nbytes_ret,
9913 size_t *actual_out_nbytes_ret);
9914  
9915  
9916 LIBDEFLATEAPI enum libdeflate_result
9917 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
9918 const void *in, size_t in_nbytes,
9919 void *out, size_t out_nbytes_avail,
9920 size_t *actual_out_nbytes_ret);
9921  
9922  
9923 LIBDEFLATEAPI enum libdeflate_result
9924 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
9925 const void *in, size_t in_nbytes,
9926 void *out, size_t out_nbytes_avail,
9927 size_t *actual_in_nbytes_ret,
9928 size_t *actual_out_nbytes_ret);
9929  
9930  
9931 LIBDEFLATEAPI void
9932 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
9933  
9934  
9935  
9936  
9937  
9938  
9939 LIBDEFLATEAPI uint32_t
9940 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
9941  
9942  
9943  
9944 LIBDEFLATEAPI uint32_t
9945 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
9946  
9947  
9948  
9949  
9950  
9951  
9952 LIBDEFLATEAPI void
9953 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
9954 void (*free_func)(void *));
9955  
9956  
9957 struct libdeflate_options {
9958  
9959
9960 size_t sizeof_options;
9961  
9962
9963 void *(*malloc_func)(size_t);
9964 void (*free_func)(void *);
9965 };
9966  
9967 #ifdef __cplusplus
9968 }
9969 #endif
9970  
9971 #endif
9972  
9973  
9974 #include <stdbool.h>
9975 #include <stddef.h>
9976 #include <stdint.h>
9977 #ifdef _MSC_VER
9978 # include <intrin.h>
9979 # include <stdlib.h>
9980
9981
9982 # pragma warning(disable : 4146)
9983
9984 # pragma warning(disable : 4018)
9985 # pragma warning(disable : 4244)
9986 # pragma warning(disable : 4267)
9987 # pragma warning(disable : 4310)
9988
9989 # pragma warning(disable : 4100)
9990 # pragma warning(disable : 4127)
9991 # pragma warning(disable : 4189)
9992 # pragma warning(disable : 4232)
9993 # pragma warning(disable : 4245)
9994 # pragma warning(disable : 4295)
9995 #endif
9996 #ifndef FREESTANDING
9997 # include <string.h>
9998 #endif
9999  
10000  
10001  
10002  
10003  
10004  
10005 #undef ARCH_X86_64
10006 #undef ARCH_X86_32
10007 #undef ARCH_ARM64
10008 #undef ARCH_ARM32
10009 #undef ARCH_RISCV
10010 #ifdef _MSC_VER
10011
10012 # if defined(_M_X64) && !defined(_M_ARM64EC)
10013 # define ARCH_X86_64
10014 # elif defined(_M_IX86)
10015 # define ARCH_X86_32
10016 # elif defined(_M_ARM64)
10017 # define ARCH_ARM64
10018 # elif defined(_M_ARM)
10019 # define ARCH_ARM32
10020 # endif
10021 #else
10022 # if defined(__x86_64__)
10023 # define ARCH_X86_64
10024 # elif defined(__i386__)
10025 # define ARCH_X86_32
10026 # elif defined(__aarch64__)
10027 # define ARCH_ARM64
10028 # elif defined(__arm__)
10029 # define ARCH_ARM32
10030 # elif defined(__riscv)
10031 # define ARCH_RISCV
10032 # endif
10033 #endif
10034  
10035  
10036  
10037  
10038  
10039  
10040 typedef uint8_t u8;
10041 typedef uint16_t u16;
10042 typedef uint32_t u32;
10043 typedef uint64_t u64;
10044 typedef int8_t s8;
10045 typedef int16_t s16;
10046 typedef int32_t s32;
10047 typedef int64_t s64;
10048  
10049  
10050 #ifdef _MSC_VER
10051 # ifdef _WIN64
10052 typedef long long ssize_t;
10053 # else
10054 typedef long ssize_t;
10055 # endif
10056 #endif
10057  
10058  
10059 typedef size_t machine_word_t;
10060  
10061  
10062 #define WORDBYTES ((int)sizeof(machine_word_t))
10063  
10064  
10065 #define WORDBITS (8 * WORDBYTES)
10066  
10067  
10068  
10069  
10070  
10071  
10072 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
10073 # define GCC_PREREQ(major, minor) \
10074 (__GNUC__ > (major) || \
10075 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
10076 # if !GCC_PREREQ(4, 9)
10077 # error "gcc versions older than 4.9 are no longer supported"
10078 # endif
10079 #else
10080 # define GCC_PREREQ(major, minor) 0
10081 #endif
10082 #ifdef __clang__
10083 # ifdef __apple_build_version__
10084 # define CLANG_PREREQ(major, minor, apple_version) \
10085 (__apple_build_version__ >= (apple_version))
10086 # else
10087 # define CLANG_PREREQ(major, minor, apple_version) \
10088 (__clang_major__ > (major) || \
10089 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
10090 # endif
10091 # if !CLANG_PREREQ(3, 9, 8000000)
10092 # error "clang versions older than 3.9 are no longer supported"
10093 # endif
10094 #else
10095 # define CLANG_PREREQ(major, minor, apple_version) 0
10096 #endif
10097 #ifdef _MSC_VER
10098 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
10099 # if !MSVC_PREREQ(1928)
10100 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
10101 # endif
10102 #else
10103 # define MSVC_PREREQ(version) 0
10104 #endif
10105  
10106  
10107 #ifndef __has_attribute
10108 # define __has_attribute(attribute) 0
10109 #endif
10110  
10111  
10112 #ifndef __has_builtin
10113 # define __has_builtin(builtin) 0
10114 #endif
10115  
10116  
10117 #ifdef _MSC_VER
10118 # define inline __inline
10119 #endif
10120  
10121  
10122 #if defined(__GNUC__) || __has_attribute(always_inline)
10123 # define forceinline inline __attribute__((always_inline))
10124 #elif defined(_MSC_VER)
10125 # define forceinline __forceinline
10126 #else
10127 # define forceinline inline
10128 #endif
10129  
10130  
10131 #if defined(__GNUC__) || __has_attribute(unused)
10132 # define MAYBE_UNUSED __attribute__((unused))
10133 #else
10134 # define MAYBE_UNUSED
10135 #endif
10136  
10137  
10138 #if defined(__GNUC__) || __has_attribute(noreturn)
10139 # define NORETURN __attribute__((noreturn))
10140 #else
10141 # define NORETURN
10142 #endif
10143  
10144  
10145 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
10146 # define likely(expr) __builtin_expect(!!(expr), 1)
10147 #else
10148 # define likely(expr) (expr)
10149 #endif
10150  
10151  
10152 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
10153 # define unlikely(expr) __builtin_expect(!!(expr), 0)
10154 #else
10155 # define unlikely(expr) (expr)
10156 #endif
10157  
10158  
10159 #undef prefetchr
10160 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
10161 # define prefetchr(addr) __builtin_prefetch((addr), 0)
10162 #elif defined(_MSC_VER)
10163 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
10164 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
10165 # elif defined(ARCH_ARM64)
10166 # define prefetchr(addr) __prefetch2((addr), 0x00 )
10167 # elif defined(ARCH_ARM32)
10168 # define prefetchr(addr) __prefetch(addr)
10169 # endif
10170 #endif
10171 #ifndef prefetchr
10172 # define prefetchr(addr)
10173 #endif
10174  
10175  
10176 #undef prefetchw
10177 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
10178 # define prefetchw(addr) __builtin_prefetch((addr), 1)
10179 #elif defined(_MSC_VER)
10180 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
10181 # define prefetchw(addr) _m_prefetchw(addr)
10182 # elif defined(ARCH_ARM64)
10183 # define prefetchw(addr) __prefetch2((addr), 0x10 )
10184 # elif defined(ARCH_ARM32)
10185 # define prefetchw(addr) __prefetchw(addr)
10186 # endif
10187 #endif
10188 #ifndef prefetchw
10189 # define prefetchw(addr)
10190 #endif
10191  
10192  
10193 #undef _aligned_attribute
10194 #if defined(__GNUC__) || __has_attribute(aligned)
10195 # define _aligned_attribute(n) __attribute__((aligned(n)))
10196 #elif defined(_MSC_VER)
10197 # define _aligned_attribute(n) __declspec(align(n))
10198 #endif
10199  
10200  
10201 #if defined(__GNUC__) || __has_attribute(target)
10202 # define _target_attribute(attrs) __attribute__((target(attrs)))
10203 #else
10204 # define _target_attribute(attrs)
10205 #endif
10206  
10207  
10208  
10209  
10210  
10211 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
10212 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
10213 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
10214 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
10215 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
10216 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
10217 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
10218  
10219  
10220  
10221  
10222  
10223  
10224 #if defined(__BYTE_ORDER__)
10225 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
10226 #elif defined(_MSC_VER)
10227 # define CPU_IS_LITTLE_ENDIAN() true
10228 #else
10229 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
10230 {
10231 union {
10232 u32 w;
10233 u8 b;
10234 } u;
10235  
10236 u.w = 1;
10237 return u.b;
10238 }
10239 #endif
10240  
10241  
10242 static forceinline u16 bswap16(u16 v)
10243 {
10244 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
10245 return __builtin_bswap16(v);
10246 #elif defined(_MSC_VER)
10247 return _byteswap_ushort(v);
10248 #else
10249 return (v << 8) | (v >> 8);
10250 #endif
10251 }
10252  
10253  
10254 static forceinline u32 bswap32(u32 v)
10255 {
10256 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
10257 return __builtin_bswap32(v);
10258 #elif defined(_MSC_VER)
10259 return _byteswap_ulong(v);
10260 #else
10261 return ((v & 0x000000FF) << 24) |
10262 ((v & 0x0000FF00) << 8) |
10263 ((v & 0x00FF0000) >> 8) |
10264 ((v & 0xFF000000) >> 24);
10265 #endif
10266 }
10267  
10268  
10269 static forceinline u64 bswap64(u64 v)
10270 {
10271 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
10272 return __builtin_bswap64(v);
10273 #elif defined(_MSC_VER)
10274 return _byteswap_uint64(v);
10275 #else
10276 return ((v & 0x00000000000000FF) << 56) |
10277 ((v & 0x000000000000FF00) << 40) |
10278 ((v & 0x0000000000FF0000) << 24) |
10279 ((v & 0x00000000FF000000) << 8) |
10280 ((v & 0x000000FF00000000) >> 8) |
10281 ((v & 0x0000FF0000000000) >> 24) |
10282 ((v & 0x00FF000000000000) >> 40) |
10283 ((v & 0xFF00000000000000) >> 56);
10284 #endif
10285 }
10286  
10287 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
10288 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
10289 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
10290 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
10291 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
10292 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
10293  
10294  
10295  
10296  
10297  
10298  
10299 #if (defined(__GNUC__) || defined(__clang__)) && \
10300 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
10301 defined(__ARM_FEATURE_UNALIGNED) || \
10302 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
10303 defined(__riscv_misaligned_fast) || \
10304 defined(__wasm__))
10305 # define UNALIGNED_ACCESS_IS_FAST 1
10306 #elif defined(_MSC_VER)
10307 # define UNALIGNED_ACCESS_IS_FAST 1
10308 #else
10309 # define UNALIGNED_ACCESS_IS_FAST 0
10310 #endif
10311  
10312  
10313  
10314 #ifdef FREESTANDING
10315 # define MEMCOPY __builtin_memcpy
10316 #else
10317 # define MEMCOPY memcpy
10318 #endif
10319  
10320  
10321  
10322 #define DEFINE_UNALIGNED_TYPE(type) \
10323 static forceinline type \
10324 load_##type##_unaligned(const void *p) \
10325 { \
10326 type v; \
10327 \
10328 MEMCOPY(&v, p, sizeof(v)); \
10329 return v; \
10330 } \
10331 \
10332 static forceinline void \
10333 store_##type##_unaligned(type v, void *p) \
10334 { \
10335 MEMCOPY(p, &v, sizeof(v)); \
10336 }
10337  
10338 DEFINE_UNALIGNED_TYPE(u16)
10339 DEFINE_UNALIGNED_TYPE(u32)
10340 DEFINE_UNALIGNED_TYPE(u64)
10341 DEFINE_UNALIGNED_TYPE(machine_word_t)
10342  
10343 #undef MEMCOPY
10344  
10345 #define load_word_unaligned load_machine_word_t_unaligned
10346 #define store_word_unaligned store_machine_word_t_unaligned
10347  
10348  
10349  
10350 static forceinline u16
10351 get_unaligned_le16(const u8 *p)
10352 {
10353 if (UNALIGNED_ACCESS_IS_FAST)
10354 return le16_bswap(load_u16_unaligned(p));
10355 else
10356 return ((u16)p[1] << 8) | p[0];
10357 }
10358  
10359 static forceinline u16
10360 get_unaligned_be16(const u8 *p)
10361 {
10362 if (UNALIGNED_ACCESS_IS_FAST)
10363 return be16_bswap(load_u16_unaligned(p));
10364 else
10365 return ((u16)p[0] << 8) | p[1];
10366 }
10367  
10368 static forceinline u32
10369 get_unaligned_le32(const u8 *p)
10370 {
10371 if (UNALIGNED_ACCESS_IS_FAST)
10372 return le32_bswap(load_u32_unaligned(p));
10373 else
10374 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
10375 ((u32)p[1] << 8) | p[0];
10376 }
10377  
10378 static forceinline u32
10379 get_unaligned_be32(const u8 *p)
10380 {
10381 if (UNALIGNED_ACCESS_IS_FAST)
10382 return be32_bswap(load_u32_unaligned(p));
10383 else
10384 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
10385 ((u32)p[2] << 8) | p[3];
10386 }
10387  
10388 static forceinline u64
10389 get_unaligned_le64(const u8 *p)
10390 {
10391 if (UNALIGNED_ACCESS_IS_FAST)
10392 return le64_bswap(load_u64_unaligned(p));
10393 else
10394 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
10395 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
10396 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
10397 ((u64)p[1] << 8) | p[0];
10398 }
10399  
10400 static forceinline machine_word_t
10401 get_unaligned_leword(const u8 *p)
10402 {
10403 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
10404 if (WORDBITS == 32)
10405 return get_unaligned_le32(p);
10406 else
10407 return get_unaligned_le64(p);
10408 }
10409  
10410  
10411  
10412 static forceinline void
10413 put_unaligned_le16(u16 v, u8 *p)
10414 {
10415 if (UNALIGNED_ACCESS_IS_FAST) {
10416 store_u16_unaligned(le16_bswap(v), p);
10417 } else {
10418 p[0] = (u8)(v >> 0);
10419 p[1] = (u8)(v >> 8);
10420 }
10421 }
10422  
10423 static forceinline void
10424 put_unaligned_be16(u16 v, u8 *p)
10425 {
10426 if (UNALIGNED_ACCESS_IS_FAST) {
10427 store_u16_unaligned(be16_bswap(v), p);
10428 } else {
10429 p[0] = (u8)(v >> 8);
10430 p[1] = (u8)(v >> 0);
10431 }
10432 }
10433  
10434 static forceinline void
10435 put_unaligned_le32(u32 v, u8 *p)
10436 {
10437 if (UNALIGNED_ACCESS_IS_FAST) {
10438 store_u32_unaligned(le32_bswap(v), p);
10439 } else {
10440 p[0] = (u8)(v >> 0);
10441 p[1] = (u8)(v >> 8);
10442 p[2] = (u8)(v >> 16);
10443 p[3] = (u8)(v >> 24);
10444 }
10445 }
10446  
10447 static forceinline void
10448 put_unaligned_be32(u32 v, u8 *p)
10449 {
10450 if (UNALIGNED_ACCESS_IS_FAST) {
10451 store_u32_unaligned(be32_bswap(v), p);
10452 } else {
10453 p[0] = (u8)(v >> 24);
10454 p[1] = (u8)(v >> 16);
10455 p[2] = (u8)(v >> 8);
10456 p[3] = (u8)(v >> 0);
10457 }
10458 }
10459  
10460 static forceinline void
10461 put_unaligned_le64(u64 v, u8 *p)
10462 {
10463 if (UNALIGNED_ACCESS_IS_FAST) {
10464 store_u64_unaligned(le64_bswap(v), p);
10465 } else {
10466 p[0] = (u8)(v >> 0);
10467 p[1] = (u8)(v >> 8);
10468 p[2] = (u8)(v >> 16);
10469 p[3] = (u8)(v >> 24);
10470 p[4] = (u8)(v >> 32);
10471 p[5] = (u8)(v >> 40);
10472 p[6] = (u8)(v >> 48);
10473 p[7] = (u8)(v >> 56);
10474 }
10475 }
10476  
10477 static forceinline void
10478 put_unaligned_leword(machine_word_t v, u8 *p)
10479 {
10480 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
10481 if (WORDBITS == 32)
10482 put_unaligned_le32(v, p);
10483 else
10484 put_unaligned_le64(v, p);
10485 }
10486  
10487  
10488  
10489  
10490  
10491  
10492  
10493 static forceinline unsigned
10494 bsr32(u32 v)
10495 {
10496 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
10497 return 31 - __builtin_clz(v);
10498 #elif defined(_MSC_VER)
10499 unsigned long i;
10500  
10501 _BitScanReverse(&i, v);
10502 return i;
10503 #else
10504 unsigned i = 0;
10505  
10506 while ((v >>= 1) != 0)
10507 i++;
10508 return i;
10509 #endif
10510 }
10511  
10512 static forceinline unsigned
10513 bsr64(u64 v)
10514 {
10515 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
10516 return 63 - __builtin_clzll(v);
10517 #elif defined(_MSC_VER) && defined(_WIN64)
10518 unsigned long i;
10519  
10520 _BitScanReverse64(&i, v);
10521 return i;
10522 #else
10523 unsigned i = 0;
10524  
10525 while ((v >>= 1) != 0)
10526 i++;
10527 return i;
10528 #endif
10529 }
10530  
10531 static forceinline unsigned
10532 bsrw(machine_word_t v)
10533 {
10534 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
10535 if (WORDBITS == 32)
10536 return bsr32(v);
10537 else
10538 return bsr64(v);
10539 }
10540  
10541  
10542  
10543 static forceinline unsigned
10544 bsf32(u32 v)
10545 {
10546 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
10547 return __builtin_ctz(v);
10548 #elif defined(_MSC_VER)
10549 unsigned long i;
10550  
10551 _BitScanForward(&i, v);
10552 return i;
10553 #else
10554 unsigned i = 0;
10555  
10556 for (; (v & 1) == 0; v >>= 1)
10557 i++;
10558 return i;
10559 #endif
10560 }
10561  
10562 static forceinline unsigned
10563 bsf64(u64 v)
10564 {
10565 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
10566 return __builtin_ctzll(v);
10567 #elif defined(_MSC_VER) && defined(_WIN64)
10568 unsigned long i;
10569  
10570 _BitScanForward64(&i, v);
10571 return i;
10572 #else
10573 unsigned i = 0;
10574  
10575 for (; (v & 1) == 0; v >>= 1)
10576 i++;
10577 return i;
10578 #endif
10579 }
10580  
10581 static forceinline unsigned
10582 bsfw(machine_word_t v)
10583 {
10584 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
10585 if (WORDBITS == 32)
10586 return bsf32(v);
10587 else
10588 return bsf64(v);
10589 }
10590  
10591  
10592 #undef rbit32
10593 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
10594 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
10595 static forceinline u32
10596 rbit32(u32 v)
10597 {
10598 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
10599 return v;
10600 }
10601 #define rbit32 rbit32
10602 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
10603 static forceinline u32
10604 rbit32(u32 v)
10605 {
10606 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
10607 return v;
10608 }
10609 #define rbit32 rbit32
10610 #endif
10611  
10612 #endif
10613  
10614  
10615 typedef void *(*malloc_func_t)(size_t);
10616 typedef void (*free_func_t)(void *);
10617  
10618 extern malloc_func_t libdeflate_default_malloc_func;
10619 extern free_func_t libdeflate_default_free_func;
10620  
10621 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
10622 size_t alignment, size_t size);
10623 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
10624  
10625 #ifdef FREESTANDING
10626  
10627 void *memset(void *s, int c, size_t n);
10628 #define memset(s, c, n) __builtin_memset((s), (c), (n))
10629  
10630 void *memcpy(void *dest, const void *src, size_t n);
10631 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
10632  
10633 void *memmove(void *dest, const void *src, size_t n);
10634 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
10635  
10636 int memcmp(const void *s1, const void *s2, size_t n);
10637 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
10638  
10639 #undef LIBDEFLATE_ENABLE_ASSERTIONS
10640 #else
10641 # include <string.h>
10642
10643 # ifdef __clang_analyzer__
10644 # define LIBDEFLATE_ENABLE_ASSERTIONS
10645 # endif
10646 #endif
10647  
10648  
10649 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
10650 NORETURN void
10651 libdeflate_assertion_failed(const char *expr, const char *file, int line);
10652 #define ASSERT(expr) { if (unlikely(!(expr))) \
10653 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
10654 #else
10655 #define ASSERT(expr) (void)(expr)
10656 #endif
10657  
10658 #define CONCAT_IMPL(a, b) a##b
10659 #define CONCAT(a, b) CONCAT_IMPL(a, b)
10660 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
10661  
10662 #endif
10663  
10664  
10665 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
10666  
10667 #define X86_CPU_FEATURE_SSE2 (1 << 0)
10668 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
10669 #define X86_CPU_FEATURE_AVX (1 << 2)
10670 #define X86_CPU_FEATURE_AVX2 (1 << 3)
10671 #define X86_CPU_FEATURE_BMI2 (1 << 4)
10672  
10673 #define X86_CPU_FEATURE_ZMM (1 << 5)
10674 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
10675 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
10676 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
10677 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
10678 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
10679  
10680 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
10681  
10682 # define X86_CPU_FEATURES_KNOWN (1U << 31)
10683 extern volatile u32 libdeflate_x86_cpu_features;
10684  
10685 void libdeflate_init_x86_cpu_features(void);
10686  
10687 static inline u32 get_x86_cpu_features(void)
10688 {
10689 if (libdeflate_x86_cpu_features == 0)
10690 libdeflate_init_x86_cpu_features();
10691 return libdeflate_x86_cpu_features;
10692 }
10693  
10694 # include <immintrin.h>
10695 # if defined(_MSC_VER) && defined(__clang__)
10696 # include <tmmintrin.h>
10697 # include <smmintrin.h>
10698 # include <wmmintrin.h>
10699 # include <avxintrin.h>
10700 # include <avx2intrin.h>
10701 # include <avx512fintrin.h>
10702 # include <avx512bwintrin.h>
10703 # include <avx512vlintrin.h>
10704 # if __has_include(<avx512vlbwintrin.h>)
10705 # include <avx512vlbwintrin.h>
10706 # endif
10707 # if __has_include(<vpclmulqdqintrin.h>)
10708 # include <vpclmulqdqintrin.h>
10709 # endif
10710 # if __has_include(<avx512vnniintrin.h>)
10711 # include <avx512vnniintrin.h>
10712 # endif
10713 # if __has_include(<avx512vlvnniintrin.h>)
10714 # include <avx512vlvnniintrin.h>
10715 # endif
10716 # if __has_include(<avxvnniintrin.h>)
10717 # include <avxvnniintrin.h>
10718 # endif
10719 # endif
10720 #else
10721 static inline u32 get_x86_cpu_features(void) { return 0; }
10722 #endif
10723  
10724 #if defined(__SSE2__) || \
10725 (defined(_MSC_VER) && \
10726 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
10727 # define HAVE_SSE2(features) 1
10728 # define HAVE_SSE2_NATIVE 1
10729 #else
10730 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
10731 # define HAVE_SSE2_NATIVE 0
10732 #endif
10733  
10734 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
10735 (defined(_MSC_VER) && defined(__AVX2__))
10736 # define HAVE_PCLMULQDQ(features) 1
10737 #else
10738 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
10739 #endif
10740  
10741 #ifdef __AVX__
10742 # define HAVE_AVX(features) 1
10743 #else
10744 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
10745 #endif
10746  
10747 #ifdef __AVX2__
10748 # define HAVE_AVX2(features) 1
10749 #else
10750 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
10751 #endif
10752  
10753 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
10754 # define HAVE_BMI2(features) 1
10755 # define HAVE_BMI2_NATIVE 1
10756 #else
10757 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
10758 # define HAVE_BMI2_NATIVE 0
10759 #endif
10760  
10761 #ifdef __AVX512BW__
10762 # define HAVE_AVX512BW(features) 1
10763 #else
10764 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
10765 #endif
10766  
10767 #ifdef __AVX512VL__
10768 # define HAVE_AVX512VL(features) 1
10769 #else
10770 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
10771 #endif
10772  
10773 #ifdef __VPCLMULQDQ__
10774 # define HAVE_VPCLMULQDQ(features) 1
10775 #else
10776 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
10777 #endif
10778  
10779 #ifdef __AVX512VNNI__
10780 # define HAVE_AVX512VNNI(features) 1
10781 #else
10782 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
10783 #endif
10784  
10785 #ifdef __AVXVNNI__
10786 # define HAVE_AVXVNNI(features) 1
10787 #else
10788 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
10789 #endif
10790  
10791 #endif
10792  
10793 #endif
10794  
10795  
10796  
10797 static const u8 MAYBE_UNUSED shift_tab[48] = {
10798 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
10799 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
10800 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
10801 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
10802 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
10803 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
10804 };
10805  
10806 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
10807  
10808 # define crc32_x86_pclmulqdq crc32_x86_pclmulqdq
10809 # define SUFFIX _pclmulqdq
10810 # define ATTRIBUTES _target_attribute("pclmul,sse4.1")
10811 # define VL 16
10812 # define USE_AVX512 0
10813 /* #include "x86-crc32_pclmul_template.h" */
10814  
10815  
10816  
10817  
10818 #if VL == 16
10819 # define vec_t __m128i
10820 # define fold_vec fold_vec128
10821 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
10822 # define VXOR(a, b) _mm_xor_si128((a), (b))
10823 # define M128I_TO_VEC(a) a
10824 # define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
10825 # define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
10826 # define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
10827 # define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
10828 #elif VL == 32
10829 # define vec_t __m256i
10830 # define fold_vec fold_vec256
10831 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
10832 # define VXOR(a, b) _mm256_xor_si256((a), (b))
10833 # define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
10834 # define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
10835 # define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
10836 # define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
10837 # define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
10838 # define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
10839 #elif VL == 64
10840 # define vec_t __m512i
10841 # define fold_vec fold_vec512
10842 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
10843 # define VXOR(a, b) _mm512_xor_si512((a), (b))
10844 # define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
10845 # define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
10846 # define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
10847 # define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
10848 # define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
10849 # define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
10850 #else
10851 # error "unsupported vector length"
10852 #endif
10853  
10854 #undef fold_vec128
10855 static forceinline ATTRIBUTES __m128i
10856 ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i mults)
10857 {
10858 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
10859 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
10860 0 return dst;
10861 }
10862 #define fold_vec128 ADD_SUFFIX(fold_vec128)
10863  
10864 #if VL >= 32
10865 #undef fold_vec256
10866 static forceinline ATTRIBUTES __m256i
10867 ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i mults)
10868 {
10869 #if USE_AVX512
10870
10871 return _mm256_ternarylogic_epi32(
10872 _mm256_clmulepi64_epi128(src, mults, 0x00),
10873 _mm256_clmulepi64_epi128(src, mults, 0x11),
10874 dst,
10875 0x96);
10876 #else
10877 return _mm256_xor_si256(
10878 _mm256_xor_si256(dst,
10879 _mm256_clmulepi64_epi128(src, mults, 0x00)),
10880 _mm256_clmulepi64_epi128(src, mults, 0x11));
10881 #endif
10882 }
10883 #define fold_vec256 ADD_SUFFIX(fold_vec256)
10884 #endif
10885  
10886 #if VL >= 64
10887 #undef fold_vec512
10888 static forceinline ATTRIBUTES __m512i
10889 ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i mults)
10890 {
10891
10892 return _mm512_ternarylogic_epi32(
10893 _mm512_clmulepi64_epi128(src, mults, 0x00),
10894 _mm512_clmulepi64_epi128(src, mults, 0x11),
10895 dst,
10896 0x96);
10897 }
10898 #define fold_vec512 ADD_SUFFIX(fold_vec512)
10899 #endif
10900  
10901  
10902 #undef fold_lessthan16bytes
10903 static forceinline ATTRIBUTES __m128i
10904 ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
10905 __m128i mults_128b)
10906 {
10907 0 __m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
10908 0 __m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
10909 __m128i x0, x1;
10910  
10911
10912 0 x0 = _mm_shuffle_epi8(x, lshift);
10913  
10914
10915 0 x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
10916 0 _mm_loadu_si128((const void *)(p + len - 16)),
10917
10918 rshift);
10919  
10920 0 return fold_vec128(x0, x1, mults_128b);
10921 }
10922 #define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
10923  
10924 static ATTRIBUTES u32
10925 0 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
10926 {
10927
10928 0 const vec_t mults_8v = MULTS_8V;
10929 0 const vec_t mults_4v = MULTS_4V;
10930 0 const vec_t mults_2v = MULTS_2V;
10931 0 const vec_t mults_1v = MULTS_1V;
10932 0 const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
10933 const __m128i barrett_reduction_constants =
10934 0 _mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
10935 vec_t v0, v1, v2, v3, v4, v5, v6, v7;
10936 0 __m128i x0 = _mm_cvtsi32_si128(crc);
10937 __m128i x1;
10938  
10939 0 ▶ if (len < 8*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 8*VL) { 0 0 0
10940 0 ▶ if (len < VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < VL) { 0 0 0
10941 STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
10942 0 ▶ if (len < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 16) { 0 0 0
10943 #if USE_AVX512
10944 if (len < 4)
10945 return crc32_slice1(crc, p, len);
10946
10947 x0 = _mm_xor_si128(
10948 x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
10949 x0 = _mm_shuffle_epi8(
10950 x0, _mm_loadu_si128((const void *)&shift_tab[len]));
10951 goto reduce_x0;
10952 #else
10953 0 return crc32_slice1(crc, p, len);
10954 #endif
10955 }
10956
10957 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
10958 0 ▶ if (len >= 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 32) { 0 0 0
10959 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
10960 mults_128b);
10961 0 ▶ if (len >= 48)
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 48) 0 0 0
10962 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
10963 mults_128b);
10964 }
10965 0 p += len & ~15;
10966 0 goto less_than_16_remaining;
10967 }
10968 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
10969 0 ▶ if (len < 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 2*VL) { 0 0 0
10970 0 p += VL;
10971 0 goto less_than_vl_remaining;
10972 }
10973 0 v1 = VLOADU(p + 1*VL);
10974 0 ▶ if (len < 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4*VL) { 0 0 0
10975 0 p += 2*VL;
10976 0 goto less_than_2vl_remaining;
10977 }
10978 0 v2 = VLOADU(p + 2*VL);
10979 0 v3 = VLOADU(p + 3*VL);
10980 0 p += 4*VL;
10981 } else {
10982
10983 0 ▶ if (len > 65536 && ((uintptr_t)p & (VL-1))) {
Branch: 2 branches
BranchTrueFalseTotal
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
10984 0 size_t align = -(uintptr_t)p & (VL-1);
10985  
10986 0 len -= align;
10987 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
10988 0 p += 16;
10989 0 ▶ if (align & 15) {
Branch: 1 branch
BranchTrueFalseTotal
if (align & 15) { 0 0 0
10990 0 x0 = fold_lessthan16bytes(x0, p, align & 15,
10991 mults_128b);
10992 0 p += align & 15;
10993 0 align &= ~15;
10994 }
10995 0 ▶ while (align) {
Branch: 1 branch
BranchTrueFalseTotal
while (align) { 0 0 0
10996 0 x0 = fold_vec128(x0, *(const __m128i *)p,
10997 mults_128b);
10998 0 p += 16;
10999 0 align -= 16;
11000 }
11001 0 v0 = M128I_TO_VEC(x0);
11002 # if VL == 32
11003 v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
11004 # elif VL == 64
11005 v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
11006 v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
11007 # endif
11008 0 p -= 16;
11009 } else {
11010 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11011 }
11012 0 v1 = VLOADU(p + 1*VL);
11013 0 v2 = VLOADU(p + 2*VL);
11014 0 v3 = VLOADU(p + 3*VL);
11015 0 v4 = VLOADU(p + 4*VL);
11016 0 v5 = VLOADU(p + 5*VL);
11017 0 v6 = VLOADU(p + 6*VL);
11018 0 v7 = VLOADU(p + 7*VL);
11019 0 p += 8*VL;
11020  
11021
11022 0 ▶ while (len >= 16*VL) {
Branch: 1 branch
BranchTrueFalseTotal
while (len >= 16*VL) { 0 0 0
11023 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
11024 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
11025 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
11026 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
11027 0 v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
11028 0 v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
11029 0 v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
11030 0 v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
11031 0 p += 8*VL;
11032 0 len -= 8*VL;
11033 }
11034  
11035
11036 0 v0 = fold_vec(v0, v4, mults_4v);
11037 0 v1 = fold_vec(v1, v5, mults_4v);
11038 0 v2 = fold_vec(v2, v6, mults_4v);
11039 0 v3 = fold_vec(v3, v7, mults_4v);
11040 0 ▶ if (len & (4*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (4*VL)) { 0 0 0
11041 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
11042 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
11043 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
11044 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
11045 0 p += 4*VL;
11046 }
11047 }
11048
11049 0 v0 = fold_vec(v0, v2, mults_2v);
11050 0 v1 = fold_vec(v1, v3, mults_2v);
11051 0 ▶ if (len & (2*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (2*VL)) { 0 0 0
11052 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
11053 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
11054 0 p += 2*VL;
11055 }
11056 0 less_than_2vl_remaining:
11057
11058 0 v0 = fold_vec(v0, v1, mults_1v);
11059 0 ▶ if (len & VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & VL) { 0 0 0
11060 0 v0 = fold_vec(v0, VLOADU(p), mults_1v);
11061 0 p += VL;
11062 }
11063 0 less_than_vl_remaining:
11064
11065 #if VL == 16
11066 0 x0 = v0;
11067 #else
11068 {
11069 #if VL == 32
11070 __m256i y0 = v0;
11071 #else
11072 const __m256i mults_256b =
11073 _mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
11074 CRC32_X223_MODG, CRC32_X287_MODG);
11075 __m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
11076 _mm512_extracti64x4_epi64(v0, 1),
11077 mults_256b);
11078 if (len & 32) {
11079 y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
11080 mults_256b);
11081 p += 32;
11082 }
11083 #endif
11084 x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
11085 _mm256_extracti128_si256(y0, 1), mults_128b);
11086 }
11087 if (len & 16) {
11088 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
11089 mults_128b);
11090 p += 16;
11091 }
11092 #endif
11093 0 less_than_16_remaining:
11094 0 len &= 15;
11095  
11096
11097 0 ▶ if (len)
Branch: 1 branch
BranchTrueFalseTotal
if (len) 0 0 0
11098 0 x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
11099 #if USE_AVX512
11100 reduce_x0:
11101 #endif
11102
11103 0 x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
11104 0 _mm_bsrli_si128(x0, 8));
11105 0 x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
11106 0 x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
11107 0 x0 = _mm_xor_si128(x0, x1);
11108 0 return _mm_extract_epi32(x0, 2);
11109 }
11110  
11111 #undef vec_t
11112 #undef fold_vec
11113 #undef VLOADU
11114 #undef VXOR
11115 #undef M128I_TO_VEC
11116 #undef MULTS
11117 #undef MULTS_8V
11118 #undef MULTS_4V
11119 #undef MULTS_2V
11120 #undef MULTS_1V
11121  
11122 #undef SUFFIX
11123 #undef ATTRIBUTES
11124 #undef VL
11125 #undef USE_AVX512
11126  
11127  
11128  
11129 # define crc32_x86_pclmulqdq_avx crc32_x86_pclmulqdq_avx
11130 # define SUFFIX _pclmulqdq_avx
11131 # define ATTRIBUTES _target_attribute("pclmul,avx")
11132 # define VL 16
11133 # define USE_AVX512 0
11134 /* #include "x86-crc32_pclmul_template.h" */
11135  
11136  
11137  
11138  
11139 #if VL == 16
11140 # define vec_t __m128i
11141 # define fold_vec fold_vec128
11142 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
11143 # define VXOR(a, b) _mm_xor_si128((a), (b))
11144 # define M128I_TO_VEC(a) a
11145 # define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
11146 # define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
11147 # define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
11148 # define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
11149 #elif VL == 32
11150 # define vec_t __m256i
11151 # define fold_vec fold_vec256
11152 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
11153 # define VXOR(a, b) _mm256_xor_si256((a), (b))
11154 # define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
11155 # define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
11156 # define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11157 # define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11158 # define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11159 # define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
11160 #elif VL == 64
11161 # define vec_t __m512i
11162 # define fold_vec fold_vec512
11163 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
11164 # define VXOR(a, b) _mm512_xor_si512((a), (b))
11165 # define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
11166 # define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
11167 # define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
11168 # define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11169 # define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11170 # define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11171 #else
11172 # error "unsupported vector length"
11173 #endif
11174  
11175 #undef fold_vec128
11176 static forceinline ATTRIBUTES __m128i
11177 ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i mults)
11178 {
11179 58 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
11180 11484 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
11181 11484 return dst;
11182 }
11183 #define fold_vec128 ADD_SUFFIX(fold_vec128)
11184  
11185 #if VL >= 32
11186 #undef fold_vec256
11187 static forceinline ATTRIBUTES __m256i
11188 ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i mults)
11189 {
11190 #if USE_AVX512
11191
11192 return _mm256_ternarylogic_epi32(
11193 _mm256_clmulepi64_epi128(src, mults, 0x00),
11194 _mm256_clmulepi64_epi128(src, mults, 0x11),
11195 dst,
11196 0x96);
11197 #else
11198 return _mm256_xor_si256(
11199 _mm256_xor_si256(dst,
11200 _mm256_clmulepi64_epi128(src, mults, 0x00)),
11201 _mm256_clmulepi64_epi128(src, mults, 0x11));
11202 #endif
11203 }
11204 #define fold_vec256 ADD_SUFFIX(fold_vec256)
11205 #endif
11206  
11207 #if VL >= 64
11208 #undef fold_vec512
11209 static forceinline ATTRIBUTES __m512i
11210 ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i mults)
11211 {
11212
11213 return _mm512_ternarylogic_epi32(
11214 _mm512_clmulepi64_epi128(src, mults, 0x00),
11215 _mm512_clmulepi64_epi128(src, mults, 0x11),
11216 dst,
11217 0x96);
11218 }
11219 #define fold_vec512 ADD_SUFFIX(fold_vec512)
11220 #endif
11221  
11222  
11223 #undef fold_lessthan16bytes
11224 static forceinline ATTRIBUTES __m128i
11225 ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
11226 __m128i mults_128b)
11227 {
11228 29 __m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
11229 58 __m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
11230 __m128i x0, x1;
11231  
11232
11233 29 x0 = _mm_shuffle_epi8(x, lshift);
11234  
11235
11236 87 x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
11237 29 _mm_loadu_si128((const void *)(p + len - 16)),
11238
11239 rshift);
11240  
11241 29 return fold_vec128(x0, x1, mults_128b);
11242 }
11243 #define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
11244  
11245 static ATTRIBUTES u32
11246 29 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
11247 {
11248
11249 29 const vec_t mults_8v = MULTS_8V;
11250 29 const vec_t mults_4v = MULTS_4V;
11251 29 const vec_t mults_2v = MULTS_2V;
11252 29 const vec_t mults_1v = MULTS_1V;
11253 29 const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
11254 const __m128i barrett_reduction_constants =
11255 29 _mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
11256 vec_t v0, v1, v2, v3, v4, v5, v6, v7;
11257 29 __m128i x0 = _mm_cvtsi32_si128(crc);
11258 __m128i x1;
11259  
11260 29 ▶ if (len < 8*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 8*VL) { 0 29 29
11261 0 ▶ if (len < VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < VL) { 0 0 0
11262 STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
11263 0 ▶ if (len < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 16) { 0 0 0
11264 #if USE_AVX512
11265 if (len < 4)
11266 return crc32_slice1(crc, p, len);
11267
11268 x0 = _mm_xor_si128(
11269 x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
11270 x0 = _mm_shuffle_epi8(
11271 x0, _mm_loadu_si128((const void *)&shift_tab[len]));
11272 goto reduce_x0;
11273 #else
11274 0 return crc32_slice1(crc, p, len);
11275 #endif
11276 }
11277
11278 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11279 0 ▶ if (len >= 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 32) { 0 0 0
11280 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
11281 mults_128b);
11282 0 ▶ if (len >= 48)
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 48) 0 0 0
11283 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
11284 mults_128b);
11285 }
11286 0 p += len & ~15;
11287 0 goto less_than_16_remaining;
11288 }
11289 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11290 0 ▶ if (len < 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 2*VL) { 0 0 0
11291 0 p += VL;
11292 0 goto less_than_vl_remaining;
11293 }
11294 0 v1 = VLOADU(p + 1*VL);
11295 0 ▶ if (len < 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4*VL) { 0 0 0
11296 0 p += 2*VL;
11297 0 goto less_than_2vl_remaining;
11298 }
11299 0 v2 = VLOADU(p + 2*VL);
11300 0 v3 = VLOADU(p + 3*VL);
11301 0 p += 4*VL;
11302 } else {
11303
11304 29 ▶ if (len > 65536 && ((uintptr_t)p & (VL-1))) {
Branch: 2 branches
BranchTrueFalseTotal
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 29 29
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
11305 0 size_t align = -(uintptr_t)p & (VL-1);
11306  
11307 0 len -= align;
11308 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11309 0 p += 16;
11310 0 ▶ if (align & 15) {
Branch: 1 branch
BranchTrueFalseTotal
if (align & 15) { 0 0 0
11311 0 x0 = fold_lessthan16bytes(x0, p, align & 15,
11312 mults_128b);
11313 0 p += align & 15;
11314 0 align &= ~15;
11315 }
11316 0 ▶ while (align) {
Branch: 1 branch
BranchTrueFalseTotal
while (align) { 0 0 0
11317 0 x0 = fold_vec128(x0, *(const __m128i *)p,
11318 mults_128b);
11319 0 p += 16;
11320 0 align -= 16;
11321 }
11322 0 v0 = M128I_TO_VEC(x0);
11323 # if VL == 32
11324 v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
11325 # elif VL == 64
11326 v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
11327 v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
11328 # endif
11329 0 p -= 16;
11330 } else {
11331 58 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11332 }
11333 29 v1 = VLOADU(p + 1*VL);
11334 29 v2 = VLOADU(p + 2*VL);
11335 29 v3 = VLOADU(p + 3*VL);
11336 29 v4 = VLOADU(p + 4*VL);
11337 29 v5 = VLOADU(p + 5*VL);
11338 29 v6 = VLOADU(p + 6*VL);
11339 29 v7 = VLOADU(p + 7*VL);
11340 29 p += 8*VL;
11341  
11342
11343 1427 ▶ while (len >= 16*VL) {
Branch: 1 branch
BranchTrueFalseTotal
while (len >= 16*VL) { 1398 29 1427
11344 1398 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
11345 2796 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
11346 2796 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
11347 2796 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
11348 2796 v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
11349 2796 v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
11350 2796 v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
11351 2796 v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
11352 1398 p += 8*VL;
11353 1398 len -= 8*VL;
11354 }
11355  
11356
11357 29 v0 = fold_vec(v0, v4, mults_4v);
11358 29 v1 = fold_vec(v1, v5, mults_4v);
11359 29 v2 = fold_vec(v2, v6, mults_4v);
11360 29 v3 = fold_vec(v3, v7, mults_4v);
11361 29 ▶ if (len & (4*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (4*VL)) { 5 24 29
11362 5 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
11363 10 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
11364 10 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
11365 10 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
11366 5 p += 4*VL;
11367 }
11368 }
11369
11370 29 v0 = fold_vec(v0, v2, mults_2v);
11371 29 v1 = fold_vec(v1, v3, mults_2v);
11372 29 ▶ if (len & (2*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (2*VL)) { 5 24 29
11373 24 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
11374 48 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
11375 24 p += 2*VL;
11376 }
11377 29 less_than_2vl_remaining:
11378
11379 29 v0 = fold_vec(v0, v1, mults_1v);
11380 29 ▶ if (len & VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & VL) { 29 0 29
11381 0 v0 = fold_vec(v0, VLOADU(p), mults_1v);
11382 0 p += VL;
11383 }
11384 29 less_than_vl_remaining:
11385
11386 #if VL == 16
11387 29 x0 = v0;
11388 #else
11389 {
11390 #if VL == 32
11391 __m256i y0 = v0;
11392 #else
11393 const __m256i mults_256b =
11394 _mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
11395 CRC32_X223_MODG, CRC32_X287_MODG);
11396 __m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
11397 _mm512_extracti64x4_epi64(v0, 1),
11398 mults_256b);
11399 if (len & 32) {
11400 y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
11401 mults_256b);
11402 p += 32;
11403 }
11404 #endif
11405 x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
11406 _mm256_extracti128_si256(y0, 1), mults_128b);
11407 }
11408 if (len & 16) {
11409 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
11410 mults_128b);
11411 p += 16;
11412 }
11413 #endif
11414 29 less_than_16_remaining:
11415 29 len &= 15;
11416  
11417
11418 29 ▶ if (len)
Branch: 1 branch
BranchTrueFalseTotal
if (len) 29 0 29
11419 29 x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
11420 #if USE_AVX512
11421 reduce_x0:
11422 #endif
11423
11424 58 x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
11425 29 _mm_bsrli_si128(x0, 8));
11426 29 x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
11427 29 x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
11428 29 x0 = _mm_xor_si128(x0, x1);
11429 29 return _mm_extract_epi32(x0, 2);
11430 }
11431  
11432 #undef vec_t
11433 #undef fold_vec
11434 #undef VLOADU
11435 #undef VXOR
11436 #undef M128I_TO_VEC
11437 #undef MULTS
11438 #undef MULTS_8V
11439 #undef MULTS_4V
11440 #undef MULTS_2V
11441 #undef MULTS_1V
11442  
11443 #undef SUFFIX
11444 #undef ATTRIBUTES
11445 #undef VL
11446 #undef USE_AVX512
11447  
11448 #endif
11449  
11450  
11451 #if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
11452 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
11453 # define crc32_x86_vpclmulqdq_avx2 crc32_x86_vpclmulqdq_avx2
11454 # define SUFFIX _vpclmulqdq_avx2
11455 # define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx2")
11456 # define VL 32
11457 # define USE_AVX512 0
11458 /* #include "x86-crc32_pclmul_template.h" */
11459  
11460  
11461  
11462  
11463 #if VL == 16
11464 # define vec_t __m128i
11465 # define fold_vec fold_vec128
11466 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
11467 # define VXOR(a, b) _mm_xor_si128((a), (b))
11468 # define M128I_TO_VEC(a) a
11469 # define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
11470 # define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
11471 # define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
11472 # define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
11473 #elif VL == 32
11474 # define vec_t __m256i
11475 # define fold_vec fold_vec256
11476 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
11477 # define VXOR(a, b) _mm256_xor_si256((a), (b))
11478 # define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
11479 # define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
11480 # define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11481 # define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11482 # define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11483 # define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
11484 #elif VL == 64
11485 # define vec_t __m512i
11486 # define fold_vec fold_vec512
11487 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
11488 # define VXOR(a, b) _mm512_xor_si512((a), (b))
11489 # define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
11490 # define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
11491 # define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
11492 # define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11493 # define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11494 # define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11495 #else
11496 # error "unsupported vector length"
11497 #endif
11498  
11499 #undef fold_vec128
11500 static forceinline ATTRIBUTES __m128i
11501 ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i mults)
11502 {
11503 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
11504 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
11505 0 return dst;
11506 }
11507 #define fold_vec128 ADD_SUFFIX(fold_vec128)
11508  
11509 #if VL >= 32
11510 #undef fold_vec256
11511 static forceinline ATTRIBUTES __m256i
11512 ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i mults)
11513 {
11514 #if USE_AVX512
11515
11516 return _mm256_ternarylogic_epi32(
11517 _mm256_clmulepi64_epi128(src, mults, 0x00),
11518 _mm256_clmulepi64_epi128(src, mults, 0x11),
11519 dst,
11520 0x96);
11521 #else
11522 0 return _mm256_xor_si256(
11523 _mm256_xor_si256(dst,
11524 0 _mm256_clmulepi64_epi128(src, mults, 0x00)),
11525 0 _mm256_clmulepi64_epi128(src, mults, 0x11));
11526 #endif
11527 }
11528 #define fold_vec256 ADD_SUFFIX(fold_vec256)
11529 #endif
11530  
11531 #if VL >= 64
11532 #undef fold_vec512
11533 static forceinline ATTRIBUTES __m512i
11534 ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i mults)
11535 {
11536
11537 return _mm512_ternarylogic_epi32(
11538 _mm512_clmulepi64_epi128(src, mults, 0x00),
11539 _mm512_clmulepi64_epi128(src, mults, 0x11),
11540 dst,
11541 0x96);
11542 }
11543 #define fold_vec512 ADD_SUFFIX(fold_vec512)
11544 #endif
11545  
11546  
11547 #undef fold_lessthan16bytes
11548 static forceinline ATTRIBUTES __m128i
11549 ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
11550 __m128i mults_128b)
11551 {
11552 0 __m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
11553 0 __m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
11554 __m128i x0, x1;
11555  
11556
11557 0 x0 = _mm_shuffle_epi8(x, lshift);
11558  
11559
11560 0 x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
11561 0 _mm_loadu_si128((const void *)(p + len - 16)),
11562
11563 rshift);
11564  
11565 0 return fold_vec128(x0, x1, mults_128b);
11566 }
11567 #define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
11568  
11569 static ATTRIBUTES u32
11570 0 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
11571 {
11572
11573 0 const vec_t mults_8v = MULTS_8V;
11574 0 const vec_t mults_4v = MULTS_4V;
11575 0 const vec_t mults_2v = MULTS_2V;
11576 0 const vec_t mults_1v = MULTS_1V;
11577 0 const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
11578 const __m128i barrett_reduction_constants =
11579 0 _mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
11580 vec_t v0, v1, v2, v3, v4, v5, v6, v7;
11581 0 __m128i x0 = _mm_cvtsi32_si128(crc);
11582 __m128i x1;
11583  
11584 0 ▶ if (len < 8*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 8*VL) { 0 0 0
11585 0 ▶ if (len < VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < VL) { 0 0 0
11586 STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
11587 0 ▶ if (len < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 16) { 0 0 0
11588 #if USE_AVX512
11589 if (len < 4)
11590 return crc32_slice1(crc, p, len);
11591
11592 x0 = _mm_xor_si128(
11593 x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
11594 x0 = _mm_shuffle_epi8(
11595 x0, _mm_loadu_si128((const void *)&shift_tab[len]));
11596 goto reduce_x0;
11597 #else
11598 0 return crc32_slice1(crc, p, len);
11599 #endif
11600 }
11601
11602 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11603 0 ▶ if (len >= 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 32) { 0 0 0
11604 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
11605 mults_128b);
11606 0 ▶ if (len >= 48)
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 48) 0 0 0
11607 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
11608 mults_128b);
11609 }
11610 0 p += len & ~15;
11611 0 goto less_than_16_remaining;
11612 }
11613 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11614 0 ▶ if (len < 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 2*VL) { 0 0 0
11615 0 p += VL;
11616 0 goto less_than_vl_remaining;
11617 }
11618 0 v1 = VLOADU(p + 1*VL);
11619 0 ▶ if (len < 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4*VL) { 0 0 0
11620 0 p += 2*VL;
11621 0 goto less_than_2vl_remaining;
11622 }
11623 0 v2 = VLOADU(p + 2*VL);
11624 0 v3 = VLOADU(p + 3*VL);
11625 0 p += 4*VL;
11626 } else {
11627
11628 0 ▶ if (len > 65536 && ((uintptr_t)p & (VL-1))) {
Branch: 2 branches
BranchTrueFalseTotal
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
11629 0 size_t align = -(uintptr_t)p & (VL-1);
11630  
11631 0 len -= align;
11632 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11633 0 p += 16;
11634 0 ▶ if (align & 15) {
Branch: 1 branch
BranchTrueFalseTotal
if (align & 15) { 0 0 0
11635 0 x0 = fold_lessthan16bytes(x0, p, align & 15,
11636 mults_128b);
11637 0 p += align & 15;
11638 0 align &= ~15;
11639 }
11640 0 ▶ while (align) {
Branch: 1 branch
BranchTrueFalseTotal
while (align) { 0 0 0
11641 0 x0 = fold_vec128(x0, *(const __m128i *)p,
11642 mults_128b);
11643 0 p += 16;
11644 0 align -= 16;
11645 }
11646 0 v0 = M128I_TO_VEC(x0);
11647 # if VL == 32
11648 0 v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
11649 # elif VL == 64
11650 v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
11651 v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
11652 # endif
11653 0 p -= 16;
11654 } else {
11655 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11656 }
11657 0 v1 = VLOADU(p + 1*VL);
11658 0 v2 = VLOADU(p + 2*VL);
11659 0 v3 = VLOADU(p + 3*VL);
11660 0 v4 = VLOADU(p + 4*VL);
11661 0 v5 = VLOADU(p + 5*VL);
11662 0 v6 = VLOADU(p + 6*VL);
11663 0 v7 = VLOADU(p + 7*VL);
11664 0 p += 8*VL;
11665  
11666
11667 0 ▶ while (len >= 16*VL) {
Branch: 1 branch
BranchTrueFalseTotal
while (len >= 16*VL) { 0 0 0
11668 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
11669 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
11670 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
11671 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
11672 0 v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
11673 0 v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
11674 0 v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
11675 0 v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
11676 0 p += 8*VL;
11677 0 len -= 8*VL;
11678 }
11679  
11680
11681 0 v0 = fold_vec(v0, v4, mults_4v);
11682 0 v1 = fold_vec(v1, v5, mults_4v);
11683 0 v2 = fold_vec(v2, v6, mults_4v);
11684 0 v3 = fold_vec(v3, v7, mults_4v);
11685 0 ▶ if (len & (4*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (4*VL)) { 0 0 0
11686 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
11687 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
11688 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
11689 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
11690 0 p += 4*VL;
11691 }
11692 }
11693
11694 0 v0 = fold_vec(v0, v2, mults_2v);
11695 0 v1 = fold_vec(v1, v3, mults_2v);
11696 0 ▶ if (len & (2*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (2*VL)) { 0 0 0
11697 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
11698 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
11699 0 p += 2*VL;
11700 }
11701 0 less_than_2vl_remaining:
11702
11703 0 v0 = fold_vec(v0, v1, mults_1v);
11704 0 ▶ if (len & VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & VL) { 0 0 0
11705 0 v0 = fold_vec(v0, VLOADU(p), mults_1v);
11706 0 p += VL;
11707 }
11708 0 less_than_vl_remaining:
11709
11710 #if VL == 16
11711 x0 = v0;
11712 #else
11713 {
11714 #if VL == 32
11715 0 __m256i y0 = v0;
11716 #else
11717 const __m256i mults_256b =
11718 _mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
11719 CRC32_X223_MODG, CRC32_X287_MODG);
11720 __m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
11721 _mm512_extracti64x4_epi64(v0, 1),
11722 mults_256b);
11723 if (len & 32) {
11724 y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
11725 mults_256b);
11726 p += 32;
11727 }
11728 #endif
11729 0 x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
11730 0 _mm256_extracti128_si256(y0, 1), mults_128b);
11731 }
11732 0 ▶ if (len & 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & 16) { 0 0 0
11733 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
11734 mults_128b);
11735 0 p += 16;
11736 }
11737 #endif
11738 0 less_than_16_remaining:
11739 0 len &= 15;
11740  
11741
11742 0 ▶ if (len)
Branch: 1 branch
BranchTrueFalseTotal
if (len) 0 0 0
11743 0 x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
11744 #if USE_AVX512
11745 reduce_x0:
11746 #endif
11747
11748 0 x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
11749 0 _mm_bsrli_si128(x0, 8));
11750 0 x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
11751 0 x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
11752 0 x0 = _mm_xor_si128(x0, x1);
11753 0 return _mm_extract_epi32(x0, 2);
11754 }
11755  
11756 #undef vec_t
11757 #undef fold_vec
11758 #undef VLOADU
11759 #undef VXOR
11760 #undef M128I_TO_VEC
11761 #undef MULTS
11762 #undef MULTS_8V
11763 #undef MULTS_4V
11764 #undef MULTS_2V
11765 #undef MULTS_1V
11766  
11767 #undef SUFFIX
11768 #undef ATTRIBUTES
11769 #undef VL
11770 #undef USE_AVX512
11771  
11772 #endif
11773  
11774 #if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
11775 !(CLANG_PREREQ(18, 0, 18000000) && !CLANG_PREREQ(19, 0, 19000000)) && \
11776 !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
11777  
11778 # define crc32_x86_vpclmulqdq_avx512_vl256 crc32_x86_vpclmulqdq_avx512_vl256
11779 # define SUFFIX _vpclmulqdq_avx512_vl256
11780 # define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
11781 # define VL 32
11782 # define USE_AVX512 1
11783 /* #include "x86-crc32_pclmul_template.h" */
11784  
11785  
11786  
11787  
11788 #if VL == 16
11789 # define vec_t __m128i
11790 # define fold_vec fold_vec128
11791 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
11792 # define VXOR(a, b) _mm_xor_si128((a), (b))
11793 # define M128I_TO_VEC(a) a
11794 # define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
11795 # define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
11796 # define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
11797 # define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
11798 #elif VL == 32
11799 # define vec_t __m256i
11800 # define fold_vec fold_vec256
11801 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
11802 # define VXOR(a, b) _mm256_xor_si256((a), (b))
11803 # define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
11804 # define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
11805 # define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11806 # define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11807 # define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11808 # define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
11809 #elif VL == 64
11810 # define vec_t __m512i
11811 # define fold_vec fold_vec512
11812 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
11813 # define VXOR(a, b) _mm512_xor_si512((a), (b))
11814 # define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
11815 # define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
11816 # define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
11817 # define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
11818 # define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
11819 # define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
11820 #else
11821 # error "unsupported vector length"
11822 #endif
11823  
11824 #undef fold_vec128
11825 static forceinline ATTRIBUTES __m128i
11826 ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i mults)
11827 {
11828 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
11829 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
11830 0 return dst;
11831 }
11832 #define fold_vec128 ADD_SUFFIX(fold_vec128)
11833  
11834 #if VL >= 32
11835 #undef fold_vec256
11836 static forceinline ATTRIBUTES __m256i
11837 ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i mults)
11838 {
11839 #if USE_AVX512
11840
11841 0 return _mm256_ternarylogic_epi32(
11842 _mm256_clmulepi64_epi128(src, mults, 0x00),
11843 _mm256_clmulepi64_epi128(src, mults, 0x11),
11844 dst,
11845 0x96);
11846 #else
11847 return _mm256_xor_si256(
11848 _mm256_xor_si256(dst,
11849 _mm256_clmulepi64_epi128(src, mults, 0x00)),
11850 _mm256_clmulepi64_epi128(src, mults, 0x11));
11851 #endif
11852 }
11853 #define fold_vec256 ADD_SUFFIX(fold_vec256)
11854 #endif
11855  
11856 #if VL >= 64
11857 #undef fold_vec512
11858 static forceinline ATTRIBUTES __m512i
11859 ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i mults)
11860 {
11861
11862 return _mm512_ternarylogic_epi32(
11863 _mm512_clmulepi64_epi128(src, mults, 0x00),
11864 _mm512_clmulepi64_epi128(src, mults, 0x11),
11865 dst,
11866 0x96);
11867 }
11868 #define fold_vec512 ADD_SUFFIX(fold_vec512)
11869 #endif
11870  
11871  
11872 #undef fold_lessthan16bytes
11873 static forceinline ATTRIBUTES __m128i
11874 ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
11875 __m128i mults_128b)
11876 {
11877 0 __m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
11878 0 __m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
11879 __m128i x0, x1;
11880  
11881
11882 0 x0 = _mm_shuffle_epi8(x, lshift);
11883  
11884
11885 0 x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
11886 0 _mm_loadu_si128((const void *)(p + len - 16)),
11887
11888 rshift);
11889  
11890 0 return fold_vec128(x0, x1, mults_128b);
11891 }
11892 #define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
11893  
11894 static ATTRIBUTES u32
11895 0 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
11896 {
11897
11898 0 const vec_t mults_8v = MULTS_8V;
11899 0 const vec_t mults_4v = MULTS_4V;
11900 0 const vec_t mults_2v = MULTS_2V;
11901 0 const vec_t mults_1v = MULTS_1V;
11902 0 const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
11903 const __m128i barrett_reduction_constants =
11904 0 _mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
11905 vec_t v0, v1, v2, v3, v4, v5, v6, v7;
11906 0 __m128i x0 = _mm_cvtsi32_si128(crc);
11907 __m128i x1;
11908  
11909 0 ▶ if (len < 8*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 8*VL) { 0 0 0
11910 0 ▶ if (len < VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < VL) { 0 0 0
11911 STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
11912 0 ▶ if (len < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 16) { 0 0 0
11913 #if USE_AVX512
11914 0 ▶ if (len < 4)
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4) 0 0 0
11915 0 return crc32_slice1(crc, p, len);
11916
11917 0 x0 = _mm_xor_si128(
11918 0 x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
11919 0 x0 = _mm_shuffle_epi8(
11920 0 x0, _mm_loadu_si128((const void *)&shift_tab[len]));
11921 0 goto reduce_x0;
11922 #else
11923 return crc32_slice1(crc, p, len);
11924 #endif
11925 }
11926
11927 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11928 0 ▶ if (len >= 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 32) { 0 0 0
11929 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
11930 mults_128b);
11931 0 ▶ if (len >= 48)
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 48) 0 0 0
11932 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
11933 mults_128b);
11934 }
11935 0 p += len & ~15;
11936 0 goto less_than_16_remaining;
11937 }
11938 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11939 0 ▶ if (len < 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 2*VL) { 0 0 0
11940 0 p += VL;
11941 0 goto less_than_vl_remaining;
11942 }
11943 0 v1 = VLOADU(p + 1*VL);
11944 0 ▶ if (len < 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4*VL) { 0 0 0
11945 0 p += 2*VL;
11946 0 goto less_than_2vl_remaining;
11947 }
11948 0 v2 = VLOADU(p + 2*VL);
11949 0 v3 = VLOADU(p + 3*VL);
11950 0 p += 4*VL;
11951 } else {
11952
11953 0 ▶ if (len > 65536 && ((uintptr_t)p & (VL-1))) {
Branch: 2 branches
BranchTrueFalseTotal
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
11954 0 size_t align = -(uintptr_t)p & (VL-1);
11955  
11956 0 len -= align;
11957 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
11958 0 p += 16;
11959 0 ▶ if (align & 15) {
Branch: 1 branch
BranchTrueFalseTotal
if (align & 15) { 0 0 0
11960 0 x0 = fold_lessthan16bytes(x0, p, align & 15,
11961 mults_128b);
11962 0 p += align & 15;
11963 0 align &= ~15;
11964 }
11965 0 ▶ while (align) {
Branch: 1 branch
BranchTrueFalseTotal
while (align) { 0 0 0
11966 0 x0 = fold_vec128(x0, *(const __m128i *)p,
11967 mults_128b);
11968 0 p += 16;
11969 0 align -= 16;
11970 }
11971 0 v0 = M128I_TO_VEC(x0);
11972 # if VL == 32
11973 0 v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
11974 # elif VL == 64
11975 v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
11976 v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
11977 # endif
11978 0 p -= 16;
11979 } else {
11980 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
11981 }
11982 0 v1 = VLOADU(p + 1*VL);
11983 0 v2 = VLOADU(p + 2*VL);
11984 0 v3 = VLOADU(p + 3*VL);
11985 0 v4 = VLOADU(p + 4*VL);
11986 0 v5 = VLOADU(p + 5*VL);
11987 0 v6 = VLOADU(p + 6*VL);
11988 0 v7 = VLOADU(p + 7*VL);
11989 0 p += 8*VL;
11990  
11991
11992 0 ▶ while (len >= 16*VL) {
Branch: 1 branch
BranchTrueFalseTotal
while (len >= 16*VL) { 0 0 0
11993 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
11994 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
11995 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
11996 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
11997 0 v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
11998 0 v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
11999 0 v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
12000 0 v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
12001 0 p += 8*VL;
12002 0 len -= 8*VL;
12003 }
12004  
12005
12006 0 v0 = fold_vec(v0, v4, mults_4v);
12007 0 v1 = fold_vec(v1, v5, mults_4v);
12008 0 v2 = fold_vec(v2, v6, mults_4v);
12009 0 v3 = fold_vec(v3, v7, mults_4v);
12010 0 ▶ if (len & (4*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (4*VL)) { 0 0 0
12011 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
12012 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
12013 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
12014 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
12015 0 p += 4*VL;
12016 }
12017 }
12018
12019 0 v0 = fold_vec(v0, v2, mults_2v);
12020 0 v1 = fold_vec(v1, v3, mults_2v);
12021 0 ▶ if (len & (2*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (2*VL)) { 0 0 0
12022 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
12023 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
12024 0 p += 2*VL;
12025 }
12026 0 less_than_2vl_remaining:
12027
12028 0 v0 = fold_vec(v0, v1, mults_1v);
12029 0 ▶ if (len & VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & VL) { 0 0 0
12030 0 v0 = fold_vec(v0, VLOADU(p), mults_1v);
12031 0 p += VL;
12032 }
12033 0 less_than_vl_remaining:
12034
12035 #if VL == 16
12036 x0 = v0;
12037 #else
12038 {
12039 #if VL == 32
12040 0 __m256i y0 = v0;
12041 #else
12042 const __m256i mults_256b =
12043 _mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
12044 CRC32_X223_MODG, CRC32_X287_MODG);
12045 __m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
12046 _mm512_extracti64x4_epi64(v0, 1),
12047 mults_256b);
12048 if (len & 32) {
12049 y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
12050 mults_256b);
12051 p += 32;
12052 }
12053 #endif
12054 0 x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
12055 0 _mm256_extracti128_si256(y0, 1), mults_128b);
12056 }
12057 0 ▶ if (len & 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & 16) { 0 0 0
12058 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
12059 mults_128b);
12060 0 p += 16;
12061 }
12062 #endif
12063 0 less_than_16_remaining:
12064 0 len &= 15;
12065  
12066
12067 0 ▶ if (len)
Branch: 1 branch
BranchTrueFalseTotal
if (len) 0 0 0
12068 0 x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
12069 #if USE_AVX512
12070 0 reduce_x0:
12071 #endif
12072
12073 0 x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
12074 0 _mm_bsrli_si128(x0, 8));
12075 0 x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
12076 0 x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
12077 0 x0 = _mm_xor_si128(x0, x1);
12078 0 return _mm_extract_epi32(x0, 2);
12079 }
12080  
12081 #undef vec_t
12082 #undef fold_vec
12083 #undef VLOADU
12084 #undef VXOR
12085 #undef M128I_TO_VEC
12086 #undef MULTS
12087 #undef MULTS_8V
12088 #undef MULTS_4V
12089 #undef MULTS_2V
12090 #undef MULTS_1V
12091  
12092 #undef SUFFIX
12093 #undef ATTRIBUTES
12094 #undef VL
12095 #undef USE_AVX512
12096  
12097  
12098  
12099 # define crc32_x86_vpclmulqdq_avx512_vl512 crc32_x86_vpclmulqdq_avx512_vl512
12100 # define SUFFIX _vpclmulqdq_avx512_vl512
12101 # define ATTRIBUTES _target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
12102 # define VL 64
12103 # define USE_AVX512 1
12104 /* #include "x86-crc32_pclmul_template.h" */
12105  
12106  
12107  
12108  
12109 #if VL == 16
12110 # define vec_t __m128i
12111 # define fold_vec fold_vec128
12112 # define VLOADU(p) _mm_loadu_si128((const void *)(p))
12113 # define VXOR(a, b) _mm_xor_si128((a), (b))
12114 # define M128I_TO_VEC(a) a
12115 # define MULTS_8V _mm_set_epi64x(CRC32_X991_MODG, CRC32_X1055_MODG)
12116 # define MULTS_4V _mm_set_epi64x(CRC32_X479_MODG, CRC32_X543_MODG)
12117 # define MULTS_2V _mm_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG)
12118 # define MULTS_1V _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG)
12119 #elif VL == 32
12120 # define vec_t __m256i
12121 # define fold_vec fold_vec256
12122 # define VLOADU(p) _mm256_loadu_si256((const void *)(p))
12123 # define VXOR(a, b) _mm256_xor_si256((a), (b))
12124 # define M128I_TO_VEC(a) _mm256_zextsi128_si256(a)
12125 # define MULTS(a, b) _mm256_set_epi64x(a, b, a, b)
12126 # define MULTS_8V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
12127 # define MULTS_4V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
12128 # define MULTS_2V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
12129 # define MULTS_1V MULTS(CRC32_X223_MODG, CRC32_X287_MODG)
12130 #elif VL == 64
12131 # define vec_t __m512i
12132 # define fold_vec fold_vec512
12133 # define VLOADU(p) _mm512_loadu_si512((const void *)(p))
12134 # define VXOR(a, b) _mm512_xor_si512((a), (b))
12135 # define M128I_TO_VEC(a) _mm512_zextsi128_si512(a)
12136 # define MULTS(a, b) _mm512_set_epi64(a, b, a, b, a, b, a, b)
12137 # define MULTS_8V MULTS(CRC32_X4063_MODG, CRC32_X4127_MODG)
12138 # define MULTS_4V MULTS(CRC32_X2015_MODG, CRC32_X2079_MODG)
12139 # define MULTS_2V MULTS(CRC32_X991_MODG, CRC32_X1055_MODG)
12140 # define MULTS_1V MULTS(CRC32_X479_MODG, CRC32_X543_MODG)
12141 #else
12142 # error "unsupported vector length"
12143 #endif
12144  
12145 #undef fold_vec128
12146 static forceinline ATTRIBUTES __m128i
12147 ADD_SUFFIX(fold_vec128)(__m128i src, __m128i dst, __m128i mults)
12148 {
12149 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x00));
12150 0 dst = _mm_xor_si128(dst, _mm_clmulepi64_si128(src, mults, 0x11));
12151 0 return dst;
12152 }
12153 #define fold_vec128 ADD_SUFFIX(fold_vec128)
12154  
12155 #if VL >= 32
12156 #undef fold_vec256
12157 static forceinline ATTRIBUTES __m256i
12158 ADD_SUFFIX(fold_vec256)(__m256i src, __m256i dst, __m256i mults)
12159 {
12160 #if USE_AVX512
12161
12162 0 return _mm256_ternarylogic_epi32(
12163 _mm256_clmulepi64_epi128(src, mults, 0x00),
12164 _mm256_clmulepi64_epi128(src, mults, 0x11),
12165 dst,
12166 0x96);
12167 #else
12168 return _mm256_xor_si256(
12169 _mm256_xor_si256(dst,
12170 _mm256_clmulepi64_epi128(src, mults, 0x00)),
12171 _mm256_clmulepi64_epi128(src, mults, 0x11));
12172 #endif
12173 }
12174 #define fold_vec256 ADD_SUFFIX(fold_vec256)
12175 #endif
12176  
12177 #if VL >= 64
12178 #undef fold_vec512
12179 static forceinline ATTRIBUTES __m512i
12180 ADD_SUFFIX(fold_vec512)(__m512i src, __m512i dst, __m512i mults)
12181 {
12182
12183 0 return _mm512_ternarylogic_epi32(
12184 _mm512_clmulepi64_epi128(src, mults, 0x00),
12185 _mm512_clmulepi64_epi128(src, mults, 0x11),
12186 dst,
12187 0x96);
12188 }
12189 #define fold_vec512 ADD_SUFFIX(fold_vec512)
12190 #endif
12191  
12192  
12193 #undef fold_lessthan16bytes
12194 static forceinline ATTRIBUTES __m128i
12195 ADD_SUFFIX(fold_lessthan16bytes)(__m128i x, const u8 *p, size_t len,
12196 __m128i mults_128b)
12197 {
12198 0 __m128i lshift = _mm_loadu_si128((const void *)&shift_tab[len]);
12199 0 __m128i rshift = _mm_loadu_si128((const void *)&shift_tab[len + 16]);
12200 __m128i x0, x1;
12201  
12202
12203 0 x0 = _mm_shuffle_epi8(x, lshift);
12204  
12205
12206 0 x1 = _mm_blendv_epi8(_mm_shuffle_epi8(x, rshift),
12207 0 _mm_loadu_si128((const void *)(p + len - 16)),
12208
12209 rshift);
12210  
12211 0 return fold_vec128(x0, x1, mults_128b);
12212 }
12213 #define fold_lessthan16bytes ADD_SUFFIX(fold_lessthan16bytes)
12214  
12215 static ATTRIBUTES u32
12216 0 ADD_SUFFIX(crc32_x86)(u32 crc, const u8 *p, size_t len)
12217 {
12218
12219 0 const vec_t mults_8v = MULTS_8V;
12220 0 const vec_t mults_4v = MULTS_4V;
12221 0 const vec_t mults_2v = MULTS_2V;
12222 0 const vec_t mults_1v = MULTS_1V;
12223 0 const __m128i mults_128b = _mm_set_epi64x(CRC32_X95_MODG, CRC32_X159_MODG);
12224 const __m128i barrett_reduction_constants =
12225 0 _mm_set_epi64x(CRC32_BARRETT_CONSTANT_2, CRC32_BARRETT_CONSTANT_1);
12226 vec_t v0, v1, v2, v3, v4, v5, v6, v7;
12227 0 __m128i x0 = _mm_cvtsi32_si128(crc);
12228 __m128i x1;
12229  
12230 0 ▶ if (len < 8*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 8*VL) { 0 0 0
12231 0 ▶ if (len < VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < VL) { 0 0 0
12232 STATIC_ASSERT(VL == 16 || VL == 32 || VL == 64);
12233 0 ▶ if (len < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 16) { 0 0 0
12234 #if USE_AVX512
12235 0 ▶ if (len < 4)
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4) 0 0 0
12236 0 return crc32_slice1(crc, p, len);
12237
12238 0 x0 = _mm_xor_si128(
12239 0 x0, _mm_maskz_loadu_epi8((1 << len) - 1, p));
12240 0 x0 = _mm_shuffle_epi8(
12241 0 x0, _mm_loadu_si128((const void *)&shift_tab[len]));
12242 0 goto reduce_x0;
12243 #else
12244 return crc32_slice1(crc, p, len);
12245 #endif
12246 }
12247
12248 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
12249 0 ▶ if (len >= 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 32) { 0 0 0
12250 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 16)),
12251 mults_128b);
12252 0 ▶ if (len >= 48)
Branch: 1 branch
BranchTrueFalseTotal
if (len >= 48) 0 0 0
12253 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)(p + 32)),
12254 mults_128b);
12255 }
12256 0 p += len & ~15;
12257 0 goto less_than_16_remaining;
12258 }
12259 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
12260 0 ▶ if (len < 2*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 2*VL) { 0 0 0
12261 0 p += VL;
12262 0 goto less_than_vl_remaining;
12263 }
12264 0 v1 = VLOADU(p + 1*VL);
12265 0 ▶ if (len < 4*VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len < 4*VL) { 0 0 0
12266 0 p += 2*VL;
12267 0 goto less_than_2vl_remaining;
12268 }
12269 0 v2 = VLOADU(p + 2*VL);
12270 0 v3 = VLOADU(p + 3*VL);
12271 0 p += 4*VL;
12272 } else {
12273
12274 0 ▶ if (len > 65536 && ((uintptr_t)p & (VL-1))) {
Branch: 2 branches
BranchTrueFalseTotal
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
if (len > 65536 && ((uintptr_t)p & (VL-1))) { 0 0 0
12275 0 size_t align = -(uintptr_t)p & (VL-1);
12276  
12277 0 len -= align;
12278 0 x0 = _mm_xor_si128(_mm_loadu_si128((const void *)p), x0);
12279 0 p += 16;
12280 0 ▶ if (align & 15) {
Branch: 1 branch
BranchTrueFalseTotal
if (align & 15) { 0 0 0
12281 0 x0 = fold_lessthan16bytes(x0, p, align & 15,
12282 mults_128b);
12283 0 p += align & 15;
12284 0 align &= ~15;
12285 }
12286 0 ▶ while (align) {
Branch: 1 branch
BranchTrueFalseTotal
while (align) { 0 0 0
12287 0 x0 = fold_vec128(x0, *(const __m128i *)p,
12288 mults_128b);
12289 0 p += 16;
12290 0 align -= 16;
12291 }
12292 0 v0 = M128I_TO_VEC(x0);
12293 # if VL == 32
12294 v0 = _mm256_inserti128_si256(v0, *(const __m128i *)p, 1);
12295 # elif VL == 64
12296 0 v0 = _mm512_inserti32x4(v0, *(const __m128i *)p, 1);
12297 0 v0 = _mm512_inserti64x4(v0, *(const __m256i *)(p + 16), 1);
12298 # endif
12299 0 p -= 16;
12300 } else {
12301 0 v0 = VXOR(VLOADU(p), M128I_TO_VEC(x0));
12302 }
12303 0 v1 = VLOADU(p + 1*VL);
12304 0 v2 = VLOADU(p + 2*VL);
12305 0 v3 = VLOADU(p + 3*VL);
12306 0 v4 = VLOADU(p + 4*VL);
12307 0 v5 = VLOADU(p + 5*VL);
12308 0 v6 = VLOADU(p + 6*VL);
12309 0 v7 = VLOADU(p + 7*VL);
12310 0 p += 8*VL;
12311  
12312
12313 0 ▶ while (len >= 16*VL) {
Branch: 1 branch
BranchTrueFalseTotal
while (len >= 16*VL) { 0 0 0
12314 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_8v);
12315 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_8v);
12316 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_8v);
12317 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_8v);
12318 0 v4 = fold_vec(v4, VLOADU(p + 4*VL), mults_8v);
12319 0 v5 = fold_vec(v5, VLOADU(p + 5*VL), mults_8v);
12320 0 v6 = fold_vec(v6, VLOADU(p + 6*VL), mults_8v);
12321 0 v7 = fold_vec(v7, VLOADU(p + 7*VL), mults_8v);
12322 0 p += 8*VL;
12323 0 len -= 8*VL;
12324 }
12325  
12326
12327 0 v0 = fold_vec(v0, v4, mults_4v);
12328 0 v1 = fold_vec(v1, v5, mults_4v);
12329 0 v2 = fold_vec(v2, v6, mults_4v);
12330 0 v3 = fold_vec(v3, v7, mults_4v);
12331 0 ▶ if (len & (4*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (4*VL)) { 0 0 0
12332 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_4v);
12333 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_4v);
12334 0 v2 = fold_vec(v2, VLOADU(p + 2*VL), mults_4v);
12335 0 v3 = fold_vec(v3, VLOADU(p + 3*VL), mults_4v);
12336 0 p += 4*VL;
12337 }
12338 }
12339
12340 0 v0 = fold_vec(v0, v2, mults_2v);
12341 0 v1 = fold_vec(v1, v3, mults_2v);
12342 0 ▶ if (len & (2*VL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & (2*VL)) { 0 0 0
12343 0 v0 = fold_vec(v0, VLOADU(p + 0*VL), mults_2v);
12344 0 v1 = fold_vec(v1, VLOADU(p + 1*VL), mults_2v);
12345 0 p += 2*VL;
12346 }
12347 0 less_than_2vl_remaining:
12348
12349 0 v0 = fold_vec(v0, v1, mults_1v);
12350 0 ▶ if (len & VL) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & VL) { 0 0 0
12351 0 v0 = fold_vec(v0, VLOADU(p), mults_1v);
12352 0 p += VL;
12353 }
12354 0 less_than_vl_remaining:
12355
12356 #if VL == 16
12357 x0 = v0;
12358 #else
12359 {
12360 #if VL == 32
12361 __m256i y0 = v0;
12362 #else
12363 const __m256i mults_256b =
12364 0 _mm256_set_epi64x(CRC32_X223_MODG, CRC32_X287_MODG,
12365 CRC32_X223_MODG, CRC32_X287_MODG);
12366 0 __m256i y0 = fold_vec256(_mm512_extracti64x4_epi64(v0, 0),
12367 0 _mm512_extracti64x4_epi64(v0, 1),
12368 mults_256b);
12369 0 ▶ if (len & 32) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & 32) { 0 0 0
12370 0 y0 = fold_vec256(y0, _mm256_loadu_si256((const void *)p),
12371 mults_256b);
12372 0 p += 32;
12373 }
12374 #endif
12375 0 x0 = fold_vec128(_mm256_extracti128_si256(y0, 0),
12376 0 _mm256_extracti128_si256(y0, 1), mults_128b);
12377 }
12378 0 ▶ if (len & 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (len & 16) { 0 0 0
12379 0 x0 = fold_vec128(x0, _mm_loadu_si128((const void *)p),
12380 mults_128b);
12381 0 p += 16;
12382 }
12383 #endif
12384 0 less_than_16_remaining:
12385 0 len &= 15;
12386  
12387
12388 0 ▶ if (len)
Branch: 1 branch
BranchTrueFalseTotal
if (len) 0 0 0
12389 0 x0 = fold_lessthan16bytes(x0, p, len, mults_128b);
12390 #if USE_AVX512
12391 0 reduce_x0:
12392 #endif
12393
12394 0 x0 = _mm_xor_si128(_mm_clmulepi64_si128(x0, mults_128b, 0x10),
12395 0 _mm_bsrli_si128(x0, 8));
12396 0 x1 = _mm_clmulepi64_si128(x0, barrett_reduction_constants, 0x00);
12397 0 x1 = _mm_clmulepi64_si128(x1, barrett_reduction_constants, 0x10);
12398 0 x0 = _mm_xor_si128(x0, x1);
12399 0 return _mm_extract_epi32(x0, 2);
12400 }
12401  
12402 #undef vec_t
12403 #undef fold_vec
12404 #undef VLOADU
12405 #undef VXOR
12406 #undef M128I_TO_VEC
12407 #undef MULTS
12408 #undef MULTS_8V
12409 #undef MULTS_4V
12410 #undef MULTS_2V
12411 #undef MULTS_1V
12412  
12413 #undef SUFFIX
12414 #undef ATTRIBUTES
12415 #undef VL
12416 #undef USE_AVX512
12417  
12418 #endif
12419  
12420 static inline crc32_func_t
12421 2 arch_select_crc32_func(void)
12422 {
12423 2 const u32 features MAYBE_UNUSED = get_x86_cpu_features();
12424  
12425 #ifdef crc32_x86_vpclmulqdq_avx512_vl512
12426 2 ▶ if ((features & X86_CPU_FEATURE_ZMM) &&
Branch: 1 branch
BranchTrueFalseTotal
if ((features & X86_CPU_FEATURE_ZMM) && 0 2 2
12427 0 ▶ HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
Branch: 2 branches
BranchTrueFalseTotal
HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 0 0
HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 0 0
12428 0 ▶ HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
Branch: 2 branches
BranchTrueFalseTotal
HAVE_AVX512BW(features) && HAVE_AVX512VL(features)) 0 0 0
HAVE_AVX512BW(features) && HAVE_AVX512VL(features)) 0 0 0
12429 0 return crc32_x86_vpclmulqdq_avx512_vl512;
12430 #endif
12431 #ifdef crc32_x86_vpclmulqdq_avx512_vl256
12432 2 ▶ if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
Branch: 2 branches
BranchTrueFalseTotal
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 2 2
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 0 0
12433 0 ▶ HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
Branch: 2 branches
BranchTrueFalseTotal
HAVE_AVX512BW(features) && HAVE_AVX512VL(features)) 0 0 0
HAVE_AVX512BW(features) && HAVE_AVX512VL(features)) 0 0 0
12434 0 return crc32_x86_vpclmulqdq_avx512_vl256;
12435 #endif
12436 #ifdef crc32_x86_vpclmulqdq_avx2
12437 2 ▶ if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
Branch: 2 branches
BranchTrueFalseTotal
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 2 2
if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) && 0 0 0
12438 0 ▶ HAVE_AVX2(features))
Branch: 1 branch
BranchTrueFalseTotal
HAVE_AVX2(features)) 0 0 0
12439 0 return crc32_x86_vpclmulqdq_avx2;
12440 #endif
12441 #ifdef crc32_x86_pclmulqdq_avx
12442 2 ▶ if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features))
Branch: 2 branches
BranchTrueFalseTotal
if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features)) 2 0 2
if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features)) 2 0 2
12443 2 return crc32_x86_pclmulqdq_avx;
12444 #endif
12445 #ifdef crc32_x86_pclmulqdq
12446 0 ▶ if (HAVE_PCLMULQDQ(features))
Branch: 1 branch
BranchTrueFalseTotal
if (HAVE_PCLMULQDQ(features)) 0 0 0
12447 0 return crc32_x86_pclmulqdq;
12448 #endif
12449 0 return NULL;
12450 }
12451 #define arch_select_crc32_func arch_select_crc32_func
12452  
12453 #endif
12454  
12455 #endif
12456  
12457 #ifndef DEFAULT_IMPL
12458 # define DEFAULT_IMPL crc32_slice8
12459 #endif
12460  
12461 #ifdef arch_select_crc32_func
12462 static u32 crc32_dispatch_crc32(u32 crc, const u8 *p, size_t len);
12463  
12464 static volatile crc32_func_t crc32_impl = crc32_dispatch_crc32;
12465  
12466  
12467 2 static u32 crc32_dispatch_crc32(u32 crc, const u8 *p, size_t len)
12468 {
12469 2 crc32_func_t f = arch_select_crc32_func();
12470  
12471 2 ▶ if (f == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (f == NULL) 0 2 2
12472 0 f = DEFAULT_IMPL;
12473  
12474 2 crc32_impl = f;
12475 2 return f(crc, p, len);
12476 }
12477 #else
12478  
12479 #define crc32_impl DEFAULT_IMPL
12480 #endif
12481  
12482 LIBDEFLATEAPI u32
12483 29 libdeflate_crc32(u32 crc, const void *p, size_t len)
12484 {
12485 29 ▶ if (p == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (p == NULL) 0 29 29
12486 0 return 0;
12487 29 return ~crc32_impl(~crc, p, len);
12488 }
12489 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/deflate_compress.c */
12490  
12491  
12492 /* #include "deflate_compress.h" */
12493 #ifndef LIB_DEFLATE_COMPRESS_H
12494 #define LIB_DEFLATE_COMPRESS_H
12495  
12496 /* #include "lib_common.h" */
12497  
12498  
12499 #ifndef LIB_LIB_COMMON_H
12500 #define LIB_LIB_COMMON_H
12501  
12502 #ifdef LIBDEFLATE_H
12503
12504 # error "lib_common.h must always be included before libdeflate.h"
12505 #endif
12506  
12507 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
12508 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
12509 #elif defined(__GNUC__)
12510 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
12511 #else
12512 # define LIBDEFLATE_EXPORT_SYM
12513 #endif
12514  
12515  
12516 #if defined(__GNUC__) && defined(__i386__)
12517 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
12518 #else
12519 # define LIBDEFLATE_ALIGN_STACK
12520 #endif
12521  
12522 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
12523  
12524 /* #include "../common_defs.h" */
12525  
12526  
12527 #ifndef COMMON_DEFS_H
12528 #define COMMON_DEFS_H
12529  
12530 /* #include "libdeflate.h" */
12531  
12532  
12533 #ifndef LIBDEFLATE_H
12534 #define LIBDEFLATE_H
12535  
12536 #include <stddef.h>
12537 #include <stdint.h>
12538  
12539 #ifdef __cplusplus
12540 extern "C" {
12541 #endif
12542  
12543 #define LIBDEFLATE_VERSION_MAJOR 1
12544 #define LIBDEFLATE_VERSION_MINOR 26
12545 #define LIBDEFLATE_VERSION_STRING "1.26"
12546  
12547  
12548 #ifndef LIBDEFLATEAPI
12549 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
12550 # define LIBDEFLATEAPI __declspec(dllimport)
12551 # else
12552 # define LIBDEFLATEAPI
12553 # endif
12554 #endif
12555  
12556  
12557  
12558  
12559  
12560 struct libdeflate_compressor;
12561 struct libdeflate_options;
12562  
12563  
12564 LIBDEFLATEAPI struct libdeflate_compressor *
12565 libdeflate_alloc_compressor(int compression_level);
12566  
12567  
12568 LIBDEFLATEAPI struct libdeflate_compressor *
12569 libdeflate_alloc_compressor_ex(int compression_level,
12570 const struct libdeflate_options *options);
12571  
12572  
12573 LIBDEFLATEAPI size_t
12574 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
12575 const void *in, size_t in_nbytes,
12576 void *out, size_t out_nbytes_avail);
12577  
12578  
12579 LIBDEFLATEAPI size_t
12580 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
12581 size_t in_nbytes);
12582  
12583  
12584 LIBDEFLATEAPI size_t
12585 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
12586 const void *in, size_t in_nbytes,
12587 void *out, size_t out_nbytes_avail);
12588  
12589  
12590 LIBDEFLATEAPI size_t
12591 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
12592 size_t in_nbytes);
12593  
12594  
12595 LIBDEFLATEAPI size_t
12596 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
12597 const void *in, size_t in_nbytes,
12598 void *out, size_t out_nbytes_avail);
12599  
12600  
12601 LIBDEFLATEAPI size_t
12602 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
12603 size_t in_nbytes);
12604  
12605  
12606 LIBDEFLATEAPI void
12607 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
12608  
12609  
12610  
12611  
12612  
12613 struct libdeflate_decompressor;
12614 struct libdeflate_options;
12615  
12616  
12617 LIBDEFLATEAPI struct libdeflate_decompressor *
12618 libdeflate_alloc_decompressor(void);
12619  
12620  
12621 LIBDEFLATEAPI struct libdeflate_decompressor *
12622 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
12623  
12624  
12625 enum libdeflate_result {
12626
12627 LIBDEFLATE_SUCCESS = 0,
12628  
12629
12630 LIBDEFLATE_BAD_DATA = 1,
12631  
12632
12633 LIBDEFLATE_SHORT_OUTPUT = 2,
12634  
12635
12636 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
12637 };
12638  
12639  
12640 LIBDEFLATEAPI enum libdeflate_result
12641 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
12642 const void *in, size_t in_nbytes,
12643 void *out, size_t out_nbytes_avail,
12644 size_t *actual_out_nbytes_ret);
12645  
12646  
12647 LIBDEFLATEAPI enum libdeflate_result
12648 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
12649 const void *in, size_t in_nbytes,
12650 void *out, size_t out_nbytes_avail,
12651 size_t *actual_in_nbytes_ret,
12652 size_t *actual_out_nbytes_ret);
12653  
12654  
12655 LIBDEFLATEAPI enum libdeflate_result
12656 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
12657 const void *in, size_t in_nbytes,
12658 void *out, size_t out_nbytes_avail,
12659 size_t *actual_out_nbytes_ret);
12660  
12661  
12662 LIBDEFLATEAPI enum libdeflate_result
12663 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
12664 const void *in, size_t in_nbytes,
12665 void *out, size_t out_nbytes_avail,
12666 size_t *actual_in_nbytes_ret,
12667 size_t *actual_out_nbytes_ret);
12668  
12669  
12670 LIBDEFLATEAPI enum libdeflate_result
12671 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
12672 const void *in, size_t in_nbytes,
12673 void *out, size_t out_nbytes_avail,
12674 size_t *actual_out_nbytes_ret);
12675  
12676  
12677 LIBDEFLATEAPI enum libdeflate_result
12678 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
12679 const void *in, size_t in_nbytes,
12680 void *out, size_t out_nbytes_avail,
12681 size_t *actual_in_nbytes_ret,
12682 size_t *actual_out_nbytes_ret);
12683  
12684  
12685 LIBDEFLATEAPI void
12686 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
12687  
12688  
12689  
12690  
12691  
12692  
12693 LIBDEFLATEAPI uint32_t
12694 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
12695  
12696  
12697  
12698 LIBDEFLATEAPI uint32_t
12699 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
12700  
12701  
12702  
12703  
12704  
12705  
12706 LIBDEFLATEAPI void
12707 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
12708 void (*free_func)(void *));
12709  
12710  
12711 struct libdeflate_options {
12712  
12713
12714 size_t sizeof_options;
12715  
12716
12717 void *(*malloc_func)(size_t);
12718 void (*free_func)(void *);
12719 };
12720  
12721 #ifdef __cplusplus
12722 }
12723 #endif
12724  
12725 #endif
12726  
12727  
12728 #include <stdbool.h>
12729 #include <stddef.h>
12730 #include <stdint.h>
12731 #ifdef _MSC_VER
12732 # include <intrin.h>
12733 # include <stdlib.h>
12734
12735
12736 # pragma warning(disable : 4146)
12737
12738 # pragma warning(disable : 4018)
12739 # pragma warning(disable : 4244)
12740 # pragma warning(disable : 4267)
12741 # pragma warning(disable : 4310)
12742
12743 # pragma warning(disable : 4100)
12744 # pragma warning(disable : 4127)
12745 # pragma warning(disable : 4189)
12746 # pragma warning(disable : 4232)
12747 # pragma warning(disable : 4245)
12748 # pragma warning(disable : 4295)
12749 #endif
12750 #ifndef FREESTANDING
12751 # include <string.h>
12752 #endif
12753  
12754  
12755  
12756  
12757  
12758  
12759 #undef ARCH_X86_64
12760 #undef ARCH_X86_32
12761 #undef ARCH_ARM64
12762 #undef ARCH_ARM32
12763 #undef ARCH_RISCV
12764 #ifdef _MSC_VER
12765
12766 # if defined(_M_X64) && !defined(_M_ARM64EC)
12767 # define ARCH_X86_64
12768 # elif defined(_M_IX86)
12769 # define ARCH_X86_32
12770 # elif defined(_M_ARM64)
12771 # define ARCH_ARM64
12772 # elif defined(_M_ARM)
12773 # define ARCH_ARM32
12774 # endif
12775 #else
12776 # if defined(__x86_64__)
12777 # define ARCH_X86_64
12778 # elif defined(__i386__)
12779 # define ARCH_X86_32
12780 # elif defined(__aarch64__)
12781 # define ARCH_ARM64
12782 # elif defined(__arm__)
12783 # define ARCH_ARM32
12784 # elif defined(__riscv)
12785 # define ARCH_RISCV
12786 # endif
12787 #endif
12788  
12789  
12790  
12791  
12792  
12793  
12794 typedef uint8_t u8;
12795 typedef uint16_t u16;
12796 typedef uint32_t u32;
12797 typedef uint64_t u64;
12798 typedef int8_t s8;
12799 typedef int16_t s16;
12800 typedef int32_t s32;
12801 typedef int64_t s64;
12802  
12803  
12804 #ifdef _MSC_VER
12805 # ifdef _WIN64
12806 typedef long long ssize_t;
12807 # else
12808 typedef long ssize_t;
12809 # endif
12810 #endif
12811  
12812  
12813 typedef size_t machine_word_t;
12814  
12815  
12816 #define WORDBYTES ((int)sizeof(machine_word_t))
12817  
12818  
12819 #define WORDBITS (8 * WORDBYTES)
12820  
12821  
12822  
12823  
12824  
12825  
12826 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
12827 # define GCC_PREREQ(major, minor) \
12828 (__GNUC__ > (major) || \
12829 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
12830 # if !GCC_PREREQ(4, 9)
12831 # error "gcc versions older than 4.9 are no longer supported"
12832 # endif
12833 #else
12834 # define GCC_PREREQ(major, minor) 0
12835 #endif
12836 #ifdef __clang__
12837 # ifdef __apple_build_version__
12838 # define CLANG_PREREQ(major, minor, apple_version) \
12839 (__apple_build_version__ >= (apple_version))
12840 # else
12841 # define CLANG_PREREQ(major, minor, apple_version) \
12842 (__clang_major__ > (major) || \
12843 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
12844 # endif
12845 # if !CLANG_PREREQ(3, 9, 8000000)
12846 # error "clang versions older than 3.9 are no longer supported"
12847 # endif
12848 #else
12849 # define CLANG_PREREQ(major, minor, apple_version) 0
12850 #endif
12851 #ifdef _MSC_VER
12852 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
12853 # if !MSVC_PREREQ(1928)
12854 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
12855 # endif
12856 #else
12857 # define MSVC_PREREQ(version) 0
12858 #endif
12859  
12860  
12861 #ifndef __has_attribute
12862 # define __has_attribute(attribute) 0
12863 #endif
12864  
12865  
12866 #ifndef __has_builtin
12867 # define __has_builtin(builtin) 0
12868 #endif
12869  
12870  
12871 #ifdef _MSC_VER
12872 # define inline __inline
12873 #endif
12874  
12875  
12876 #if defined(__GNUC__) || __has_attribute(always_inline)
12877 # define forceinline inline __attribute__((always_inline))
12878 #elif defined(_MSC_VER)
12879 # define forceinline __forceinline
12880 #else
12881 # define forceinline inline
12882 #endif
12883  
12884  
12885 #if defined(__GNUC__) || __has_attribute(unused)
12886 # define MAYBE_UNUSED __attribute__((unused))
12887 #else
12888 # define MAYBE_UNUSED
12889 #endif
12890  
12891  
12892 #if defined(__GNUC__) || __has_attribute(noreturn)
12893 # define NORETURN __attribute__((noreturn))
12894 #else
12895 # define NORETURN
12896 #endif
12897  
12898  
12899 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
12900 # define likely(expr) __builtin_expect(!!(expr), 1)
12901 #else
12902 # define likely(expr) (expr)
12903 #endif
12904  
12905  
12906 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
12907 # define unlikely(expr) __builtin_expect(!!(expr), 0)
12908 #else
12909 # define unlikely(expr) (expr)
12910 #endif
12911  
12912  
12913 #undef prefetchr
12914 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
12915 # define prefetchr(addr) __builtin_prefetch((addr), 0)
12916 #elif defined(_MSC_VER)
12917 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
12918 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
12919 # elif defined(ARCH_ARM64)
12920 # define prefetchr(addr) __prefetch2((addr), 0x00 )
12921 # elif defined(ARCH_ARM32)
12922 # define prefetchr(addr) __prefetch(addr)
12923 # endif
12924 #endif
12925 #ifndef prefetchr
12926 # define prefetchr(addr)
12927 #endif
12928  
12929  
12930 #undef prefetchw
12931 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
12932 # define prefetchw(addr) __builtin_prefetch((addr), 1)
12933 #elif defined(_MSC_VER)
12934 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
12935 # define prefetchw(addr) _m_prefetchw(addr)
12936 # elif defined(ARCH_ARM64)
12937 # define prefetchw(addr) __prefetch2((addr), 0x10 )
12938 # elif defined(ARCH_ARM32)
12939 # define prefetchw(addr) __prefetchw(addr)
12940 # endif
12941 #endif
12942 #ifndef prefetchw
12943 # define prefetchw(addr)
12944 #endif
12945  
12946  
12947 #undef _aligned_attribute
12948 #if defined(__GNUC__) || __has_attribute(aligned)
12949 # define _aligned_attribute(n) __attribute__((aligned(n)))
12950 #elif defined(_MSC_VER)
12951 # define _aligned_attribute(n) __declspec(align(n))
12952 #endif
12953  
12954  
12955 #if defined(__GNUC__) || __has_attribute(target)
12956 # define _target_attribute(attrs) __attribute__((target(attrs)))
12957 #else
12958 # define _target_attribute(attrs)
12959 #endif
12960  
12961  
12962  
12963  
12964  
12965 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
12966 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
12967 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
12968 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
12969 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
12970 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
12971 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
12972  
12973  
12974  
12975  
12976  
12977  
12978 #if defined(__BYTE_ORDER__)
12979 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
12980 #elif defined(_MSC_VER)
12981 # define CPU_IS_LITTLE_ENDIAN() true
12982 #else
12983 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
12984 {
12985 union {
12986 u32 w;
12987 u8 b;
12988 } u;
12989  
12990 u.w = 1;
12991 return u.b;
12992 }
12993 #endif
12994  
12995  
12996 static forceinline u16 bswap16(u16 v)
12997 {
12998 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
12999 return __builtin_bswap16(v);
13000 #elif defined(_MSC_VER)
13001 return _byteswap_ushort(v);
13002 #else
13003 return (v << 8) | (v >> 8);
13004 #endif
13005 }
13006  
13007  
13008 static forceinline u32 bswap32(u32 v)
13009 {
13010 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
13011 return __builtin_bswap32(v);
13012 #elif defined(_MSC_VER)
13013 return _byteswap_ulong(v);
13014 #else
13015 return ((v & 0x000000FF) << 24) |
13016 ((v & 0x0000FF00) << 8) |
13017 ((v & 0x00FF0000) >> 8) |
13018 ((v & 0xFF000000) >> 24);
13019 #endif
13020 }
13021  
13022  
13023 static forceinline u64 bswap64(u64 v)
13024 {
13025 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
13026 return __builtin_bswap64(v);
13027 #elif defined(_MSC_VER)
13028 return _byteswap_uint64(v);
13029 #else
13030 return ((v & 0x00000000000000FF) << 56) |
13031 ((v & 0x000000000000FF00) << 40) |
13032 ((v & 0x0000000000FF0000) << 24) |
13033 ((v & 0x00000000FF000000) << 8) |
13034 ((v & 0x000000FF00000000) >> 8) |
13035 ((v & 0x0000FF0000000000) >> 24) |
13036 ((v & 0x00FF000000000000) >> 40) |
13037 ((v & 0xFF00000000000000) >> 56);
13038 #endif
13039 }
13040  
13041 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
13042 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
13043 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
13044 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
13045 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
13046 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
13047  
13048  
13049  
13050  
13051  
13052  
13053 #if (defined(__GNUC__) || defined(__clang__)) && \
13054 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
13055 defined(__ARM_FEATURE_UNALIGNED) || \
13056 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
13057 defined(__riscv_misaligned_fast) || \
13058 defined(__wasm__))
13059 # define UNALIGNED_ACCESS_IS_FAST 1
13060 #elif defined(_MSC_VER)
13061 # define UNALIGNED_ACCESS_IS_FAST 1
13062 #else
13063 # define UNALIGNED_ACCESS_IS_FAST 0
13064 #endif
13065  
13066  
13067  
13068 #ifdef FREESTANDING
13069 # define MEMCOPY __builtin_memcpy
13070 #else
13071 # define MEMCOPY memcpy
13072 #endif
13073  
13074  
13075  
13076 #define DEFINE_UNALIGNED_TYPE(type) \
13077 static forceinline type \
13078 load_##type##_unaligned(const void *p) \
13079 { \
13080 type v; \
13081 \
13082 MEMCOPY(&v, p, sizeof(v)); \
13083 return v; \
13084 } \
13085 \
13086 static forceinline void \
13087 store_##type##_unaligned(type v, void *p) \
13088 { \
13089 MEMCOPY(p, &v, sizeof(v)); \
13090 }
13091  
13092 DEFINE_UNALIGNED_TYPE(u16)
13093 DEFINE_UNALIGNED_TYPE(u32)
13094 DEFINE_UNALIGNED_TYPE(u64)
13095 DEFINE_UNALIGNED_TYPE(machine_word_t)
13096  
13097 #undef MEMCOPY
13098  
13099 #define load_word_unaligned load_machine_word_t_unaligned
13100 #define store_word_unaligned store_machine_word_t_unaligned
13101  
13102  
13103  
13104 static forceinline u16
13105 get_unaligned_le16(const u8 *p)
13106 {
13107 if (UNALIGNED_ACCESS_IS_FAST)
13108 return le16_bswap(load_u16_unaligned(p));
13109 else
13110 return ((u16)p[1] << 8) | p[0];
13111 }
13112  
13113 static forceinline u16
13114 get_unaligned_be16(const u8 *p)
13115 {
13116 if (UNALIGNED_ACCESS_IS_FAST)
13117 return be16_bswap(load_u16_unaligned(p));
13118 else
13119 return ((u16)p[0] << 8) | p[1];
13120 }
13121  
13122 static forceinline u32
13123 get_unaligned_le32(const u8 *p)
13124 {
13125 if (UNALIGNED_ACCESS_IS_FAST)
13126 return le32_bswap(load_u32_unaligned(p));
13127 else
13128 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
13129 ((u32)p[1] << 8) | p[0];
13130 }
13131  
13132 static forceinline u32
13133 get_unaligned_be32(const u8 *p)
13134 {
13135 if (UNALIGNED_ACCESS_IS_FAST)
13136 return be32_bswap(load_u32_unaligned(p));
13137 else
13138 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
13139 ((u32)p[2] << 8) | p[3];
13140 }
13141  
13142 static forceinline u64
13143 get_unaligned_le64(const u8 *p)
13144 {
13145 if (UNALIGNED_ACCESS_IS_FAST)
13146 return le64_bswap(load_u64_unaligned(p));
13147 else
13148 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
13149 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
13150 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
13151 ((u64)p[1] << 8) | p[0];
13152 }
13153  
13154 static forceinline machine_word_t
13155 get_unaligned_leword(const u8 *p)
13156 {
13157 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
13158 if (WORDBITS == 32)
13159 return get_unaligned_le32(p);
13160 else
13161 return get_unaligned_le64(p);
13162 }
13163  
13164  
13165  
13166 static forceinline void
13167 put_unaligned_le16(u16 v, u8 *p)
13168 {
13169 if (UNALIGNED_ACCESS_IS_FAST) {
13170 store_u16_unaligned(le16_bswap(v), p);
13171 } else {
13172 p[0] = (u8)(v >> 0);
13173 p[1] = (u8)(v >> 8);
13174 }
13175 }
13176  
13177 static forceinline void
13178 put_unaligned_be16(u16 v, u8 *p)
13179 {
13180 if (UNALIGNED_ACCESS_IS_FAST) {
13181 store_u16_unaligned(be16_bswap(v), p);
13182 } else {
13183 p[0] = (u8)(v >> 8);
13184 p[1] = (u8)(v >> 0);
13185 }
13186 }
13187  
13188 static forceinline void
13189 put_unaligned_le32(u32 v, u8 *p)
13190 {
13191 if (UNALIGNED_ACCESS_IS_FAST) {
13192 store_u32_unaligned(le32_bswap(v), p);
13193 } else {
13194 p[0] = (u8)(v >> 0);
13195 p[1] = (u8)(v >> 8);
13196 p[2] = (u8)(v >> 16);
13197 p[3] = (u8)(v >> 24);
13198 }
13199 }
13200  
13201 static forceinline void
13202 put_unaligned_be32(u32 v, u8 *p)
13203 {
13204 if (UNALIGNED_ACCESS_IS_FAST) {
13205 store_u32_unaligned(be32_bswap(v), p);
13206 } else {
13207 p[0] = (u8)(v >> 24);
13208 p[1] = (u8)(v >> 16);
13209 p[2] = (u8)(v >> 8);
13210 p[3] = (u8)(v >> 0);
13211 }
13212 }
13213  
13214 static forceinline void
13215 put_unaligned_le64(u64 v, u8 *p)
13216 {
13217 if (UNALIGNED_ACCESS_IS_FAST) {
13218 store_u64_unaligned(le64_bswap(v), p);
13219 } else {
13220 p[0] = (u8)(v >> 0);
13221 p[1] = (u8)(v >> 8);
13222 p[2] = (u8)(v >> 16);
13223 p[3] = (u8)(v >> 24);
13224 p[4] = (u8)(v >> 32);
13225 p[5] = (u8)(v >> 40);
13226 p[6] = (u8)(v >> 48);
13227 p[7] = (u8)(v >> 56);
13228 }
13229 }
13230  
13231 static forceinline void
13232 put_unaligned_leword(machine_word_t v, u8 *p)
13233 {
13234 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
13235 if (WORDBITS == 32)
13236 put_unaligned_le32(v, p);
13237 else
13238 put_unaligned_le64(v, p);
13239 }
13240  
13241  
13242  
13243  
13244  
13245  
13246  
13247 static forceinline unsigned
13248 bsr32(u32 v)
13249 {
13250 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
13251 return 31 - __builtin_clz(v);
13252 #elif defined(_MSC_VER)
13253 unsigned long i;
13254  
13255 _BitScanReverse(&i, v);
13256 return i;
13257 #else
13258 unsigned i = 0;
13259  
13260 while ((v >>= 1) != 0)
13261 i++;
13262 return i;
13263 #endif
13264 }
13265  
13266 static forceinline unsigned
13267 bsr64(u64 v)
13268 {
13269 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
13270 return 63 - __builtin_clzll(v);
13271 #elif defined(_MSC_VER) && defined(_WIN64)
13272 unsigned long i;
13273  
13274 _BitScanReverse64(&i, v);
13275 return i;
13276 #else
13277 unsigned i = 0;
13278  
13279 while ((v >>= 1) != 0)
13280 i++;
13281 return i;
13282 #endif
13283 }
13284  
13285 static forceinline unsigned
13286 bsrw(machine_word_t v)
13287 {
13288 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
13289 if (WORDBITS == 32)
13290 return bsr32(v);
13291 else
13292 return bsr64(v);
13293 }
13294  
13295  
13296  
13297 static forceinline unsigned
13298 bsf32(u32 v)
13299 {
13300 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
13301 return __builtin_ctz(v);
13302 #elif defined(_MSC_VER)
13303 unsigned long i;
13304  
13305 _BitScanForward(&i, v);
13306 return i;
13307 #else
13308 unsigned i = 0;
13309  
13310 for (; (v & 1) == 0; v >>= 1)
13311 i++;
13312 return i;
13313 #endif
13314 }
13315  
13316 static forceinline unsigned
13317 bsf64(u64 v)
13318 {
13319 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
13320 return __builtin_ctzll(v);
13321 #elif defined(_MSC_VER) && defined(_WIN64)
13322 unsigned long i;
13323  
13324 _BitScanForward64(&i, v);
13325 return i;
13326 #else
13327 unsigned i = 0;
13328  
13329 for (; (v & 1) == 0; v >>= 1)
13330 i++;
13331 return i;
13332 #endif
13333 }
13334  
13335 static forceinline unsigned
13336 bsfw(machine_word_t v)
13337 {
13338 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
13339 if (WORDBITS == 32)
13340 return bsf32(v);
13341 else
13342 return bsf64(v);
13343 }
13344  
13345  
13346 #undef rbit32
13347 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
13348 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
13349 static forceinline u32
13350 rbit32(u32 v)
13351 {
13352 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
13353 return v;
13354 }
13355 #define rbit32 rbit32
13356 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
13357 static forceinline u32
13358 rbit32(u32 v)
13359 {
13360 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
13361 return v;
13362 }
13363 #define rbit32 rbit32
13364 #endif
13365  
13366 #endif
13367  
13368  
13369 typedef void *(*malloc_func_t)(size_t);
13370 typedef void (*free_func_t)(void *);
13371  
13372 extern malloc_func_t libdeflate_default_malloc_func;
13373 extern free_func_t libdeflate_default_free_func;
13374  
13375 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
13376 size_t alignment, size_t size);
13377 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
13378  
13379 #ifdef FREESTANDING
13380  
13381 void *memset(void *s, int c, size_t n);
13382 #define memset(s, c, n) __builtin_memset((s), (c), (n))
13383  
13384 void *memcpy(void *dest, const void *src, size_t n);
13385 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
13386  
13387 void *memmove(void *dest, const void *src, size_t n);
13388 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
13389  
13390 int memcmp(const void *s1, const void *s2, size_t n);
13391 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
13392  
13393 #undef LIBDEFLATE_ENABLE_ASSERTIONS
13394 #else
13395 # include <string.h>
13396
13397 # ifdef __clang_analyzer__
13398 # define LIBDEFLATE_ENABLE_ASSERTIONS
13399 # endif
13400 #endif
13401  
13402  
13403 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
13404 NORETURN void
13405 libdeflate_assertion_failed(const char *expr, const char *file, int line);
13406 #define ASSERT(expr) { if (unlikely(!(expr))) \
13407 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
13408 #else
13409 #define ASSERT(expr) (void)(expr)
13410 #endif
13411  
13412 #define CONCAT_IMPL(a, b) a##b
13413 #define CONCAT(a, b) CONCAT_IMPL(a, b)
13414 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
13415  
13416 #endif
13417  
13418  
13419  
13420  
13421 struct libdeflate_compressor;
13422  
13423 unsigned int libdeflate_get_compression_level(struct libdeflate_compressor *c);
13424  
13425 #endif
13426  
13427 /* #include "deflate_constants.h" */
13428  
13429  
13430 #ifndef LIB_DEFLATE_CONSTANTS_H
13431 #define LIB_DEFLATE_CONSTANTS_H
13432  
13433  
13434 #define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
13435 #define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
13436 #define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
13437  
13438  
13439 #define DEFLATE_MIN_MATCH_LEN 3
13440 #define DEFLATE_MAX_MATCH_LEN 258
13441  
13442  
13443 #define DEFLATE_MAX_MATCH_OFFSET 32768
13444  
13445  
13446 #define DEFLATE_WINDOW_ORDER 15
13447  
13448  
13449 #define DEFLATE_NUM_PRECODE_SYMS 19
13450 #define DEFLATE_NUM_LITLEN_SYMS 288
13451 #define DEFLATE_NUM_OFFSET_SYMS 32
13452  
13453  
13454 #define DEFLATE_MAX_NUM_SYMS 288
13455  
13456  
13457 #define DEFLATE_NUM_LITERALS 256
13458 #define DEFLATE_END_OF_BLOCK 256
13459 #define DEFLATE_FIRST_LEN_SYM 257
13460  
13461  
13462 #define DEFLATE_MAX_PRE_CODEWORD_LEN 7
13463 #define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
13464 #define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
13465  
13466  
13467 #define DEFLATE_MAX_CODEWORD_LEN 15
13468  
13469  
13470 #define DEFLATE_MAX_LENS_OVERRUN 137
13471  
13472  
13473 #define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
13474 #define DEFLATE_MAX_EXTRA_OFFSET_BITS 13
13475  
13476 #endif
13477  
13478  
13479  
13480  
13481  
13482  
13483  
13484 #define SUPPORT_NEAR_OPTIMAL_PARSING 1
13485  
13486  
13487 #define MIN_BLOCK_LENGTH 5000
13488  
13489  
13490 #define SOFT_MAX_BLOCK_LENGTH 300000
13491  
13492  
13493 #define SEQ_STORE_LENGTH 50000
13494  
13495  
13496 #define FAST_SOFT_MAX_BLOCK_LENGTH 65535
13497  
13498  
13499 #define FAST_SEQ_STORE_LENGTH 8192
13500  
13501  
13502 #define MAX_LITLEN_CODEWORD_LEN 14
13503 #define MAX_OFFSET_CODEWORD_LEN DEFLATE_MAX_OFFSET_CODEWORD_LEN
13504 #define MAX_PRE_CODEWORD_LEN DEFLATE_MAX_PRE_CODEWORD_LEN
13505  
13506 #if SUPPORT_NEAR_OPTIMAL_PARSING
13507  
13508  
13509  
13510  
13511 #define BIT_COST 16
13512  
13513  
13514 #define LITERAL_NOSTAT_BITS 13
13515 #define LENGTH_NOSTAT_BITS 13
13516 #define OFFSET_NOSTAT_BITS 10
13517  
13518  
13519 #define MATCH_CACHE_LENGTH (SOFT_MAX_BLOCK_LENGTH * 5)
13520  
13521 #endif
13522  
13523  
13524  
13525  
13526 #define MATCHFINDER_WINDOW_ORDER DEFLATE_WINDOW_ORDER
13527 /* #include "hc_matchfinder.h" */
13528  
13529  
13530 #ifndef LIB_HC_MATCHFINDER_H
13531 #define LIB_HC_MATCHFINDER_H
13532  
13533 /* #include "matchfinder_common.h" */
13534  
13535  
13536 #ifndef LIB_MATCHFINDER_COMMON_H
13537 #define LIB_MATCHFINDER_COMMON_H
13538  
13539 /* #include "lib_common.h" */
13540  
13541  
13542 #ifndef LIB_LIB_COMMON_H
13543 #define LIB_LIB_COMMON_H
13544  
13545 #ifdef LIBDEFLATE_H
13546
13547 # error "lib_common.h must always be included before libdeflate.h"
13548 #endif
13549  
13550 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
13551 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
13552 #elif defined(__GNUC__)
13553 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
13554 #else
13555 # define LIBDEFLATE_EXPORT_SYM
13556 #endif
13557  
13558  
13559 #if defined(__GNUC__) && defined(__i386__)
13560 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
13561 #else
13562 # define LIBDEFLATE_ALIGN_STACK
13563 #endif
13564  
13565 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
13566  
13567 /* #include "../common_defs.h" */
13568  
13569  
13570 #ifndef COMMON_DEFS_H
13571 #define COMMON_DEFS_H
13572  
13573 /* #include "libdeflate.h" */
13574  
13575  
13576 #ifndef LIBDEFLATE_H
13577 #define LIBDEFLATE_H
13578  
13579 #include <stddef.h>
13580 #include <stdint.h>
13581  
13582 #ifdef __cplusplus
13583 extern "C" {
13584 #endif
13585  
13586 #define LIBDEFLATE_VERSION_MAJOR 1
13587 #define LIBDEFLATE_VERSION_MINOR 26
13588 #define LIBDEFLATE_VERSION_STRING "1.26"
13589  
13590  
13591 #ifndef LIBDEFLATEAPI
13592 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
13593 # define LIBDEFLATEAPI __declspec(dllimport)
13594 # else
13595 # define LIBDEFLATEAPI
13596 # endif
13597 #endif
13598  
13599  
13600  
13601  
13602  
13603 struct libdeflate_compressor;
13604 struct libdeflate_options;
13605  
13606  
13607 LIBDEFLATEAPI struct libdeflate_compressor *
13608 libdeflate_alloc_compressor(int compression_level);
13609  
13610  
13611 LIBDEFLATEAPI struct libdeflate_compressor *
13612 libdeflate_alloc_compressor_ex(int compression_level,
13613 const struct libdeflate_options *options);
13614  
13615  
13616 LIBDEFLATEAPI size_t
13617 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
13618 const void *in, size_t in_nbytes,
13619 void *out, size_t out_nbytes_avail);
13620  
13621  
13622 LIBDEFLATEAPI size_t
13623 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
13624 size_t in_nbytes);
13625  
13626  
13627 LIBDEFLATEAPI size_t
13628 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
13629 const void *in, size_t in_nbytes,
13630 void *out, size_t out_nbytes_avail);
13631  
13632  
13633 LIBDEFLATEAPI size_t
13634 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
13635 size_t in_nbytes);
13636  
13637  
13638 LIBDEFLATEAPI size_t
13639 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
13640 const void *in, size_t in_nbytes,
13641 void *out, size_t out_nbytes_avail);
13642  
13643  
13644 LIBDEFLATEAPI size_t
13645 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
13646 size_t in_nbytes);
13647  
13648  
13649 LIBDEFLATEAPI void
13650 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
13651  
13652  
13653  
13654  
13655  
13656 struct libdeflate_decompressor;
13657 struct libdeflate_options;
13658  
13659  
13660 LIBDEFLATEAPI struct libdeflate_decompressor *
13661 libdeflate_alloc_decompressor(void);
13662  
13663  
13664 LIBDEFLATEAPI struct libdeflate_decompressor *
13665 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
13666  
13667  
13668 enum libdeflate_result {
13669
13670 LIBDEFLATE_SUCCESS = 0,
13671  
13672
13673 LIBDEFLATE_BAD_DATA = 1,
13674  
13675
13676 LIBDEFLATE_SHORT_OUTPUT = 2,
13677  
13678
13679 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
13680 };
13681  
13682  
13683 LIBDEFLATEAPI enum libdeflate_result
13684 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
13685 const void *in, size_t in_nbytes,
13686 void *out, size_t out_nbytes_avail,
13687 size_t *actual_out_nbytes_ret);
13688  
13689  
13690 LIBDEFLATEAPI enum libdeflate_result
13691 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
13692 const void *in, size_t in_nbytes,
13693 void *out, size_t out_nbytes_avail,
13694 size_t *actual_in_nbytes_ret,
13695 size_t *actual_out_nbytes_ret);
13696  
13697  
13698 LIBDEFLATEAPI enum libdeflate_result
13699 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
13700 const void *in, size_t in_nbytes,
13701 void *out, size_t out_nbytes_avail,
13702 size_t *actual_out_nbytes_ret);
13703  
13704  
13705 LIBDEFLATEAPI enum libdeflate_result
13706 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
13707 const void *in, size_t in_nbytes,
13708 void *out, size_t out_nbytes_avail,
13709 size_t *actual_in_nbytes_ret,
13710 size_t *actual_out_nbytes_ret);
13711  
13712  
13713 LIBDEFLATEAPI enum libdeflate_result
13714 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
13715 const void *in, size_t in_nbytes,
13716 void *out, size_t out_nbytes_avail,
13717 size_t *actual_out_nbytes_ret);
13718  
13719  
13720 LIBDEFLATEAPI enum libdeflate_result
13721 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
13722 const void *in, size_t in_nbytes,
13723 void *out, size_t out_nbytes_avail,
13724 size_t *actual_in_nbytes_ret,
13725 size_t *actual_out_nbytes_ret);
13726  
13727  
13728 LIBDEFLATEAPI void
13729 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
13730  
13731  
13732  
13733  
13734  
13735  
13736 LIBDEFLATEAPI uint32_t
13737 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
13738  
13739  
13740  
13741 LIBDEFLATEAPI uint32_t
13742 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
13743  
13744  
13745  
13746  
13747  
13748  
13749 LIBDEFLATEAPI void
13750 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
13751 void (*free_func)(void *));
13752  
13753  
13754 struct libdeflate_options {
13755  
13756
13757 size_t sizeof_options;
13758  
13759
13760 void *(*malloc_func)(size_t);
13761 void (*free_func)(void *);
13762 };
13763  
13764 #ifdef __cplusplus
13765 }
13766 #endif
13767  
13768 #endif
13769  
13770  
13771 #include <stdbool.h>
13772 #include <stddef.h>
13773 #include <stdint.h>
13774 #ifdef _MSC_VER
13775 # include <intrin.h>
13776 # include <stdlib.h>
13777
13778
13779 # pragma warning(disable : 4146)
13780
13781 # pragma warning(disable : 4018)
13782 # pragma warning(disable : 4244)
13783 # pragma warning(disable : 4267)
13784 # pragma warning(disable : 4310)
13785
13786 # pragma warning(disable : 4100)
13787 # pragma warning(disable : 4127)
13788 # pragma warning(disable : 4189)
13789 # pragma warning(disable : 4232)
13790 # pragma warning(disable : 4245)
13791 # pragma warning(disable : 4295)
13792 #endif
13793 #ifndef FREESTANDING
13794 # include <string.h>
13795 #endif
13796  
13797  
13798  
13799  
13800  
13801  
13802 #undef ARCH_X86_64
13803 #undef ARCH_X86_32
13804 #undef ARCH_ARM64
13805 #undef ARCH_ARM32
13806 #undef ARCH_RISCV
13807 #ifdef _MSC_VER
13808
13809 # if defined(_M_X64) && !defined(_M_ARM64EC)
13810 # define ARCH_X86_64
13811 # elif defined(_M_IX86)
13812 # define ARCH_X86_32
13813 # elif defined(_M_ARM64)
13814 # define ARCH_ARM64
13815 # elif defined(_M_ARM)
13816 # define ARCH_ARM32
13817 # endif
13818 #else
13819 # if defined(__x86_64__)
13820 # define ARCH_X86_64
13821 # elif defined(__i386__)
13822 # define ARCH_X86_32
13823 # elif defined(__aarch64__)
13824 # define ARCH_ARM64
13825 # elif defined(__arm__)
13826 # define ARCH_ARM32
13827 # elif defined(__riscv)
13828 # define ARCH_RISCV
13829 # endif
13830 #endif
13831  
13832  
13833  
13834  
13835  
13836  
13837 typedef uint8_t u8;
13838 typedef uint16_t u16;
13839 typedef uint32_t u32;
13840 typedef uint64_t u64;
13841 typedef int8_t s8;
13842 typedef int16_t s16;
13843 typedef int32_t s32;
13844 typedef int64_t s64;
13845  
13846  
13847 #ifdef _MSC_VER
13848 # ifdef _WIN64
13849 typedef long long ssize_t;
13850 # else
13851 typedef long ssize_t;
13852 # endif
13853 #endif
13854  
13855  
13856 typedef size_t machine_word_t;
13857  
13858  
13859 #define WORDBYTES ((int)sizeof(machine_word_t))
13860  
13861  
13862 #define WORDBITS (8 * WORDBYTES)
13863  
13864  
13865  
13866  
13867  
13868  
13869 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
13870 # define GCC_PREREQ(major, minor) \
13871 (__GNUC__ > (major) || \
13872 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
13873 # if !GCC_PREREQ(4, 9)
13874 # error "gcc versions older than 4.9 are no longer supported"
13875 # endif
13876 #else
13877 # define GCC_PREREQ(major, minor) 0
13878 #endif
13879 #ifdef __clang__
13880 # ifdef __apple_build_version__
13881 # define CLANG_PREREQ(major, minor, apple_version) \
13882 (__apple_build_version__ >= (apple_version))
13883 # else
13884 # define CLANG_PREREQ(major, minor, apple_version) \
13885 (__clang_major__ > (major) || \
13886 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
13887 # endif
13888 # if !CLANG_PREREQ(3, 9, 8000000)
13889 # error "clang versions older than 3.9 are no longer supported"
13890 # endif
13891 #else
13892 # define CLANG_PREREQ(major, minor, apple_version) 0
13893 #endif
13894 #ifdef _MSC_VER
13895 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
13896 # if !MSVC_PREREQ(1928)
13897 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
13898 # endif
13899 #else
13900 # define MSVC_PREREQ(version) 0
13901 #endif
13902  
13903  
13904 #ifndef __has_attribute
13905 # define __has_attribute(attribute) 0
13906 #endif
13907  
13908  
13909 #ifndef __has_builtin
13910 # define __has_builtin(builtin) 0
13911 #endif
13912  
13913  
13914 #ifdef _MSC_VER
13915 # define inline __inline
13916 #endif
13917  
13918  
13919 #if defined(__GNUC__) || __has_attribute(always_inline)
13920 # define forceinline inline __attribute__((always_inline))
13921 #elif defined(_MSC_VER)
13922 # define forceinline __forceinline
13923 #else
13924 # define forceinline inline
13925 #endif
13926  
13927  
13928 #if defined(__GNUC__) || __has_attribute(unused)
13929 # define MAYBE_UNUSED __attribute__((unused))
13930 #else
13931 # define MAYBE_UNUSED
13932 #endif
13933  
13934  
13935 #if defined(__GNUC__) || __has_attribute(noreturn)
13936 # define NORETURN __attribute__((noreturn))
13937 #else
13938 # define NORETURN
13939 #endif
13940  
13941  
13942 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
13943 # define likely(expr) __builtin_expect(!!(expr), 1)
13944 #else
13945 # define likely(expr) (expr)
13946 #endif
13947  
13948  
13949 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
13950 # define unlikely(expr) __builtin_expect(!!(expr), 0)
13951 #else
13952 # define unlikely(expr) (expr)
13953 #endif
13954  
13955  
13956 #undef prefetchr
13957 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
13958 # define prefetchr(addr) __builtin_prefetch((addr), 0)
13959 #elif defined(_MSC_VER)
13960 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
13961 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
13962 # elif defined(ARCH_ARM64)
13963 # define prefetchr(addr) __prefetch2((addr), 0x00 )
13964 # elif defined(ARCH_ARM32)
13965 # define prefetchr(addr) __prefetch(addr)
13966 # endif
13967 #endif
13968 #ifndef prefetchr
13969 # define prefetchr(addr)
13970 #endif
13971  
13972  
13973 #undef prefetchw
13974 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
13975 # define prefetchw(addr) __builtin_prefetch((addr), 1)
13976 #elif defined(_MSC_VER)
13977 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
13978 # define prefetchw(addr) _m_prefetchw(addr)
13979 # elif defined(ARCH_ARM64)
13980 # define prefetchw(addr) __prefetch2((addr), 0x10 )
13981 # elif defined(ARCH_ARM32)
13982 # define prefetchw(addr) __prefetchw(addr)
13983 # endif
13984 #endif
13985 #ifndef prefetchw
13986 # define prefetchw(addr)
13987 #endif
13988  
13989  
13990 #undef _aligned_attribute
13991 #if defined(__GNUC__) || __has_attribute(aligned)
13992 # define _aligned_attribute(n) __attribute__((aligned(n)))
13993 #elif defined(_MSC_VER)
13994 # define _aligned_attribute(n) __declspec(align(n))
13995 #endif
13996  
13997  
13998 #if defined(__GNUC__) || __has_attribute(target)
13999 # define _target_attribute(attrs) __attribute__((target(attrs)))
14000 #else
14001 # define _target_attribute(attrs)
14002 #endif
14003  
14004  
14005  
14006  
14007  
14008 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
14009 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
14010 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
14011 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
14012 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
14013 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
14014 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
14015  
14016  
14017  
14018  
14019  
14020  
14021 #if defined(__BYTE_ORDER__)
14022 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
14023 #elif defined(_MSC_VER)
14024 # define CPU_IS_LITTLE_ENDIAN() true
14025 #else
14026 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
14027 {
14028 union {
14029 u32 w;
14030 u8 b;
14031 } u;
14032  
14033 u.w = 1;
14034 return u.b;
14035 }
14036 #endif
14037  
14038  
14039 static forceinline u16 bswap16(u16 v)
14040 {
14041 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
14042 return __builtin_bswap16(v);
14043 #elif defined(_MSC_VER)
14044 return _byteswap_ushort(v);
14045 #else
14046 return (v << 8) | (v >> 8);
14047 #endif
14048 }
14049  
14050  
14051 static forceinline u32 bswap32(u32 v)
14052 {
14053 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
14054 return __builtin_bswap32(v);
14055 #elif defined(_MSC_VER)
14056 return _byteswap_ulong(v);
14057 #else
14058 return ((v & 0x000000FF) << 24) |
14059 ((v & 0x0000FF00) << 8) |
14060 ((v & 0x00FF0000) >> 8) |
14061 ((v & 0xFF000000) >> 24);
14062 #endif
14063 }
14064  
14065  
14066 static forceinline u64 bswap64(u64 v)
14067 {
14068 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
14069 return __builtin_bswap64(v);
14070 #elif defined(_MSC_VER)
14071 return _byteswap_uint64(v);
14072 #else
14073 return ((v & 0x00000000000000FF) << 56) |
14074 ((v & 0x000000000000FF00) << 40) |
14075 ((v & 0x0000000000FF0000) << 24) |
14076 ((v & 0x00000000FF000000) << 8) |
14077 ((v & 0x000000FF00000000) >> 8) |
14078 ((v & 0x0000FF0000000000) >> 24) |
14079 ((v & 0x00FF000000000000) >> 40) |
14080 ((v & 0xFF00000000000000) >> 56);
14081 #endif
14082 }
14083  
14084 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
14085 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
14086 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
14087 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
14088 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
14089 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
14090  
14091  
14092  
14093  
14094  
14095  
14096 #if (defined(__GNUC__) || defined(__clang__)) && \
14097 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
14098 defined(__ARM_FEATURE_UNALIGNED) || \
14099 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
14100 defined(__riscv_misaligned_fast) || \
14101 defined(__wasm__))
14102 # define UNALIGNED_ACCESS_IS_FAST 1
14103 #elif defined(_MSC_VER)
14104 # define UNALIGNED_ACCESS_IS_FAST 1
14105 #else
14106 # define UNALIGNED_ACCESS_IS_FAST 0
14107 #endif
14108  
14109  
14110  
14111 #ifdef FREESTANDING
14112 # define MEMCOPY __builtin_memcpy
14113 #else
14114 # define MEMCOPY memcpy
14115 #endif
14116  
14117  
14118  
14119 #define DEFINE_UNALIGNED_TYPE(type) \
14120 static forceinline type \
14121 load_##type##_unaligned(const void *p) \
14122 { \
14123 type v; \
14124 \
14125 MEMCOPY(&v, p, sizeof(v)); \
14126 return v; \
14127 } \
14128 \
14129 static forceinline void \
14130 store_##type##_unaligned(type v, void *p) \
14131 { \
14132 MEMCOPY(p, &v, sizeof(v)); \
14133 }
14134  
14135 DEFINE_UNALIGNED_TYPE(u16)
14136 DEFINE_UNALIGNED_TYPE(u32)
14137 DEFINE_UNALIGNED_TYPE(u64)
14138 DEFINE_UNALIGNED_TYPE(machine_word_t)
14139  
14140 #undef MEMCOPY
14141  
14142 #define load_word_unaligned load_machine_word_t_unaligned
14143 #define store_word_unaligned store_machine_word_t_unaligned
14144  
14145  
14146  
14147 static forceinline u16
14148 get_unaligned_le16(const u8 *p)
14149 {
14150 if (UNALIGNED_ACCESS_IS_FAST)
14151 return le16_bswap(load_u16_unaligned(p));
14152 else
14153 return ((u16)p[1] << 8) | p[0];
14154 }
14155  
14156 static forceinline u16
14157 get_unaligned_be16(const u8 *p)
14158 {
14159 if (UNALIGNED_ACCESS_IS_FAST)
14160 return be16_bswap(load_u16_unaligned(p));
14161 else
14162 return ((u16)p[0] << 8) | p[1];
14163 }
14164  
14165 static forceinline u32
14166 get_unaligned_le32(const u8 *p)
14167 {
14168 if (UNALIGNED_ACCESS_IS_FAST)
14169 return le32_bswap(load_u32_unaligned(p));
14170 else
14171 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
14172 ((u32)p[1] << 8) | p[0];
14173 }
14174  
14175 static forceinline u32
14176 get_unaligned_be32(const u8 *p)
14177 {
14178 if (UNALIGNED_ACCESS_IS_FAST)
14179 return be32_bswap(load_u32_unaligned(p));
14180 else
14181 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
14182 ((u32)p[2] << 8) | p[3];
14183 }
14184  
14185 static forceinline u64
14186 get_unaligned_le64(const u8 *p)
14187 {
14188 if (UNALIGNED_ACCESS_IS_FAST)
14189 return le64_bswap(load_u64_unaligned(p));
14190 else
14191 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
14192 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
14193 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
14194 ((u64)p[1] << 8) | p[0];
14195 }
14196  
14197 static forceinline machine_word_t
14198 get_unaligned_leword(const u8 *p)
14199 {
14200 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
14201 if (WORDBITS == 32)
14202 return get_unaligned_le32(p);
14203 else
14204 return get_unaligned_le64(p);
14205 }
14206  
14207  
14208  
14209 static forceinline void
14210 put_unaligned_le16(u16 v, u8 *p)
14211 {
14212 if (UNALIGNED_ACCESS_IS_FAST) {
14213 store_u16_unaligned(le16_bswap(v), p);
14214 } else {
14215 p[0] = (u8)(v >> 0);
14216 p[1] = (u8)(v >> 8);
14217 }
14218 }
14219  
14220 static forceinline void
14221 put_unaligned_be16(u16 v, u8 *p)
14222 {
14223 if (UNALIGNED_ACCESS_IS_FAST) {
14224 store_u16_unaligned(be16_bswap(v), p);
14225 } else {
14226 p[0] = (u8)(v >> 8);
14227 p[1] = (u8)(v >> 0);
14228 }
14229 }
14230  
14231 static forceinline void
14232 put_unaligned_le32(u32 v, u8 *p)
14233 {
14234 if (UNALIGNED_ACCESS_IS_FAST) {
14235 store_u32_unaligned(le32_bswap(v), p);
14236 } else {
14237 p[0] = (u8)(v >> 0);
14238 p[1] = (u8)(v >> 8);
14239 p[2] = (u8)(v >> 16);
14240 p[3] = (u8)(v >> 24);
14241 }
14242 }
14243  
14244 static forceinline void
14245 put_unaligned_be32(u32 v, u8 *p)
14246 {
14247 if (UNALIGNED_ACCESS_IS_FAST) {
14248 store_u32_unaligned(be32_bswap(v), p);
14249 } else {
14250 p[0] = (u8)(v >> 24);
14251 p[1] = (u8)(v >> 16);
14252 p[2] = (u8)(v >> 8);
14253 p[3] = (u8)(v >> 0);
14254 }
14255 }
14256  
14257 static forceinline void
14258 put_unaligned_le64(u64 v, u8 *p)
14259 {
14260 if (UNALIGNED_ACCESS_IS_FAST) {
14261 store_u64_unaligned(le64_bswap(v), p);
14262 } else {
14263 p[0] = (u8)(v >> 0);
14264 p[1] = (u8)(v >> 8);
14265 p[2] = (u8)(v >> 16);
14266 p[3] = (u8)(v >> 24);
14267 p[4] = (u8)(v >> 32);
14268 p[5] = (u8)(v >> 40);
14269 p[6] = (u8)(v >> 48);
14270 p[7] = (u8)(v >> 56);
14271 }
14272 }
14273  
14274 static forceinline void
14275 put_unaligned_leword(machine_word_t v, u8 *p)
14276 {
14277 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
14278 if (WORDBITS == 32)
14279 put_unaligned_le32(v, p);
14280 else
14281 put_unaligned_le64(v, p);
14282 }
14283  
14284  
14285  
14286  
14287  
14288  
14289  
14290 static forceinline unsigned
14291 bsr32(u32 v)
14292 {
14293 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
14294 return 31 - __builtin_clz(v);
14295 #elif defined(_MSC_VER)
14296 unsigned long i;
14297  
14298 _BitScanReverse(&i, v);
14299 return i;
14300 #else
14301 unsigned i = 0;
14302  
14303 while ((v >>= 1) != 0)
14304 i++;
14305 return i;
14306 #endif
14307 }
14308  
14309 static forceinline unsigned
14310 bsr64(u64 v)
14311 {
14312 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
14313 return 63 - __builtin_clzll(v);
14314 #elif defined(_MSC_VER) && defined(_WIN64)
14315 unsigned long i;
14316  
14317 _BitScanReverse64(&i, v);
14318 return i;
14319 #else
14320 unsigned i = 0;
14321  
14322 while ((v >>= 1) != 0)
14323 i++;
14324 return i;
14325 #endif
14326 }
14327  
14328 static forceinline unsigned
14329 bsrw(machine_word_t v)
14330 {
14331 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
14332 if (WORDBITS == 32)
14333 return bsr32(v);
14334 else
14335 return bsr64(v);
14336 }
14337  
14338  
14339  
14340 static forceinline unsigned
14341 bsf32(u32 v)
14342 {
14343 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
14344 return __builtin_ctz(v);
14345 #elif defined(_MSC_VER)
14346 unsigned long i;
14347  
14348 _BitScanForward(&i, v);
14349 return i;
14350 #else
14351 unsigned i = 0;
14352  
14353 for (; (v & 1) == 0; v >>= 1)
14354 i++;
14355 return i;
14356 #endif
14357 }
14358  
14359 static forceinline unsigned
14360 bsf64(u64 v)
14361 {
14362 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
14363 return __builtin_ctzll(v);
14364 #elif defined(_MSC_VER) && defined(_WIN64)
14365 unsigned long i;
14366  
14367 _BitScanForward64(&i, v);
14368 return i;
14369 #else
14370 unsigned i = 0;
14371  
14372 for (; (v & 1) == 0; v >>= 1)
14373 i++;
14374 return i;
14375 #endif
14376 }
14377  
14378 static forceinline unsigned
14379 bsfw(machine_word_t v)
14380 {
14381 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
14382 if (WORDBITS == 32)
14383 return bsf32(v);
14384 else
14385 return bsf64(v);
14386 }
14387  
14388  
14389 #undef rbit32
14390 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
14391 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
14392 static forceinline u32
14393 rbit32(u32 v)
14394 {
14395 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
14396 return v;
14397 }
14398 #define rbit32 rbit32
14399 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
14400 static forceinline u32
14401 rbit32(u32 v)
14402 {
14403 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
14404 return v;
14405 }
14406 #define rbit32 rbit32
14407 #endif
14408  
14409 #endif
14410  
14411  
14412 typedef void *(*malloc_func_t)(size_t);
14413 typedef void (*free_func_t)(void *);
14414  
14415 extern malloc_func_t libdeflate_default_malloc_func;
14416 extern free_func_t libdeflate_default_free_func;
14417  
14418 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
14419 size_t alignment, size_t size);
14420 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
14421  
14422 #ifdef FREESTANDING
14423  
14424 void *memset(void *s, int c, size_t n);
14425 #define memset(s, c, n) __builtin_memset((s), (c), (n))
14426  
14427 void *memcpy(void *dest, const void *src, size_t n);
14428 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
14429  
14430 void *memmove(void *dest, const void *src, size_t n);
14431 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
14432  
14433 int memcmp(const void *s1, const void *s2, size_t n);
14434 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
14435  
14436 #undef LIBDEFLATE_ENABLE_ASSERTIONS
14437 #else
14438 # include <string.h>
14439
14440 # ifdef __clang_analyzer__
14441 # define LIBDEFLATE_ENABLE_ASSERTIONS
14442 # endif
14443 #endif
14444  
14445  
14446 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
14447 NORETURN void
14448 libdeflate_assertion_failed(const char *expr, const char *file, int line);
14449 #define ASSERT(expr) { if (unlikely(!(expr))) \
14450 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
14451 #else
14452 #define ASSERT(expr) (void)(expr)
14453 #endif
14454  
14455 #define CONCAT_IMPL(a, b) a##b
14456 #define CONCAT(a, b) CONCAT_IMPL(a, b)
14457 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
14458  
14459 #endif
14460  
14461  
14462 #ifndef MATCHFINDER_WINDOW_ORDER
14463 # error "MATCHFINDER_WINDOW_ORDER must be defined!"
14464 #endif
14465  
14466  
14467 static forceinline u32
14468 loaded_u32_to_u24(u32 v)
14469 {
14470 if (CPU_IS_LITTLE_ENDIAN())
14471 126 return v & 0xFFFFFF;
14472 else
14473 return v >> 8;
14474 }
14475  
14476  
14477 static forceinline u32
14478 load_u24_unaligned(const u8 *p)
14479 {
14480 #if UNALIGNED_ACCESS_IS_FAST
14481 126 return loaded_u32_to_u24(load_u32_unaligned(p));
14482 #else
14483 if (CPU_IS_LITTLE_ENDIAN())
14484 return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
14485 else
14486 return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
14487 #endif
14488 }
14489  
14490 #define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER)
14491  
14492 typedef s16 mf_pos_t;
14493  
14494 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
14495  
14496  
14497 #define MATCHFINDER_MEM_ALIGNMENT 32
14498  
14499  
14500 #define MATCHFINDER_SIZE_ALIGNMENT 1024
14501  
14502 #undef matchfinder_init
14503 #undef matchfinder_rebase
14504 #ifdef _aligned_attribute
14505 # define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
14506 # if defined(ARCH_ARM32) || defined(ARCH_ARM64)
14507 /* # include "arm/matchfinder_impl.h" */
14508  
14509  
14510 #ifndef LIB_ARM_MATCHFINDER_IMPL_H
14511 #define LIB_ARM_MATCHFINDER_IMPL_H
14512  
14513 /* #include "arm-cpu_features.h" */
14514  
14515  
14516 #ifndef LIB_ARM_CPU_FEATURES_H
14517 #define LIB_ARM_CPU_FEATURES_H
14518  
14519 /* #include "lib_common.h" */
14520  
14521  
14522 #ifndef LIB_LIB_COMMON_H
14523 #define LIB_LIB_COMMON_H
14524  
14525 #ifdef LIBDEFLATE_H
14526
14527 # error "lib_common.h must always be included before libdeflate.h"
14528 #endif
14529  
14530 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
14531 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
14532 #elif defined(__GNUC__)
14533 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
14534 #else
14535 # define LIBDEFLATE_EXPORT_SYM
14536 #endif
14537  
14538  
14539 #if defined(__GNUC__) && defined(__i386__)
14540 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
14541 #else
14542 # define LIBDEFLATE_ALIGN_STACK
14543 #endif
14544  
14545 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
14546  
14547 /* #include "../common_defs.h" */
14548  
14549  
14550 #ifndef COMMON_DEFS_H
14551 #define COMMON_DEFS_H
14552  
14553 /* #include "libdeflate.h" */
14554  
14555  
14556 #ifndef LIBDEFLATE_H
14557 #define LIBDEFLATE_H
14558  
14559 #include <stddef.h>
14560 #include <stdint.h>
14561  
14562 #ifdef __cplusplus
14563 extern "C" {
14564 #endif
14565  
14566 #define LIBDEFLATE_VERSION_MAJOR 1
14567 #define LIBDEFLATE_VERSION_MINOR 26
14568 #define LIBDEFLATE_VERSION_STRING "1.26"
14569  
14570  
14571 #ifndef LIBDEFLATEAPI
14572 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
14573 # define LIBDEFLATEAPI __declspec(dllimport)
14574 # else
14575 # define LIBDEFLATEAPI
14576 # endif
14577 #endif
14578  
14579  
14580  
14581  
14582  
14583 struct libdeflate_compressor;
14584 struct libdeflate_options;
14585  
14586  
14587 LIBDEFLATEAPI struct libdeflate_compressor *
14588 libdeflate_alloc_compressor(int compression_level);
14589  
14590  
14591 LIBDEFLATEAPI struct libdeflate_compressor *
14592 libdeflate_alloc_compressor_ex(int compression_level,
14593 const struct libdeflate_options *options);
14594  
14595  
14596 LIBDEFLATEAPI size_t
14597 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
14598 const void *in, size_t in_nbytes,
14599 void *out, size_t out_nbytes_avail);
14600  
14601  
14602 LIBDEFLATEAPI size_t
14603 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
14604 size_t in_nbytes);
14605  
14606  
14607 LIBDEFLATEAPI size_t
14608 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
14609 const void *in, size_t in_nbytes,
14610 void *out, size_t out_nbytes_avail);
14611  
14612  
14613 LIBDEFLATEAPI size_t
14614 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
14615 size_t in_nbytes);
14616  
14617  
14618 LIBDEFLATEAPI size_t
14619 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
14620 const void *in, size_t in_nbytes,
14621 void *out, size_t out_nbytes_avail);
14622  
14623  
14624 LIBDEFLATEAPI size_t
14625 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
14626 size_t in_nbytes);
14627  
14628  
14629 LIBDEFLATEAPI void
14630 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
14631  
14632  
14633  
14634  
14635  
14636 struct libdeflate_decompressor;
14637 struct libdeflate_options;
14638  
14639  
14640 LIBDEFLATEAPI struct libdeflate_decompressor *
14641 libdeflate_alloc_decompressor(void);
14642  
14643  
14644 LIBDEFLATEAPI struct libdeflate_decompressor *
14645 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
14646  
14647  
14648 enum libdeflate_result {
14649
14650 LIBDEFLATE_SUCCESS = 0,
14651  
14652
14653 LIBDEFLATE_BAD_DATA = 1,
14654  
14655
14656 LIBDEFLATE_SHORT_OUTPUT = 2,
14657  
14658
14659 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
14660 };
14661  
14662  
14663 LIBDEFLATEAPI enum libdeflate_result
14664 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
14665 const void *in, size_t in_nbytes,
14666 void *out, size_t out_nbytes_avail,
14667 size_t *actual_out_nbytes_ret);
14668  
14669  
14670 LIBDEFLATEAPI enum libdeflate_result
14671 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
14672 const void *in, size_t in_nbytes,
14673 void *out, size_t out_nbytes_avail,
14674 size_t *actual_in_nbytes_ret,
14675 size_t *actual_out_nbytes_ret);
14676  
14677  
14678 LIBDEFLATEAPI enum libdeflate_result
14679 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
14680 const void *in, size_t in_nbytes,
14681 void *out, size_t out_nbytes_avail,
14682 size_t *actual_out_nbytes_ret);
14683  
14684  
14685 LIBDEFLATEAPI enum libdeflate_result
14686 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
14687 const void *in, size_t in_nbytes,
14688 void *out, size_t out_nbytes_avail,
14689 size_t *actual_in_nbytes_ret,
14690 size_t *actual_out_nbytes_ret);
14691  
14692  
14693 LIBDEFLATEAPI enum libdeflate_result
14694 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
14695 const void *in, size_t in_nbytes,
14696 void *out, size_t out_nbytes_avail,
14697 size_t *actual_out_nbytes_ret);
14698  
14699  
14700 LIBDEFLATEAPI enum libdeflate_result
14701 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
14702 const void *in, size_t in_nbytes,
14703 void *out, size_t out_nbytes_avail,
14704 size_t *actual_in_nbytes_ret,
14705 size_t *actual_out_nbytes_ret);
14706  
14707  
14708 LIBDEFLATEAPI void
14709 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
14710  
14711  
14712  
14713  
14714  
14715  
14716 LIBDEFLATEAPI uint32_t
14717 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
14718  
14719  
14720  
14721 LIBDEFLATEAPI uint32_t
14722 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
14723  
14724  
14725  
14726  
14727  
14728  
14729 LIBDEFLATEAPI void
14730 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
14731 void (*free_func)(void *));
14732  
14733  
14734 struct libdeflate_options {
14735  
14736
14737 size_t sizeof_options;
14738  
14739
14740 void *(*malloc_func)(size_t);
14741 void (*free_func)(void *);
14742 };
14743  
14744 #ifdef __cplusplus
14745 }
14746 #endif
14747  
14748 #endif
14749  
14750  
14751 #include <stdbool.h>
14752 #include <stddef.h>
14753 #include <stdint.h>
14754 #ifdef _MSC_VER
14755 # include <intrin.h>
14756 # include <stdlib.h>
14757
14758
14759 # pragma warning(disable : 4146)
14760
14761 # pragma warning(disable : 4018)
14762 # pragma warning(disable : 4244)
14763 # pragma warning(disable : 4267)
14764 # pragma warning(disable : 4310)
14765
14766 # pragma warning(disable : 4100)
14767 # pragma warning(disable : 4127)
14768 # pragma warning(disable : 4189)
14769 # pragma warning(disable : 4232)
14770 # pragma warning(disable : 4245)
14771 # pragma warning(disable : 4295)
14772 #endif
14773 #ifndef FREESTANDING
14774 # include <string.h>
14775 #endif
14776  
14777  
14778  
14779  
14780  
14781  
14782 #undef ARCH_X86_64
14783 #undef ARCH_X86_32
14784 #undef ARCH_ARM64
14785 #undef ARCH_ARM32
14786 #undef ARCH_RISCV
14787 #ifdef _MSC_VER
14788
14789 # if defined(_M_X64) && !defined(_M_ARM64EC)
14790 # define ARCH_X86_64
14791 # elif defined(_M_IX86)
14792 # define ARCH_X86_32
14793 # elif defined(_M_ARM64)
14794 # define ARCH_ARM64
14795 # elif defined(_M_ARM)
14796 # define ARCH_ARM32
14797 # endif
14798 #else
14799 # if defined(__x86_64__)
14800 # define ARCH_X86_64
14801 # elif defined(__i386__)
14802 # define ARCH_X86_32
14803 # elif defined(__aarch64__)
14804 # define ARCH_ARM64
14805 # elif defined(__arm__)
14806 # define ARCH_ARM32
14807 # elif defined(__riscv)
14808 # define ARCH_RISCV
14809 # endif
14810 #endif
14811  
14812  
14813  
14814  
14815  
14816  
14817 typedef uint8_t u8;
14818 typedef uint16_t u16;
14819 typedef uint32_t u32;
14820 typedef uint64_t u64;
14821 typedef int8_t s8;
14822 typedef int16_t s16;
14823 typedef int32_t s32;
14824 typedef int64_t s64;
14825  
14826  
14827 #ifdef _MSC_VER
14828 # ifdef _WIN64
14829 typedef long long ssize_t;
14830 # else
14831 typedef long ssize_t;
14832 # endif
14833 #endif
14834  
14835  
14836 typedef size_t machine_word_t;
14837  
14838  
14839 #define WORDBYTES ((int)sizeof(machine_word_t))
14840  
14841  
14842 #define WORDBITS (8 * WORDBYTES)
14843  
14844  
14845  
14846  
14847  
14848  
14849 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
14850 # define GCC_PREREQ(major, minor) \
14851 (__GNUC__ > (major) || \
14852 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
14853 # if !GCC_PREREQ(4, 9)
14854 # error "gcc versions older than 4.9 are no longer supported"
14855 # endif
14856 #else
14857 # define GCC_PREREQ(major, minor) 0
14858 #endif
14859 #ifdef __clang__
14860 # ifdef __apple_build_version__
14861 # define CLANG_PREREQ(major, minor, apple_version) \
14862 (__apple_build_version__ >= (apple_version))
14863 # else
14864 # define CLANG_PREREQ(major, minor, apple_version) \
14865 (__clang_major__ > (major) || \
14866 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
14867 # endif
14868 # if !CLANG_PREREQ(3, 9, 8000000)
14869 # error "clang versions older than 3.9 are no longer supported"
14870 # endif
14871 #else
14872 # define CLANG_PREREQ(major, minor, apple_version) 0
14873 #endif
14874 #ifdef _MSC_VER
14875 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
14876 # if !MSVC_PREREQ(1928)
14877 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
14878 # endif
14879 #else
14880 # define MSVC_PREREQ(version) 0
14881 #endif
14882  
14883  
14884 #ifndef __has_attribute
14885 # define __has_attribute(attribute) 0
14886 #endif
14887  
14888  
14889 #ifndef __has_builtin
14890 # define __has_builtin(builtin) 0
14891 #endif
14892  
14893  
14894 #ifdef _MSC_VER
14895 # define inline __inline
14896 #endif
14897  
14898  
14899 #if defined(__GNUC__) || __has_attribute(always_inline)
14900 # define forceinline inline __attribute__((always_inline))
14901 #elif defined(_MSC_VER)
14902 # define forceinline __forceinline
14903 #else
14904 # define forceinline inline
14905 #endif
14906  
14907  
14908 #if defined(__GNUC__) || __has_attribute(unused)
14909 # define MAYBE_UNUSED __attribute__((unused))
14910 #else
14911 # define MAYBE_UNUSED
14912 #endif
14913  
14914  
14915 #if defined(__GNUC__) || __has_attribute(noreturn)
14916 # define NORETURN __attribute__((noreturn))
14917 #else
14918 # define NORETURN
14919 #endif
14920  
14921  
14922 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
14923 # define likely(expr) __builtin_expect(!!(expr), 1)
14924 #else
14925 # define likely(expr) (expr)
14926 #endif
14927  
14928  
14929 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
14930 # define unlikely(expr) __builtin_expect(!!(expr), 0)
14931 #else
14932 # define unlikely(expr) (expr)
14933 #endif
14934  
14935  
14936 #undef prefetchr
14937 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
14938 # define prefetchr(addr) __builtin_prefetch((addr), 0)
14939 #elif defined(_MSC_VER)
14940 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
14941 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
14942 # elif defined(ARCH_ARM64)
14943 # define prefetchr(addr) __prefetch2((addr), 0x00 )
14944 # elif defined(ARCH_ARM32)
14945 # define prefetchr(addr) __prefetch(addr)
14946 # endif
14947 #endif
14948 #ifndef prefetchr
14949 # define prefetchr(addr)
14950 #endif
14951  
14952  
14953 #undef prefetchw
14954 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
14955 # define prefetchw(addr) __builtin_prefetch((addr), 1)
14956 #elif defined(_MSC_VER)
14957 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
14958 # define prefetchw(addr) _m_prefetchw(addr)
14959 # elif defined(ARCH_ARM64)
14960 # define prefetchw(addr) __prefetch2((addr), 0x10 )
14961 # elif defined(ARCH_ARM32)
14962 # define prefetchw(addr) __prefetchw(addr)
14963 # endif
14964 #endif
14965 #ifndef prefetchw
14966 # define prefetchw(addr)
14967 #endif
14968  
14969  
14970 #undef _aligned_attribute
14971 #if defined(__GNUC__) || __has_attribute(aligned)
14972 # define _aligned_attribute(n) __attribute__((aligned(n)))
14973 #elif defined(_MSC_VER)
14974 # define _aligned_attribute(n) __declspec(align(n))
14975 #endif
14976  
14977  
14978 #if defined(__GNUC__) || __has_attribute(target)
14979 # define _target_attribute(attrs) __attribute__((target(attrs)))
14980 #else
14981 # define _target_attribute(attrs)
14982 #endif
14983  
14984  
14985  
14986  
14987  
14988 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
14989 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
14990 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
14991 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
14992 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
14993 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
14994 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
14995  
14996  
14997  
14998  
14999  
15000  
15001 #if defined(__BYTE_ORDER__)
15002 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
15003 #elif defined(_MSC_VER)
15004 # define CPU_IS_LITTLE_ENDIAN() true
15005 #else
15006 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
15007 {
15008 union {
15009 u32 w;
15010 u8 b;
15011 } u;
15012  
15013 u.w = 1;
15014 return u.b;
15015 }
15016 #endif
15017  
15018  
15019 static forceinline u16 bswap16(u16 v)
15020 {
15021 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
15022 return __builtin_bswap16(v);
15023 #elif defined(_MSC_VER)
15024 return _byteswap_ushort(v);
15025 #else
15026 return (v << 8) | (v >> 8);
15027 #endif
15028 }
15029  
15030  
15031 static forceinline u32 bswap32(u32 v)
15032 {
15033 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
15034 return __builtin_bswap32(v);
15035 #elif defined(_MSC_VER)
15036 return _byteswap_ulong(v);
15037 #else
15038 return ((v & 0x000000FF) << 24) |
15039 ((v & 0x0000FF00) << 8) |
15040 ((v & 0x00FF0000) >> 8) |
15041 ((v & 0xFF000000) >> 24);
15042 #endif
15043 }
15044  
15045  
15046 static forceinline u64 bswap64(u64 v)
15047 {
15048 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
15049 return __builtin_bswap64(v);
15050 #elif defined(_MSC_VER)
15051 return _byteswap_uint64(v);
15052 #else
15053 return ((v & 0x00000000000000FF) << 56) |
15054 ((v & 0x000000000000FF00) << 40) |
15055 ((v & 0x0000000000FF0000) << 24) |
15056 ((v & 0x00000000FF000000) << 8) |
15057 ((v & 0x000000FF00000000) >> 8) |
15058 ((v & 0x0000FF0000000000) >> 24) |
15059 ((v & 0x00FF000000000000) >> 40) |
15060 ((v & 0xFF00000000000000) >> 56);
15061 #endif
15062 }
15063  
15064 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
15065 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
15066 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
15067 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
15068 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
15069 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
15070  
15071  
15072  
15073  
15074  
15075  
15076 #if (defined(__GNUC__) || defined(__clang__)) && \
15077 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
15078 defined(__ARM_FEATURE_UNALIGNED) || \
15079 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
15080 defined(__riscv_misaligned_fast) || \
15081 defined(__wasm__))
15082 # define UNALIGNED_ACCESS_IS_FAST 1
15083 #elif defined(_MSC_VER)
15084 # define UNALIGNED_ACCESS_IS_FAST 1
15085 #else
15086 # define UNALIGNED_ACCESS_IS_FAST 0
15087 #endif
15088  
15089  
15090  
15091 #ifdef FREESTANDING
15092 # define MEMCOPY __builtin_memcpy
15093 #else
15094 # define MEMCOPY memcpy
15095 #endif
15096  
15097  
15098  
15099 #define DEFINE_UNALIGNED_TYPE(type) \
15100 static forceinline type \
15101 load_##type##_unaligned(const void *p) \
15102 { \
15103 type v; \
15104 \
15105 MEMCOPY(&v, p, sizeof(v)); \
15106 return v; \
15107 } \
15108 \
15109 static forceinline void \
15110 store_##type##_unaligned(type v, void *p) \
15111 { \
15112 MEMCOPY(p, &v, sizeof(v)); \
15113 }
15114  
15115 DEFINE_UNALIGNED_TYPE(u16)
15116 DEFINE_UNALIGNED_TYPE(u32)
15117 DEFINE_UNALIGNED_TYPE(u64)
15118 DEFINE_UNALIGNED_TYPE(machine_word_t)
15119  
15120 #undef MEMCOPY
15121  
15122 #define load_word_unaligned load_machine_word_t_unaligned
15123 #define store_word_unaligned store_machine_word_t_unaligned
15124  
15125  
15126  
15127 static forceinline u16
15128 get_unaligned_le16(const u8 *p)
15129 {
15130 if (UNALIGNED_ACCESS_IS_FAST)
15131 return le16_bswap(load_u16_unaligned(p));
15132 else
15133 return ((u16)p[1] << 8) | p[0];
15134 }
15135  
15136 static forceinline u16
15137 get_unaligned_be16(const u8 *p)
15138 {
15139 if (UNALIGNED_ACCESS_IS_FAST)
15140 return be16_bswap(load_u16_unaligned(p));
15141 else
15142 return ((u16)p[0] << 8) | p[1];
15143 }
15144  
15145 static forceinline u32
15146 get_unaligned_le32(const u8 *p)
15147 {
15148 if (UNALIGNED_ACCESS_IS_FAST)
15149 return le32_bswap(load_u32_unaligned(p));
15150 else
15151 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
15152 ((u32)p[1] << 8) | p[0];
15153 }
15154  
15155 static forceinline u32
15156 get_unaligned_be32(const u8 *p)
15157 {
15158 if (UNALIGNED_ACCESS_IS_FAST)
15159 return be32_bswap(load_u32_unaligned(p));
15160 else
15161 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
15162 ((u32)p[2] << 8) | p[3];
15163 }
15164  
15165 static forceinline u64
15166 get_unaligned_le64(const u8 *p)
15167 {
15168 if (UNALIGNED_ACCESS_IS_FAST)
15169 return le64_bswap(load_u64_unaligned(p));
15170 else
15171 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
15172 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
15173 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
15174 ((u64)p[1] << 8) | p[0];
15175 }
15176  
15177 static forceinline machine_word_t
15178 get_unaligned_leword(const u8 *p)
15179 {
15180 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
15181 if (WORDBITS == 32)
15182 return get_unaligned_le32(p);
15183 else
15184 return get_unaligned_le64(p);
15185 }
15186  
15187  
15188  
15189 static forceinline void
15190 put_unaligned_le16(u16 v, u8 *p)
15191 {
15192 if (UNALIGNED_ACCESS_IS_FAST) {
15193 store_u16_unaligned(le16_bswap(v), p);
15194 } else {
15195 p[0] = (u8)(v >> 0);
15196 p[1] = (u8)(v >> 8);
15197 }
15198 }
15199  
15200 static forceinline void
15201 put_unaligned_be16(u16 v, u8 *p)
15202 {
15203 if (UNALIGNED_ACCESS_IS_FAST) {
15204 store_u16_unaligned(be16_bswap(v), p);
15205 } else {
15206 p[0] = (u8)(v >> 8);
15207 p[1] = (u8)(v >> 0);
15208 }
15209 }
15210  
15211 static forceinline void
15212 put_unaligned_le32(u32 v, u8 *p)
15213 {
15214 if (UNALIGNED_ACCESS_IS_FAST) {
15215 store_u32_unaligned(le32_bswap(v), p);
15216 } else {
15217 p[0] = (u8)(v >> 0);
15218 p[1] = (u8)(v >> 8);
15219 p[2] = (u8)(v >> 16);
15220 p[3] = (u8)(v >> 24);
15221 }
15222 }
15223  
15224 static forceinline void
15225 put_unaligned_be32(u32 v, u8 *p)
15226 {
15227 if (UNALIGNED_ACCESS_IS_FAST) {
15228 store_u32_unaligned(be32_bswap(v), p);
15229 } else {
15230 p[0] = (u8)(v >> 24);
15231 p[1] = (u8)(v >> 16);
15232 p[2] = (u8)(v >> 8);
15233 p[3] = (u8)(v >> 0);
15234 }
15235 }
15236  
15237 static forceinline void
15238 put_unaligned_le64(u64 v, u8 *p)
15239 {
15240 if (UNALIGNED_ACCESS_IS_FAST) {
15241 store_u64_unaligned(le64_bswap(v), p);
15242 } else {
15243 p[0] = (u8)(v >> 0);
15244 p[1] = (u8)(v >> 8);
15245 p[2] = (u8)(v >> 16);
15246 p[3] = (u8)(v >> 24);
15247 p[4] = (u8)(v >> 32);
15248 p[5] = (u8)(v >> 40);
15249 p[6] = (u8)(v >> 48);
15250 p[7] = (u8)(v >> 56);
15251 }
15252 }
15253  
15254 static forceinline void
15255 put_unaligned_leword(machine_word_t v, u8 *p)
15256 {
15257 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
15258 if (WORDBITS == 32)
15259 put_unaligned_le32(v, p);
15260 else
15261 put_unaligned_le64(v, p);
15262 }
15263  
15264  
15265  
15266  
15267  
15268  
15269  
15270 static forceinline unsigned
15271 bsr32(u32 v)
15272 {
15273 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
15274 return 31 - __builtin_clz(v);
15275 #elif defined(_MSC_VER)
15276 unsigned long i;
15277  
15278 _BitScanReverse(&i, v);
15279 return i;
15280 #else
15281 unsigned i = 0;
15282  
15283 while ((v >>= 1) != 0)
15284 i++;
15285 return i;
15286 #endif
15287 }
15288  
15289 static forceinline unsigned
15290 bsr64(u64 v)
15291 {
15292 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
15293 return 63 - __builtin_clzll(v);
15294 #elif defined(_MSC_VER) && defined(_WIN64)
15295 unsigned long i;
15296  
15297 _BitScanReverse64(&i, v);
15298 return i;
15299 #else
15300 unsigned i = 0;
15301  
15302 while ((v >>= 1) != 0)
15303 i++;
15304 return i;
15305 #endif
15306 }
15307  
15308 static forceinline unsigned
15309 bsrw(machine_word_t v)
15310 {
15311 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
15312 if (WORDBITS == 32)
15313 return bsr32(v);
15314 else
15315 return bsr64(v);
15316 }
15317  
15318  
15319  
15320 static forceinline unsigned
15321 bsf32(u32 v)
15322 {
15323 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
15324 return __builtin_ctz(v);
15325 #elif defined(_MSC_VER)
15326 unsigned long i;
15327  
15328 _BitScanForward(&i, v);
15329 return i;
15330 #else
15331 unsigned i = 0;
15332  
15333 for (; (v & 1) == 0; v >>= 1)
15334 i++;
15335 return i;
15336 #endif
15337 }
15338  
15339 static forceinline unsigned
15340 bsf64(u64 v)
15341 {
15342 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
15343 return __builtin_ctzll(v);
15344 #elif defined(_MSC_VER) && defined(_WIN64)
15345 unsigned long i;
15346  
15347 _BitScanForward64(&i, v);
15348 return i;
15349 #else
15350 unsigned i = 0;
15351  
15352 for (; (v & 1) == 0; v >>= 1)
15353 i++;
15354 return i;
15355 #endif
15356 }
15357  
15358 static forceinline unsigned
15359 bsfw(machine_word_t v)
15360 {
15361 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
15362 if (WORDBITS == 32)
15363 return bsf32(v);
15364 else
15365 return bsf64(v);
15366 }
15367  
15368  
15369 #undef rbit32
15370 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
15371 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
15372 static forceinline u32
15373 rbit32(u32 v)
15374 {
15375 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
15376 return v;
15377 }
15378 #define rbit32 rbit32
15379 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
15380 static forceinline u32
15381 rbit32(u32 v)
15382 {
15383 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
15384 return v;
15385 }
15386 #define rbit32 rbit32
15387 #endif
15388  
15389 #endif
15390  
15391  
15392 typedef void *(*malloc_func_t)(size_t);
15393 typedef void (*free_func_t)(void *);
15394  
15395 extern malloc_func_t libdeflate_default_malloc_func;
15396 extern free_func_t libdeflate_default_free_func;
15397  
15398 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
15399 size_t alignment, size_t size);
15400 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
15401  
15402 #ifdef FREESTANDING
15403  
15404 void *memset(void *s, int c, size_t n);
15405 #define memset(s, c, n) __builtin_memset((s), (c), (n))
15406  
15407 void *memcpy(void *dest, const void *src, size_t n);
15408 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
15409  
15410 void *memmove(void *dest, const void *src, size_t n);
15411 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
15412  
15413 int memcmp(const void *s1, const void *s2, size_t n);
15414 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
15415  
15416 #undef LIBDEFLATE_ENABLE_ASSERTIONS
15417 #else
15418 # include <string.h>
15419
15420 # ifdef __clang_analyzer__
15421 # define LIBDEFLATE_ENABLE_ASSERTIONS
15422 # endif
15423 #endif
15424  
15425  
15426 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
15427 NORETURN void
15428 libdeflate_assertion_failed(const char *expr, const char *file, int line);
15429 #define ASSERT(expr) { if (unlikely(!(expr))) \
15430 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
15431 #else
15432 #define ASSERT(expr) (void)(expr)
15433 #endif
15434  
15435 #define CONCAT_IMPL(a, b) a##b
15436 #define CONCAT(a, b) CONCAT_IMPL(a, b)
15437 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
15438  
15439 #endif
15440  
15441  
15442 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
15443  
15444 #define ARM_CPU_FEATURE_NEON (1 << 0)
15445 #define ARM_CPU_FEATURE_PMULL (1 << 1)
15446  
15447 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
15448 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
15449 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
15450 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
15451  
15452 #if !defined(FREESTANDING) && \
15453 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
15454 (defined(__linux__) || \
15455 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
15456 (defined(_WIN32) && defined(ARCH_ARM64)))
15457  
15458 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
15459 extern volatile u32 libdeflate_arm_cpu_features;
15460  
15461 void libdeflate_init_arm_cpu_features(void);
15462  
15463 static inline u32 get_arm_cpu_features(void)
15464 {
15465 if (libdeflate_arm_cpu_features == 0)
15466 libdeflate_init_arm_cpu_features();
15467 return libdeflate_arm_cpu_features;
15468 }
15469 #else
15470 static inline u32 get_arm_cpu_features(void) { return 0; }
15471 #endif
15472  
15473  
15474 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
15475 # define HAVE_NEON(features) 1
15476 # define HAVE_NEON_NATIVE 1
15477 #else
15478 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
15479 # define HAVE_NEON_NATIVE 0
15480 #endif
15481  
15482 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
15483 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
15484 # define HAVE_NEON_INTRIN 1
15485 # include <arm_neon.h>
15486 #else
15487 # define HAVE_NEON_INTRIN 0
15488 #endif
15489  
15490  
15491 #ifdef __ARM_FEATURE_CRYPTO
15492 # define HAVE_PMULL(features) 1
15493 #else
15494 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
15495 #endif
15496 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
15497 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
15498 CPU_IS_LITTLE_ENDIAN()
15499 # define HAVE_PMULL_INTRIN 1
15500
15501 # ifdef _MSC_VER
15502 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
15503 # else
15504 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
15505 # endif
15506 #else
15507 # define HAVE_PMULL_INTRIN 0
15508 #endif
15509  
15510  
15511 #ifdef __ARM_FEATURE_CRC32
15512 # define HAVE_CRC32(features) 1
15513 #else
15514 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
15515 #endif
15516 #if defined(ARCH_ARM64) && \
15517 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
15518 # define HAVE_CRC32_INTRIN 1
15519 # if defined(__GNUC__) || defined(__clang__)
15520 # include <arm_acle.h>
15521 # endif
15522
15523 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
15524 !defined(__ARM_FEATURE_CRC32)
15525 # undef __crc32b
15526 # define __crc32b(a, b) \
15527 ({ uint32_t res; \
15528 __asm__("crc32b %w0, %w1, %w2" \
15529 : "=r" (res) : "r" (a), "r" (b)); \
15530 res; })
15531 # undef __crc32h
15532 # define __crc32h(a, b) \
15533 ({ uint32_t res; \
15534 __asm__("crc32h %w0, %w1, %w2" \
15535 : "=r" (res) : "r" (a), "r" (b)); \
15536 res; })
15537 # undef __crc32w
15538 # define __crc32w(a, b) \
15539 ({ uint32_t res; \
15540 __asm__("crc32w %w0, %w1, %w2" \
15541 : "=r" (res) : "r" (a), "r" (b)); \
15542 res; })
15543 # undef __crc32d
15544 # define __crc32d(a, b) \
15545 ({ uint32_t res; \
15546 __asm__("crc32x %w0, %w1, %2" \
15547 : "=r" (res) : "r" (a), "r" (b)); \
15548 res; })
15549 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
15550 # endif
15551 #else
15552 # define HAVE_CRC32_INTRIN 0
15553 #endif
15554  
15555  
15556 #ifdef __ARM_FEATURE_SHA3
15557 # define HAVE_SHA3(features) 1
15558 #else
15559 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
15560 #endif
15561 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
15562 (GCC_PREREQ(9, 1) || \
15563 CLANG_PREREQ(7, 0, 10010463) )
15564 # define HAVE_SHA3_INTRIN 1
15565
15566 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
15567 !defined(__ARM_FEATURE_SHA3)
15568 # undef veor3q_u8
15569 # define veor3q_u8(a, b, c) \
15570 ({ uint8x16_t res; \
15571 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
15572 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
15573 res; })
15574 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
15575 # endif
15576 #else
15577 # define HAVE_SHA3_INTRIN 0
15578 #endif
15579  
15580  
15581 #ifdef __ARM_FEATURE_DOTPROD
15582 # define HAVE_DOTPROD(features) 1
15583 #else
15584 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
15585 #endif
15586 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
15587 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
15588 # define HAVE_DOTPROD_INTRIN 1
15589
15590 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
15591 !defined(__ARM_FEATURE_DOTPROD)
15592 # undef vdotq_u32
15593 # define vdotq_u32(a, b, c) \
15594 ({ uint32x4_t res = (a); \
15595 __asm__("udot %0.4s, %1.16b, %2.16b" \
15596 : "+w" (res) : "w" (b), "w" (c)); \
15597 res; })
15598 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
15599 # endif
15600 #else
15601 # define HAVE_DOTPROD_INTRIN 0
15602 #endif
15603  
15604 #endif
15605  
15606 #endif
15607  
15608  
15609 #if HAVE_NEON_NATIVE
15610 static forceinline void
15611 matchfinder_init_neon(mf_pos_t *data, size_t size)
15612 {
15613 int16x8_t *p = (int16x8_t *)data;
15614 int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);
15615  
15616 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
15617 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
15618 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
15619  
15620 do {
15621 p[0] = v;
15622 p[1] = v;
15623 p[2] = v;
15624 p[3] = v;
15625 p += 4;
15626 size -= 4 * sizeof(*p);
15627 } while (size != 0);
15628 }
15629 #define matchfinder_init matchfinder_init_neon
15630  
15631 static forceinline void
15632 matchfinder_rebase_neon(mf_pos_t *data, size_t size)
15633 {
15634 int16x8_t *p = (int16x8_t *)data;
15635 int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);
15636  
15637 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
15638 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
15639 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
15640  
15641 do {
15642 p[0] = vqaddq_s16(p[0], v);
15643 p[1] = vqaddq_s16(p[1], v);
15644 p[2] = vqaddq_s16(p[2], v);
15645 p[3] = vqaddq_s16(p[3], v);
15646 p += 4;
15647 size -= 4 * sizeof(*p);
15648 } while (size != 0);
15649 }
15650 #define matchfinder_rebase matchfinder_rebase_neon
15651  
15652 #endif
15653  
15654 #endif
15655  
15656 # elif defined(ARCH_RISCV)
15657 # include "riscv/matchfinder_impl.h"
15658 # elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
15659 /* # include "x86/matchfinder_impl.h" */
15660  
15661  
15662 #ifndef LIB_X86_MATCHFINDER_IMPL_H
15663 #define LIB_X86_MATCHFINDER_IMPL_H
15664  
15665 /* #include "x86-cpu_features.h" */
15666  
15667  
15668 #ifndef LIB_X86_CPU_FEATURES_H
15669 #define LIB_X86_CPU_FEATURES_H
15670  
15671 /* #include "lib_common.h" */
15672  
15673  
15674 #ifndef LIB_LIB_COMMON_H
15675 #define LIB_LIB_COMMON_H
15676  
15677 #ifdef LIBDEFLATE_H
15678
15679 # error "lib_common.h must always be included before libdeflate.h"
15680 #endif
15681  
15682 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
15683 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
15684 #elif defined(__GNUC__)
15685 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
15686 #else
15687 # define LIBDEFLATE_EXPORT_SYM
15688 #endif
15689  
15690  
15691 #if defined(__GNUC__) && defined(__i386__)
15692 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
15693 #else
15694 # define LIBDEFLATE_ALIGN_STACK
15695 #endif
15696  
15697 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
15698  
15699 /* #include "../common_defs.h" */
15700  
15701  
15702 #ifndef COMMON_DEFS_H
15703 #define COMMON_DEFS_H
15704  
15705 /* #include "libdeflate.h" */
15706  
15707  
15708 #ifndef LIBDEFLATE_H
15709 #define LIBDEFLATE_H
15710  
15711 #include <stddef.h>
15712 #include <stdint.h>
15713  
15714 #ifdef __cplusplus
15715 extern "C" {
15716 #endif
15717  
15718 #define LIBDEFLATE_VERSION_MAJOR 1
15719 #define LIBDEFLATE_VERSION_MINOR 26
15720 #define LIBDEFLATE_VERSION_STRING "1.26"
15721  
15722  
15723 #ifndef LIBDEFLATEAPI
15724 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
15725 # define LIBDEFLATEAPI __declspec(dllimport)
15726 # else
15727 # define LIBDEFLATEAPI
15728 # endif
15729 #endif
15730  
15731  
15732  
15733  
15734  
15735 struct libdeflate_compressor;
15736 struct libdeflate_options;
15737  
15738  
15739 LIBDEFLATEAPI struct libdeflate_compressor *
15740 libdeflate_alloc_compressor(int compression_level);
15741  
15742  
15743 LIBDEFLATEAPI struct libdeflate_compressor *
15744 libdeflate_alloc_compressor_ex(int compression_level,
15745 const struct libdeflate_options *options);
15746  
15747  
15748 LIBDEFLATEAPI size_t
15749 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
15750 const void *in, size_t in_nbytes,
15751 void *out, size_t out_nbytes_avail);
15752  
15753  
15754 LIBDEFLATEAPI size_t
15755 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
15756 size_t in_nbytes);
15757  
15758  
15759 LIBDEFLATEAPI size_t
15760 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
15761 const void *in, size_t in_nbytes,
15762 void *out, size_t out_nbytes_avail);
15763  
15764  
15765 LIBDEFLATEAPI size_t
15766 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
15767 size_t in_nbytes);
15768  
15769  
15770 LIBDEFLATEAPI size_t
15771 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
15772 const void *in, size_t in_nbytes,
15773 void *out, size_t out_nbytes_avail);
15774  
15775  
15776 LIBDEFLATEAPI size_t
15777 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
15778 size_t in_nbytes);
15779  
15780  
15781 LIBDEFLATEAPI void
15782 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
15783  
15784  
15785  
15786  
15787  
15788 struct libdeflate_decompressor;
15789 struct libdeflate_options;
15790  
15791  
15792 LIBDEFLATEAPI struct libdeflate_decompressor *
15793 libdeflate_alloc_decompressor(void);
15794  
15795  
15796 LIBDEFLATEAPI struct libdeflate_decompressor *
15797 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
15798  
15799  
15800 enum libdeflate_result {
15801
15802 LIBDEFLATE_SUCCESS = 0,
15803  
15804
15805 LIBDEFLATE_BAD_DATA = 1,
15806  
15807
15808 LIBDEFLATE_SHORT_OUTPUT = 2,
15809  
15810
15811 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
15812 };
15813  
15814  
15815 LIBDEFLATEAPI enum libdeflate_result
15816 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
15817 const void *in, size_t in_nbytes,
15818 void *out, size_t out_nbytes_avail,
15819 size_t *actual_out_nbytes_ret);
15820  
15821  
15822 LIBDEFLATEAPI enum libdeflate_result
15823 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
15824 const void *in, size_t in_nbytes,
15825 void *out, size_t out_nbytes_avail,
15826 size_t *actual_in_nbytes_ret,
15827 size_t *actual_out_nbytes_ret);
15828  
15829  
15830 LIBDEFLATEAPI enum libdeflate_result
15831 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
15832 const void *in, size_t in_nbytes,
15833 void *out, size_t out_nbytes_avail,
15834 size_t *actual_out_nbytes_ret);
15835  
15836  
15837 LIBDEFLATEAPI enum libdeflate_result
15838 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
15839 const void *in, size_t in_nbytes,
15840 void *out, size_t out_nbytes_avail,
15841 size_t *actual_in_nbytes_ret,
15842 size_t *actual_out_nbytes_ret);
15843  
15844  
15845 LIBDEFLATEAPI enum libdeflate_result
15846 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
15847 const void *in, size_t in_nbytes,
15848 void *out, size_t out_nbytes_avail,
15849 size_t *actual_out_nbytes_ret);
15850  
15851  
15852 LIBDEFLATEAPI enum libdeflate_result
15853 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
15854 const void *in, size_t in_nbytes,
15855 void *out, size_t out_nbytes_avail,
15856 size_t *actual_in_nbytes_ret,
15857 size_t *actual_out_nbytes_ret);
15858  
15859  
15860 LIBDEFLATEAPI void
15861 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
15862  
15863  
15864  
15865  
15866  
15867  
15868 LIBDEFLATEAPI uint32_t
15869 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
15870  
15871  
15872  
15873 LIBDEFLATEAPI uint32_t
15874 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
15875  
15876  
15877  
15878  
15879  
15880  
15881 LIBDEFLATEAPI void
15882 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
15883 void (*free_func)(void *));
15884  
15885  
15886 struct libdeflate_options {
15887  
15888
15889 size_t sizeof_options;
15890  
15891
15892 void *(*malloc_func)(size_t);
15893 void (*free_func)(void *);
15894 };
15895  
15896 #ifdef __cplusplus
15897 }
15898 #endif
15899  
15900 #endif
15901  
15902  
15903 #include <stdbool.h>
15904 #include <stddef.h>
15905 #include <stdint.h>
15906 #ifdef _MSC_VER
15907 # include <intrin.h>
15908 # include <stdlib.h>
15909
15910
15911 # pragma warning(disable : 4146)
15912
15913 # pragma warning(disable : 4018)
15914 # pragma warning(disable : 4244)
15915 # pragma warning(disable : 4267)
15916 # pragma warning(disable : 4310)
15917
15918 # pragma warning(disable : 4100)
15919 # pragma warning(disable : 4127)
15920 # pragma warning(disable : 4189)
15921 # pragma warning(disable : 4232)
15922 # pragma warning(disable : 4245)
15923 # pragma warning(disable : 4295)
15924 #endif
15925 #ifndef FREESTANDING
15926 # include <string.h>
15927 #endif
15928  
15929  
15930  
15931  
15932  
15933  
15934 #undef ARCH_X86_64
15935 #undef ARCH_X86_32
15936 #undef ARCH_ARM64
15937 #undef ARCH_ARM32
15938 #undef ARCH_RISCV
15939 #ifdef _MSC_VER
15940
15941 # if defined(_M_X64) && !defined(_M_ARM64EC)
15942 # define ARCH_X86_64
15943 # elif defined(_M_IX86)
15944 # define ARCH_X86_32
15945 # elif defined(_M_ARM64)
15946 # define ARCH_ARM64
15947 # elif defined(_M_ARM)
15948 # define ARCH_ARM32
15949 # endif
15950 #else
15951 # if defined(__x86_64__)
15952 # define ARCH_X86_64
15953 # elif defined(__i386__)
15954 # define ARCH_X86_32
15955 # elif defined(__aarch64__)
15956 # define ARCH_ARM64
15957 # elif defined(__arm__)
15958 # define ARCH_ARM32
15959 # elif defined(__riscv)
15960 # define ARCH_RISCV
15961 # endif
15962 #endif
15963  
15964  
15965  
15966  
15967  
15968  
15969 typedef uint8_t u8;
15970 typedef uint16_t u16;
15971 typedef uint32_t u32;
15972 typedef uint64_t u64;
15973 typedef int8_t s8;
15974 typedef int16_t s16;
15975 typedef int32_t s32;
15976 typedef int64_t s64;
15977  
15978  
15979 #ifdef _MSC_VER
15980 # ifdef _WIN64
15981 typedef long long ssize_t;
15982 # else
15983 typedef long ssize_t;
15984 # endif
15985 #endif
15986  
15987  
15988 typedef size_t machine_word_t;
15989  
15990  
15991 #define WORDBYTES ((int)sizeof(machine_word_t))
15992  
15993  
15994 #define WORDBITS (8 * WORDBYTES)
15995  
15996  
15997  
15998  
15999  
16000  
16001 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
16002 # define GCC_PREREQ(major, minor) \
16003 (__GNUC__ > (major) || \
16004 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
16005 # if !GCC_PREREQ(4, 9)
16006 # error "gcc versions older than 4.9 are no longer supported"
16007 # endif
16008 #else
16009 # define GCC_PREREQ(major, minor) 0
16010 #endif
16011 #ifdef __clang__
16012 # ifdef __apple_build_version__
16013 # define CLANG_PREREQ(major, minor, apple_version) \
16014 (__apple_build_version__ >= (apple_version))
16015 # else
16016 # define CLANG_PREREQ(major, minor, apple_version) \
16017 (__clang_major__ > (major) || \
16018 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
16019 # endif
16020 # if !CLANG_PREREQ(3, 9, 8000000)
16021 # error "clang versions older than 3.9 are no longer supported"
16022 # endif
16023 #else
16024 # define CLANG_PREREQ(major, minor, apple_version) 0
16025 #endif
16026 #ifdef _MSC_VER
16027 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
16028 # if !MSVC_PREREQ(1928)
16029 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
16030 # endif
16031 #else
16032 # define MSVC_PREREQ(version) 0
16033 #endif
16034  
16035  
16036 #ifndef __has_attribute
16037 # define __has_attribute(attribute) 0
16038 #endif
16039  
16040  
16041 #ifndef __has_builtin
16042 # define __has_builtin(builtin) 0
16043 #endif
16044  
16045  
16046 #ifdef _MSC_VER
16047 # define inline __inline
16048 #endif
16049  
16050  
16051 #if defined(__GNUC__) || __has_attribute(always_inline)
16052 # define forceinline inline __attribute__((always_inline))
16053 #elif defined(_MSC_VER)
16054 # define forceinline __forceinline
16055 #else
16056 # define forceinline inline
16057 #endif
16058  
16059  
16060 #if defined(__GNUC__) || __has_attribute(unused)
16061 # define MAYBE_UNUSED __attribute__((unused))
16062 #else
16063 # define MAYBE_UNUSED
16064 #endif
16065  
16066  
16067 #if defined(__GNUC__) || __has_attribute(noreturn)
16068 # define NORETURN __attribute__((noreturn))
16069 #else
16070 # define NORETURN
16071 #endif
16072  
16073  
16074 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
16075 # define likely(expr) __builtin_expect(!!(expr), 1)
16076 #else
16077 # define likely(expr) (expr)
16078 #endif
16079  
16080  
16081 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
16082 # define unlikely(expr) __builtin_expect(!!(expr), 0)
16083 #else
16084 # define unlikely(expr) (expr)
16085 #endif
16086  
16087  
16088 #undef prefetchr
16089 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
16090 # define prefetchr(addr) __builtin_prefetch((addr), 0)
16091 #elif defined(_MSC_VER)
16092 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
16093 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
16094 # elif defined(ARCH_ARM64)
16095 # define prefetchr(addr) __prefetch2((addr), 0x00 )
16096 # elif defined(ARCH_ARM32)
16097 # define prefetchr(addr) __prefetch(addr)
16098 # endif
16099 #endif
16100 #ifndef prefetchr
16101 # define prefetchr(addr)
16102 #endif
16103  
16104  
16105 #undef prefetchw
16106 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
16107 # define prefetchw(addr) __builtin_prefetch((addr), 1)
16108 #elif defined(_MSC_VER)
16109 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
16110 # define prefetchw(addr) _m_prefetchw(addr)
16111 # elif defined(ARCH_ARM64)
16112 # define prefetchw(addr) __prefetch2((addr), 0x10 )
16113 # elif defined(ARCH_ARM32)
16114 # define prefetchw(addr) __prefetchw(addr)
16115 # endif
16116 #endif
16117 #ifndef prefetchw
16118 # define prefetchw(addr)
16119 #endif
16120  
16121  
16122 #undef _aligned_attribute
16123 #if defined(__GNUC__) || __has_attribute(aligned)
16124 # define _aligned_attribute(n) __attribute__((aligned(n)))
16125 #elif defined(_MSC_VER)
16126 # define _aligned_attribute(n) __declspec(align(n))
16127 #endif
16128  
16129  
16130 #if defined(__GNUC__) || __has_attribute(target)
16131 # define _target_attribute(attrs) __attribute__((target(attrs)))
16132 #else
16133 # define _target_attribute(attrs)
16134 #endif
16135  
16136  
16137  
16138  
16139  
16140 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
16141 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
16142 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
16143 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
16144 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
16145 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
16146 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
16147  
16148  
16149  
16150  
16151  
16152  
16153 #if defined(__BYTE_ORDER__)
16154 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
16155 #elif defined(_MSC_VER)
16156 # define CPU_IS_LITTLE_ENDIAN() true
16157 #else
16158 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
16159 {
16160 union {
16161 u32 w;
16162 u8 b;
16163 } u;
16164  
16165 u.w = 1;
16166 return u.b;
16167 }
16168 #endif
16169  
16170  
16171 static forceinline u16 bswap16(u16 v)
16172 {
16173 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
16174 return __builtin_bswap16(v);
16175 #elif defined(_MSC_VER)
16176 return _byteswap_ushort(v);
16177 #else
16178 return (v << 8) | (v >> 8);
16179 #endif
16180 }
16181  
16182  
16183 static forceinline u32 bswap32(u32 v)
16184 {
16185 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
16186 return __builtin_bswap32(v);
16187 #elif defined(_MSC_VER)
16188 return _byteswap_ulong(v);
16189 #else
16190 return ((v & 0x000000FF) << 24) |
16191 ((v & 0x0000FF00) << 8) |
16192 ((v & 0x00FF0000) >> 8) |
16193 ((v & 0xFF000000) >> 24);
16194 #endif
16195 }
16196  
16197  
16198 static forceinline u64 bswap64(u64 v)
16199 {
16200 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
16201 return __builtin_bswap64(v);
16202 #elif defined(_MSC_VER)
16203 return _byteswap_uint64(v);
16204 #else
16205 return ((v & 0x00000000000000FF) << 56) |
16206 ((v & 0x000000000000FF00) << 40) |
16207 ((v & 0x0000000000FF0000) << 24) |
16208 ((v & 0x00000000FF000000) << 8) |
16209 ((v & 0x000000FF00000000) >> 8) |
16210 ((v & 0x0000FF0000000000) >> 24) |
16211 ((v & 0x00FF000000000000) >> 40) |
16212 ((v & 0xFF00000000000000) >> 56);
16213 #endif
16214 }
16215  
16216 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
16217 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
16218 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
16219 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
16220 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
16221 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
16222  
16223  
16224  
16225  
16226  
16227  
16228 #if (defined(__GNUC__) || defined(__clang__)) && \
16229 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
16230 defined(__ARM_FEATURE_UNALIGNED) || \
16231 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
16232 defined(__riscv_misaligned_fast) || \
16233 defined(__wasm__))
16234 # define UNALIGNED_ACCESS_IS_FAST 1
16235 #elif defined(_MSC_VER)
16236 # define UNALIGNED_ACCESS_IS_FAST 1
16237 #else
16238 # define UNALIGNED_ACCESS_IS_FAST 0
16239 #endif
16240  
16241  
16242  
16243 #ifdef FREESTANDING
16244 # define MEMCOPY __builtin_memcpy
16245 #else
16246 # define MEMCOPY memcpy
16247 #endif
16248  
16249  
16250  
16251 #define DEFINE_UNALIGNED_TYPE(type) \
16252 static forceinline type \
16253 load_##type##_unaligned(const void *p) \
16254 { \
16255 type v; \
16256 \
16257 MEMCOPY(&v, p, sizeof(v)); \
16258 return v; \
16259 } \
16260 \
16261 static forceinline void \
16262 store_##type##_unaligned(type v, void *p) \
16263 { \
16264 MEMCOPY(p, &v, sizeof(v)); \
16265 }
16266  
16267 DEFINE_UNALIGNED_TYPE(u16)
16268 DEFINE_UNALIGNED_TYPE(u32)
16269 DEFINE_UNALIGNED_TYPE(u64)
16270 DEFINE_UNALIGNED_TYPE(machine_word_t)
16271  
16272 #undef MEMCOPY
16273  
16274 #define load_word_unaligned load_machine_word_t_unaligned
16275 #define store_word_unaligned store_machine_word_t_unaligned
16276  
16277  
16278  
16279 static forceinline u16
16280 get_unaligned_le16(const u8 *p)
16281 {
16282 if (UNALIGNED_ACCESS_IS_FAST)
16283 return le16_bswap(load_u16_unaligned(p));
16284 else
16285 return ((u16)p[1] << 8) | p[0];
16286 }
16287  
16288 static forceinline u16
16289 get_unaligned_be16(const u8 *p)
16290 {
16291 if (UNALIGNED_ACCESS_IS_FAST)
16292 return be16_bswap(load_u16_unaligned(p));
16293 else
16294 return ((u16)p[0] << 8) | p[1];
16295 }
16296  
16297 static forceinline u32
16298 get_unaligned_le32(const u8 *p)
16299 {
16300 if (UNALIGNED_ACCESS_IS_FAST)
16301 return le32_bswap(load_u32_unaligned(p));
16302 else
16303 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
16304 ((u32)p[1] << 8) | p[0];
16305 }
16306  
16307 static forceinline u32
16308 get_unaligned_be32(const u8 *p)
16309 {
16310 if (UNALIGNED_ACCESS_IS_FAST)
16311 return be32_bswap(load_u32_unaligned(p));
16312 else
16313 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
16314 ((u32)p[2] << 8) | p[3];
16315 }
16316  
16317 static forceinline u64
16318 get_unaligned_le64(const u8 *p)
16319 {
16320 if (UNALIGNED_ACCESS_IS_FAST)
16321 return le64_bswap(load_u64_unaligned(p));
16322 else
16323 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
16324 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
16325 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
16326 ((u64)p[1] << 8) | p[0];
16327 }
16328  
16329 static forceinline machine_word_t
16330 get_unaligned_leword(const u8 *p)
16331 {
16332 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
16333 if (WORDBITS == 32)
16334 return get_unaligned_le32(p);
16335 else
16336 return get_unaligned_le64(p);
16337 }
16338  
16339  
16340  
16341 static forceinline void
16342 put_unaligned_le16(u16 v, u8 *p)
16343 {
16344 if (UNALIGNED_ACCESS_IS_FAST) {
16345 store_u16_unaligned(le16_bswap(v), p);
16346 } else {
16347 p[0] = (u8)(v >> 0);
16348 p[1] = (u8)(v >> 8);
16349 }
16350 }
16351  
16352 static forceinline void
16353 put_unaligned_be16(u16 v, u8 *p)
16354 {
16355 if (UNALIGNED_ACCESS_IS_FAST) {
16356 store_u16_unaligned(be16_bswap(v), p);
16357 } else {
16358 p[0] = (u8)(v >> 8);
16359 p[1] = (u8)(v >> 0);
16360 }
16361 }
16362  
16363 static forceinline void
16364 put_unaligned_le32(u32 v, u8 *p)
16365 {
16366 if (UNALIGNED_ACCESS_IS_FAST) {
16367 store_u32_unaligned(le32_bswap(v), p);
16368 } else {
16369 p[0] = (u8)(v >> 0);
16370 p[1] = (u8)(v >> 8);
16371 p[2] = (u8)(v >> 16);
16372 p[3] = (u8)(v >> 24);
16373 }
16374 }
16375  
16376 static forceinline void
16377 put_unaligned_be32(u32 v, u8 *p)
16378 {
16379 if (UNALIGNED_ACCESS_IS_FAST) {
16380 store_u32_unaligned(be32_bswap(v), p);
16381 } else {
16382 p[0] = (u8)(v >> 24);
16383 p[1] = (u8)(v >> 16);
16384 p[2] = (u8)(v >> 8);
16385 p[3] = (u8)(v >> 0);
16386 }
16387 }
16388  
16389 static forceinline void
16390 put_unaligned_le64(u64 v, u8 *p)
16391 {
16392 if (UNALIGNED_ACCESS_IS_FAST) {
16393 store_u64_unaligned(le64_bswap(v), p);
16394 } else {
16395 p[0] = (u8)(v >> 0);
16396 p[1] = (u8)(v >> 8);
16397 p[2] = (u8)(v >> 16);
16398 p[3] = (u8)(v >> 24);
16399 p[4] = (u8)(v >> 32);
16400 p[5] = (u8)(v >> 40);
16401 p[6] = (u8)(v >> 48);
16402 p[7] = (u8)(v >> 56);
16403 }
16404 }
16405  
16406 static forceinline void
16407 put_unaligned_leword(machine_word_t v, u8 *p)
16408 {
16409 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
16410 if (WORDBITS == 32)
16411 put_unaligned_le32(v, p);
16412 else
16413 put_unaligned_le64(v, p);
16414 }
16415  
16416  
16417  
16418  
16419  
16420  
16421  
16422 static forceinline unsigned
16423 bsr32(u32 v)
16424 {
16425 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
16426 return 31 - __builtin_clz(v);
16427 #elif defined(_MSC_VER)
16428 unsigned long i;
16429  
16430 _BitScanReverse(&i, v);
16431 return i;
16432 #else
16433 unsigned i = 0;
16434  
16435 while ((v >>= 1) != 0)
16436 i++;
16437 return i;
16438 #endif
16439 }
16440  
16441 static forceinline unsigned
16442 bsr64(u64 v)
16443 {
16444 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
16445 return 63 - __builtin_clzll(v);
16446 #elif defined(_MSC_VER) && defined(_WIN64)
16447 unsigned long i;
16448  
16449 _BitScanReverse64(&i, v);
16450 return i;
16451 #else
16452 unsigned i = 0;
16453  
16454 while ((v >>= 1) != 0)
16455 i++;
16456 return i;
16457 #endif
16458 }
16459  
16460 static forceinline unsigned
16461 bsrw(machine_word_t v)
16462 {
16463 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
16464 if (WORDBITS == 32)
16465 return bsr32(v);
16466 else
16467 return bsr64(v);
16468 }
16469  
16470  
16471  
16472 static forceinline unsigned
16473 bsf32(u32 v)
16474 {
16475 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
16476 return __builtin_ctz(v);
16477 #elif defined(_MSC_VER)
16478 unsigned long i;
16479  
16480 _BitScanForward(&i, v);
16481 return i;
16482 #else
16483 unsigned i = 0;
16484  
16485 for (; (v & 1) == 0; v >>= 1)
16486 i++;
16487 return i;
16488 #endif
16489 }
16490  
16491 static forceinline unsigned
16492 bsf64(u64 v)
16493 {
16494 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
16495 return __builtin_ctzll(v);
16496 #elif defined(_MSC_VER) && defined(_WIN64)
16497 unsigned long i;
16498  
16499 _BitScanForward64(&i, v);
16500 return i;
16501 #else
16502 unsigned i = 0;
16503  
16504 for (; (v & 1) == 0; v >>= 1)
16505 i++;
16506 return i;
16507 #endif
16508 }
16509  
16510 static forceinline unsigned
16511 bsfw(machine_word_t v)
16512 {
16513 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
16514 if (WORDBITS == 32)
16515 return bsf32(v);
16516 else
16517 return bsf64(v);
16518 }
16519  
16520  
16521 #undef rbit32
16522 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
16523 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
16524 static forceinline u32
16525 rbit32(u32 v)
16526 {
16527 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
16528 return v;
16529 }
16530 #define rbit32 rbit32
16531 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
16532 static forceinline u32
16533 rbit32(u32 v)
16534 {
16535 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
16536 return v;
16537 }
16538 #define rbit32 rbit32
16539 #endif
16540  
16541 #endif
16542  
16543  
16544 typedef void *(*malloc_func_t)(size_t);
16545 typedef void (*free_func_t)(void *);
16546  
16547 extern malloc_func_t libdeflate_default_malloc_func;
16548 extern free_func_t libdeflate_default_free_func;
16549  
16550 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
16551 size_t alignment, size_t size);
16552 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
16553  
16554 #ifdef FREESTANDING
16555  
16556 void *memset(void *s, int c, size_t n);
16557 #define memset(s, c, n) __builtin_memset((s), (c), (n))
16558  
16559 void *memcpy(void *dest, const void *src, size_t n);
16560 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
16561  
16562 void *memmove(void *dest, const void *src, size_t n);
16563 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
16564  
16565 int memcmp(const void *s1, const void *s2, size_t n);
16566 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
16567  
16568 #undef LIBDEFLATE_ENABLE_ASSERTIONS
16569 #else
16570 # include <string.h>
16571
16572 # ifdef __clang_analyzer__
16573 # define LIBDEFLATE_ENABLE_ASSERTIONS
16574 # endif
16575 #endif
16576  
16577  
16578 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
16579 NORETURN void
16580 libdeflate_assertion_failed(const char *expr, const char *file, int line);
16581 #define ASSERT(expr) { if (unlikely(!(expr))) \
16582 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
16583 #else
16584 #define ASSERT(expr) (void)(expr)
16585 #endif
16586  
16587 #define CONCAT_IMPL(a, b) a##b
16588 #define CONCAT(a, b) CONCAT_IMPL(a, b)
16589 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
16590  
16591 #endif
16592  
16593  
16594 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
16595  
16596 #define X86_CPU_FEATURE_SSE2 (1 << 0)
16597 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
16598 #define X86_CPU_FEATURE_AVX (1 << 2)
16599 #define X86_CPU_FEATURE_AVX2 (1 << 3)
16600 #define X86_CPU_FEATURE_BMI2 (1 << 4)
16601  
16602 #define X86_CPU_FEATURE_ZMM (1 << 5)
16603 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
16604 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
16605 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
16606 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
16607 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
16608  
16609 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
16610  
16611 # define X86_CPU_FEATURES_KNOWN (1U << 31)
16612 extern volatile u32 libdeflate_x86_cpu_features;
16613  
16614 void libdeflate_init_x86_cpu_features(void);
16615  
16616 static inline u32 get_x86_cpu_features(void)
16617 {
16618 if (libdeflate_x86_cpu_features == 0)
16619 libdeflate_init_x86_cpu_features();
16620 return libdeflate_x86_cpu_features;
16621 }
16622  
16623 # include <immintrin.h>
16624 # if defined(_MSC_VER) && defined(__clang__)
16625 # include <tmmintrin.h>
16626 # include <smmintrin.h>
16627 # include <wmmintrin.h>
16628 # include <avxintrin.h>
16629 # include <avx2intrin.h>
16630 # include <avx512fintrin.h>
16631 # include <avx512bwintrin.h>
16632 # include <avx512vlintrin.h>
16633 # if __has_include(<avx512vlbwintrin.h>)
16634 # include <avx512vlbwintrin.h>
16635 # endif
16636 # if __has_include(<vpclmulqdqintrin.h>)
16637 # include <vpclmulqdqintrin.h>
16638 # endif
16639 # if __has_include(<avx512vnniintrin.h>)
16640 # include <avx512vnniintrin.h>
16641 # endif
16642 # if __has_include(<avx512vlvnniintrin.h>)
16643 # include <avx512vlvnniintrin.h>
16644 # endif
16645 # if __has_include(<avxvnniintrin.h>)
16646 # include <avxvnniintrin.h>
16647 # endif
16648 # endif
16649 #else
16650 static inline u32 get_x86_cpu_features(void) { return 0; }
16651 #endif
16652  
16653 #if defined(__SSE2__) || \
16654 (defined(_MSC_VER) && \
16655 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
16656 # define HAVE_SSE2(features) 1
16657 # define HAVE_SSE2_NATIVE 1
16658 #else
16659 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
16660 # define HAVE_SSE2_NATIVE 0
16661 #endif
16662  
16663 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
16664 (defined(_MSC_VER) && defined(__AVX2__))
16665 # define HAVE_PCLMULQDQ(features) 1
16666 #else
16667 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
16668 #endif
16669  
16670 #ifdef __AVX__
16671 # define HAVE_AVX(features) 1
16672 #else
16673 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
16674 #endif
16675  
16676 #ifdef __AVX2__
16677 # define HAVE_AVX2(features) 1
16678 #else
16679 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
16680 #endif
16681  
16682 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
16683 # define HAVE_BMI2(features) 1
16684 # define HAVE_BMI2_NATIVE 1
16685 #else
16686 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
16687 # define HAVE_BMI2_NATIVE 0
16688 #endif
16689  
16690 #ifdef __AVX512BW__
16691 # define HAVE_AVX512BW(features) 1
16692 #else
16693 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
16694 #endif
16695  
16696 #ifdef __AVX512VL__
16697 # define HAVE_AVX512VL(features) 1
16698 #else
16699 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
16700 #endif
16701  
16702 #ifdef __VPCLMULQDQ__
16703 # define HAVE_VPCLMULQDQ(features) 1
16704 #else
16705 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
16706 #endif
16707  
16708 #ifdef __AVX512VNNI__
16709 # define HAVE_AVX512VNNI(features) 1
16710 #else
16711 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
16712 #endif
16713  
16714 #ifdef __AVXVNNI__
16715 # define HAVE_AVXVNNI(features) 1
16716 #else
16717 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
16718 #endif
16719  
16720 #endif
16721  
16722 #endif
16723  
16724  
16725 #ifdef __AVX2__
16726 static forceinline void
16727 matchfinder_init_avx2(mf_pos_t *data, size_t size)
16728 {
16729 __m256i *p = (__m256i *)data;
16730 __m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
16731  
16732 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
16733 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
16734 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
16735  
16736 do {
16737 p[0] = v;
16738 p[1] = v;
16739 p[2] = v;
16740 p[3] = v;
16741 p += 4;
16742 size -= 4 * sizeof(*p);
16743 } while (size != 0);
16744 }
16745 #define matchfinder_init matchfinder_init_avx2
16746  
16747 static forceinline void
16748 matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
16749 {
16750 __m256i *p = (__m256i *)data;
16751 __m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
16752  
16753 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
16754 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
16755 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
16756  
16757 do {
16758
16759 p[0] = _mm256_adds_epi16(p[0], v);
16760 p[1] = _mm256_adds_epi16(p[1], v);
16761 p[2] = _mm256_adds_epi16(p[2], v);
16762 p[3] = _mm256_adds_epi16(p[3], v);
16763 p += 4;
16764 size -= 4 * sizeof(*p);
16765 } while (size != 0);
16766 }
16767 #define matchfinder_rebase matchfinder_rebase_avx2
16768  
16769 #elif HAVE_SSE2_NATIVE
16770 static forceinline void
16771 matchfinder_init_sse2(mf_pos_t *data, size_t size)
16772 {
16773 37 __m128i *p = (__m128i *)data;
16774 37 __m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
16775  
16776 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
16777 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
16778 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
16779  
16780 do {
16781 138240 p[0] = v;
16782 138240 p[1] = v;
16783 138240 p[2] = v;
16784 138240 p[3] = v;
16785 138240 p += 4;
16786 138240 size -= 4 * sizeof(*p);
16787 138240 ▶ } while (size != 0);
Branch: 5 branches
BranchTrueFalseTotal
} while (size != 0); 55287 9 55296
} while (size != 0); 18426 6 18432
} while (size != 0); 30710 10 30720
} while (size != 0); 27639 9 27648
} while (size != 0); 6141 3 6144
16788 37 }
16789 #define matchfinder_init matchfinder_init_sse2
16790  
16791 static forceinline void
16792 matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
16793 {
16794 0 __m128i *p = (__m128i *)data;
16795 0 __m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
16796  
16797 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
16798 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
16799 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
16800  
16801 do {
16802
16803 0 p[0] = _mm_adds_epi16(p[0], v);
16804 0 p[1] = _mm_adds_epi16(p[1], v);
16805 0 p[2] = _mm_adds_epi16(p[2], v);
16806 0 p[3] = _mm_adds_epi16(p[3], v);
16807 0 p += 4;
16808 0 size -= 4 * sizeof(*p);
16809 0 ▶ } while (size != 0);
Branch: 18 branches
BranchTrueFalseTotal
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
} while (size != 0); 0 0 0
16810 0 }
16811 #define matchfinder_rebase matchfinder_rebase_sse2
16812 #endif
16813  
16814 #endif
16815  
16816 # endif
16817 #else
16818 # define MATCHFINDER_ALIGNED
16819 #endif
16820  
16821  
16822 #ifndef matchfinder_init
16823 static forceinline void
16824 matchfinder_init(mf_pos_t *data, size_t size)
16825 {
16826 size_t num_entries = size / sizeof(*data);
16827 size_t i;
16828  
16829 for (i = 0; i < num_entries; i++)
16830 data[i] = MATCHFINDER_INITVAL;
16831 }
16832 #endif
16833  
16834  
16835 #ifndef matchfinder_rebase
16836 static forceinline void
16837 matchfinder_rebase(mf_pos_t *data, size_t size)
16838 {
16839 size_t num_entries = size / sizeof(*data);
16840 size_t i;
16841  
16842 if (MATCHFINDER_WINDOW_SIZE == 32768) {
16843
16844 for (i = 0; i < num_entries; i++)
16845 data[i] = 0x8000 | (data[i] & ~(data[i] >> 15));
16846 } else {
16847 for (i = 0; i < num_entries; i++) {
16848 if (data[i] >= 0)
16849 data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
16850 else
16851 data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
16852 }
16853 }
16854 }
16855 #endif
16856  
16857  
16858 static forceinline u32
16859 lz_hash(u32 seq, unsigned num_bits)
16860 {
16861 167673 return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
16862 }
16863  
16864  
16865 static forceinline u32
16866 lz_extend(const u8 * const strptr, const u8 * const matchptr,
16867 const u32 start_len, const u32 max_len)
16868 {
16869 21 u32 len = start_len;
16870 machine_word_t v_word;
16871  
16872 if (UNALIGNED_ACCESS_IS_FAST) {
16873  
16874 17924 ▶ if (likely(max_len - len >= 4 * WORDBYTES)) {
Branch: 19 branches
BranchTrueFalseTotal
if (likely(max_len - len >= 4 * WORDBYTES)) { 63 0 63
if (likely(max_len - len >= 4 * WORDBYTES)) { 14787 252 15039
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 42 0 42
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 1899 3 1902
if (likely(max_len - len >= 4 * WORDBYTES)) { 650 0 650
if (likely(max_len - len >= 4 * WORDBYTES)) { 61 1 62
if (likely(max_len - len >= 4 * WORDBYTES)) { 82 0 82
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 63 0 63
if (likely(max_len - len >= 4 * WORDBYTES)) { 21 0 21
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
if (likely(max_len - len >= 4 * WORDBYTES)) { 0 0 0
16875  
16876 #define COMPARE_WORD_STEP \
16877 v_word = load_word_unaligned(&matchptr[len]) ^ \
16878 load_word_unaligned(&strptr[len]); \
16879 if (v_word != 0) \
16880 goto word_differs; \
16881 len += WORDBYTES; \
16882  
16883 53004 ▶ COMPARE_WORD_STEP
Branch: 19 branches
BranchTrueFalseTotal
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 14787 14787
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 42 42
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 1259 640 1899
COMPARE_WORD_STEP 253 397 650
COMPARE_WORD_STEP 53 8 61
COMPARE_WORD_STEP 26 56 82
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 21 21
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
16884 48231 ▶ COMPARE_WORD_STEP
Branch: 19 branches
BranchTrueFalseTotal
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 14787 14787
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 42 42
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 366 274 640
COMPARE_WORD_STEP 197 200 397
COMPARE_WORD_STEP 2 6 8
COMPARE_WORD_STEP 16 40 56
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 21 21
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
16885 46488 ▶ COMPARE_WORD_STEP
Branch: 19 branches
BranchTrueFalseTotal
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 14787 14787
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 42 42
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 185 89 274
COMPARE_WORD_STEP 96 104 200
COMPARE_WORD_STEP 4 2 6
COMPARE_WORD_STEP 25 15 40
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 21 21
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
16886 45558 ▶ COMPARE_WORD_STEP
Branch: 19 branches
BranchTrueFalseTotal
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 14787 14787
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 42 42
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 25 64 89
COMPARE_WORD_STEP 29 75 104
COMPARE_WORD_STEP 2 0 2
COMPARE_WORD_STEP 4 11 15
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 63 63
COMPARE_WORD_STEP 0 21 21
COMPARE_WORD_STEP 0 0 0
COMPARE_WORD_STEP 0 0 0
16887 #undef COMPARE_WORD_STEP
16888 }
16889  
16890 241392 ▶ while (len + WORDBYTES <= max_len) {
Branch: 19 branches
BranchTrueFalseTotal
while (len + WORDBYTES <= max_len) { 1620 63 1683
while (len + WORDBYTES <= max_len) { 219525 15039 234564
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 1044 42 1086
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 192 1 193
while (len + WORDBYTES <= max_len) { 1593 63 1656
while (len + WORDBYTES <= max_len) { 2 0 2
while (len + WORDBYTES <= max_len) { 36 0 36
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 1566 63 1629
while (len + WORDBYTES <= max_len) { 522 21 543
while (len + WORDBYTES <= max_len) { 0 0 0
while (len + WORDBYTES <= max_len) { 0 0 0
16891 226100 v_word = load_word_unaligned(&matchptr[len]) ^
16892 226100 load_word_unaligned(&strptr[len]);
16893 226100 ▶ if (v_word != 0)
Branch: 19 branches
BranchTrueFalseTotal
if (v_word != 0) 0 1620 1620
if (v_word != 0) 0 219525 219525
if (v_word != 0) 0 0 0
if (v_word != 0) 0 1044 1044
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
if (v_word != 0) 66 126 192
if (v_word != 0) 12 1581 1593
if (v_word != 0) 1 1 2
if (v_word != 0) 11 25 36
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
if (v_word != 0) 0 1566 1566
if (v_word != 0) 0 522 522
if (v_word != 0) 0 0 0
if (v_word != 0) 0 0 0
16894 90 goto word_differs;
16895 226010 len += WORDBYTES;
16896 }
16897 }
16898  
16899 58096 ▶ while (len < max_len && matchptr[len] == strptr[len])
Branch: 38 branches
BranchTrueFalseTotal
while (len < max_len && matchptr[len] == strptr[len]) 99 63 162
while (len < max_len && matchptr[len] == strptr[len]) 99 0 99
while (len < max_len && matchptr[len] == strptr[len]) 41673 15039 56712
while (len < max_len && matchptr[len] == strptr[len]) 41673 0 41673
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 228 42 270
while (len < max_len && matchptr[len] == strptr[len]) 228 0 228
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 6 1 7
while (len < max_len && matchptr[len] == strptr[len]) 6 0 6
while (len < max_len && matchptr[len] == strptr[len]) 342 63 405
while (len < max_len && matchptr[len] == strptr[len]) 342 0 342
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 342 63 405
while (len < max_len && matchptr[len] == strptr[len]) 342 0 342
while (len < max_len && matchptr[len] == strptr[len]) 114 21 135
while (len < max_len && matchptr[len] == strptr[len]) 114 0 114
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
while (len < max_len && matchptr[len] == strptr[len]) 0 0 0
16900 42804 len++;
16901 15292 return len;
16902  
16903 2632 word_differs:
16904 if (CPU_IS_LITTLE_ENDIAN())
16905 2632 len += (bsfw(v_word) >> 3);
16906 else
16907 len += (WORDBITS - 1 - bsrw(v_word)) >> 3;
16908 2632 return len;
16909 }
16910  
16911 #endif
16912  
16913  
16914 #define HC_MATCHFINDER_HASH3_ORDER 15
16915 #define HC_MATCHFINDER_HASH4_ORDER 16
16916  
16917 #define HC_MATCHFINDER_TOTAL_HASH_SIZE \
16918 (((1UL << HC_MATCHFINDER_HASH3_ORDER) + \
16919 (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
16920  
16921 struct MATCHFINDER_ALIGNED hc_matchfinder {
16922  
16923
16924 mf_pos_t hash3_tab[1UL << HC_MATCHFINDER_HASH3_ORDER];
16925  
16926
16927 mf_pos_t hash4_tab[1UL << HC_MATCHFINDER_HASH4_ORDER];
16928  
16929
16930 mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE];
16931 };
16932  
16933  
16934 static forceinline void
16935 hc_matchfinder_init(struct hc_matchfinder *mf)
16936 {
16937 STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE %
16938 MATCHFINDER_SIZE_ALIGNMENT == 0);
16939  
16940 matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE);
16941 25 }
16942  
16943 static forceinline void
16944 hc_matchfinder_slide_window(struct hc_matchfinder *mf)
16945 {
16946 STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
16947  
16948 matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
16949 0 }
16950  
16951  
16952 static forceinline u32
16953 hc_matchfinder_longest_match(struct hc_matchfinder * const mf,
16954 const u8 ** const in_base_p,
16955 const u8 * const in_next,
16956 u32 best_len,
16957 const u32 max_len,
16958 const u32 nice_len,
16959 const u32 max_search_depth,
16960 u32 * const next_hashes,
16961 u32 * const offset_ret)
16962 {
16963 7046 u32 depth_remaining = max_search_depth;
16964 7046 const u8 *best_matchptr = in_next;
16965 mf_pos_t cur_node3, cur_node4;
16966 u32 hash3, hash4;
16967 u32 next_hashseq;
16968 u32 seq4;
16969 const u8 *matchptr;
16970 u32 len;
16971 7046 u32 cur_pos = in_next - *in_base_p;
16972 const u8 *in_base;
16973 mf_pos_t cutoff;
16974  
16975 7046 if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
16976 hc_matchfinder_slide_window(mf);
16977 0 *in_base_p += MATCHFINDER_WINDOW_SIZE;
16978 0 cur_pos = 0;
16979 }
16980  
16981 7046 in_base = *in_base_p;
16982 7046 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
16983  
16984 7046 ▶ if (unlikely(max_len < 5))
Branch: 7 branches
BranchTrueFalseTotal
if (unlikely(max_len < 5)) 0 150 150
if (unlikely(max_len < 5)) 0 0 0
if (unlikely(max_len < 5)) 0 0 0
if (unlikely(max_len < 5)) 0 4679 4679
if (unlikely(max_len < 5)) 0 1992 1992
if (unlikely(max_len < 5)) 0 0 0
if (unlikely(max_len < 5)) 0 225 225
16985 0 goto out;
16986  
16987
16988 7046 hash3 = next_hashes[0];
16989 7046 hash4 = next_hashes[1];
16990  
16991
16992 7046 cur_node3 = mf->hash3_tab[hash3];
16993 7046 cur_node4 = mf->hash4_tab[hash4];
16994  
16995
16996 7046 mf->hash3_tab[hash3] = cur_pos;
16997  
16998
16999 7046 mf->hash4_tab[hash4] = cur_pos;
17000 7046 mf->next_tab[cur_pos] = cur_node4;
17001  
17002
17003 7046 next_hashseq = get_unaligned_le32(in_next + 1);
17004 7046 next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
17005 7046 next_hashes[1] = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
17006 7046 prefetchw(&mf->hash3_tab[next_hashes[0]]);
17007 7046 prefetchw(&mf->hash4_tab[next_hashes[1]]);
17008  
17009 7046 if (best_len < 4) {
17010  
17011
17012  
17013 4747 ▶ if (cur_node3 <= cutoff)
Branch: 7 branches
BranchTrueFalseTotal
if (cur_node3 <= cutoff) 0 0 0
if (cur_node3 <= cutoff) 0 0 0
if (cur_node3 <= cutoff) 0 0 0
if (cur_node3 <= cutoff) 1888 2566 4454
if (cur_node3 <= cutoff) 0 293 293
if (cur_node3 <= cutoff) 0 0 0
if (cur_node3 <= cutoff) 0 0 0
17014 1888 goto out;
17015  
17016 2859 seq4 = load_u32_unaligned(in_next);
17017  
17018 2859 ▶ if (best_len < 3) {
Branch: 7 branches
BranchTrueFalseTotal
if (best_len < 3) { 0 0 0
if (best_len < 3) { 0 0 0
if (best_len < 3) { 0 0 0
if (best_len < 3) { 0 2566 2566
if (best_len < 3) { 0 293 293
if (best_len < 3) { 0 0 0
if (best_len < 3) { 0 0 0
17019 0 matchptr = &in_base[cur_node3];
17020 0 ▶ if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) {
Branch: 7 branches
BranchTrueFalseTotal
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { 0 0 0
17021 0 best_len = 3;
17022 0 best_matchptr = matchptr;
17023 }
17024 }
17025  
17026
17027  
17028 2859 ▶ if (cur_node4 <= cutoff)
Branch: 7 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff) 0 0 0
if (cur_node4 <= cutoff) 0 0 0
if (cur_node4 <= cutoff) 0 0 0
if (cur_node4 <= cutoff) 631 1935 2566
if (cur_node4 <= cutoff) 224 69 293
if (cur_node4 <= cutoff) 0 0 0
if (cur_node4 <= cutoff) 0 0 0
17029 855 goto out;
17030  
17031 for (;;) {
17032
17033 2180 matchptr = &in_base[cur_node4];
17034  
17035 2180 ▶ if (load_u32_unaligned(matchptr) == seq4)
Branch: 7 branches
BranchTrueFalseTotal
if (load_u32_unaligned(matchptr) == seq4) 0 0 0
if (load_u32_unaligned(matchptr) == seq4) 0 0 0
if (load_u32_unaligned(matchptr) == seq4) 0 0 0
if (load_u32_unaligned(matchptr) == seq4) 1902 186 2088
if (load_u32_unaligned(matchptr) == seq4) 62 30 92
if (load_u32_unaligned(matchptr) == seq4) 0 0 0
if (load_u32_unaligned(matchptr) == seq4) 0 0 0
17036 1964 break;
17037  
17038
17039 216 cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
17040 216 ▶ if (cur_node4 <= cutoff || !--depth_remaining)
Branch: 14 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 155 31 186
if (cur_node4 <= cutoff || !--depth_remaining) 2 153 155
if (cur_node4 <= cutoff || !--depth_remaining) 23 7 30
if (cur_node4 <= cutoff || !--depth_remaining) 0 23 23
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
17041 40 goto out;
17042 }
17043  
17044
17045 1964 best_matchptr = matchptr;
17046 1964 best_len = lz_extend(in_next, best_matchptr, 4, max_len);
17047 1964 ▶ if (best_len >= nice_len)
Branch: 7 branches
BranchTrueFalseTotal
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 1 1901 1902
if (best_len >= nice_len) 0 62 62
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 0 0 0
17048 1 goto out;
17049 1963 cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
17050 1963 ▶ if (cur_node4 <= cutoff || !--depth_remaining)
Branch: 14 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 1525 376 1901
if (cur_node4 <= cutoff || !--depth_remaining) 0 1525 1525
if (cur_node4 <= cutoff || !--depth_remaining) 38 24 62
if (cur_node4 <= cutoff || !--depth_remaining) 0 38 38
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
17051 400 goto out;
17052 } else {
17053 2299 ▶ if (cur_node4 <= cutoff || best_len >= nice_len)
Branch: 14 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff || best_len >= nice_len) 42 108 150
if (cur_node4 <= cutoff || best_len >= nice_len) 42 0 42
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 63 162 225
if (cur_node4 <= cutoff || best_len >= nice_len) 63 0 63
if (cur_node4 <= cutoff || best_len >= nice_len) 1699 0 1699
if (cur_node4 <= cutoff || best_len >= nice_len) 1699 0 1699
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 0 0 0
if (cur_node4 <= cutoff || best_len >= nice_len) 63 162 225
if (cur_node4 <= cutoff || best_len >= nice_len) 63 0 63
17054 432 goto out;
17055 }
17056  
17057
17058  
17059 604 for (;;) {
17060 for (;;) {
17061 36147 matchptr = &in_base[cur_node4];
17062  
17063
17064 #if UNALIGNED_ACCESS_IS_FAST
17065 36147 if ((load_u32_unaligned(matchptr + best_len - 3) ==
17066 72294 ▶ load_u32_unaligned(in_next + best_len - 3)) &&
Branch: 7 branches
BranchTrueFalseTotal
load_u32_unaligned(in_next + best_len - 3)) && 42 0 42
load_u32_unaligned(in_next + best_len - 3)) && 0 0 0
load_u32_unaligned(in_next + best_len - 3)) && 0 0 0
load_u32_unaligned(in_next + best_len - 3)) && 650 20012 20662
load_u32_unaligned(in_next + best_len - 3)) && 82 15298 15380
load_u32_unaligned(in_next + best_len - 3)) && 0 0 0
load_u32_unaligned(in_next + best_len - 3)) && 63 0 63
17067 837 (load_u32_unaligned(matchptr) ==
17068 837 ▶ load_u32_unaligned(in_next)))
Branch: 7 branches
BranchTrueFalseTotal
load_u32_unaligned(in_next))) 42 0 42
load_u32_unaligned(in_next))) 0 0 0
load_u32_unaligned(in_next))) 0 0 0
load_u32_unaligned(in_next))) 650 0 650
load_u32_unaligned(in_next))) 82 0 82
load_u32_unaligned(in_next))) 0 0 0
load_u32_unaligned(in_next))) 63 0 63
17069 #else
17070 if (matchptr[best_len] == in_next[best_len])
17071 #endif
17072 837 break;
17073  
17074
17075 35310 cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
17076 35310 ▶ if (cur_node4 <= cutoff || !--depth_remaining)
Branch: 14 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 18893 1119 20012
if (cur_node4 <= cutoff || !--depth_remaining) 348 18545 18893
if (cur_node4 <= cutoff || !--depth_remaining) 14154 1144 15298
if (cur_node4 <= cutoff || !--depth_remaining) 586 13568 14154
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
17077 3197 goto out;
17078 }
17079  
17080 #if UNALIGNED_ACCESS_IS_FAST
17081 837 len = 4;
17082 #else
17083 len = 0;
17084 #endif
17085 837 len = lz_extend(in_next, matchptr, len, max_len);
17086 837 ▶ if (len > best_len) {
Branch: 7 branches
BranchTrueFalseTotal
if (len > best_len) { 42 0 42
if (len > best_len) { 0 0 0
if (len > best_len) { 0 0 0
if (len > best_len) { 648 2 650
if (len > best_len) { 78 4 82
if (len > best_len) { 0 0 0
if (len > best_len) { 63 0 63
17087
17088 831 best_len = len;
17089 831 best_matchptr = matchptr;
17090 831 ▶ if (best_len >= nice_len)
Branch: 7 branches
BranchTrueFalseTotal
if (best_len >= nice_len) 42 0 42
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 63 585 648
if (best_len >= nice_len) 0 78 78
if (best_len >= nice_len) 0 0 0
if (best_len >= nice_len) 63 0 63
17091 168 goto out;
17092 }
17093  
17094
17095 669 cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
17096 669 ▶ if (cur_node4 <= cutoff || !--depth_remaining)
Branch: 14 branches
BranchTrueFalseTotal
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 534 53 587
if (cur_node4 <= cutoff || !--depth_remaining) 5 529 534
if (cur_node4 <= cutoff || !--depth_remaining) 75 7 82
if (cur_node4 <= cutoff || !--depth_remaining) 0 75 75
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
if (cur_node4 <= cutoff || !--depth_remaining) 0 0 0
17097 65 goto out;
17098 }
17099 7046 out:
17100 7046 *offset_ret = in_next - best_matchptr;
17101 7046 return best_len;
17102 }
17103  
17104  
17105 static forceinline void
17106 hc_matchfinder_skip_bytes(struct hc_matchfinder * const mf,
17107 const u8 ** const in_base_p,
17108 const u8 *in_next,
17109 const u8 * const in_end,
17110 const u32 count,
17111 u32 * const next_hashes)
17112 {
17113 u32 cur_pos;
17114 u32 hash3, hash4;
17115 u32 next_hashseq;
17116 2070 u32 remaining = count;
17117  
17118 2070 if (unlikely(count + 5 > in_end - in_next))
17119 25 return;
17120  
17121 2045 cur_pos = in_next - *in_base_p;
17122 2045 hash3 = next_hashes[0];
17123 2045 hash4 = next_hashes[1];
17124 do {
17125 59176 ▶ if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
Branch: 7 branches
BranchTrueFalseTotal
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 9252 9252
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 0 0
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 0 0
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 13878 13878
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 0 0
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 22168 22168
if (cur_pos == MATCHFINDER_WINDOW_SIZE) { 0 13878 13878
17126 hc_matchfinder_slide_window(mf);
17127 0 *in_base_p += MATCHFINDER_WINDOW_SIZE;
17128 0 cur_pos = 0;
17129 }
17130 59176 mf->hash3_tab[hash3] = cur_pos;
17131 59176 mf->next_tab[cur_pos] = mf->hash4_tab[hash4];
17132 59176 mf->hash4_tab[hash4] = cur_pos;
17133  
17134 59176 next_hashseq = get_unaligned_le32(++in_next);
17135 118352 hash3 = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
17136 59176 hash4 = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
17137 59176 cur_pos++;
17138 59176 ▶ } while (--remaining);
Branch: 7 branches
BranchTrueFalseTotal
} while (--remaining); 9216 36 9252
} while (--remaining); 0 0 0
} while (--remaining); 0 0 0
} while (--remaining); 13824 54 13878
} while (--remaining); 0 0 0
} while (--remaining); 20267 1901 22168
} while (--remaining); 13824 54 13878
17139  
17140 2045 prefetchw(&mf->hash3_tab[hash3]);
17141 2045 prefetchw(&mf->hash4_tab[hash4]);
17142 2045 next_hashes[0] = hash3;
17143 2045 next_hashes[1] = hash4;
17144 }
17145  
17146 #endif
17147  
17148 /* #include "ht_matchfinder.h" */
17149  
17150  
17151 #ifndef LIB_HT_MATCHFINDER_H
17152 #define LIB_HT_MATCHFINDER_H
17153  
17154 /* #include "matchfinder_common.h" */
17155  
17156  
17157 #ifndef LIB_MATCHFINDER_COMMON_H
17158 #define LIB_MATCHFINDER_COMMON_H
17159  
17160 /* #include "lib_common.h" */
17161  
17162  
17163 #ifndef LIB_LIB_COMMON_H
17164 #define LIB_LIB_COMMON_H
17165  
17166 #ifdef LIBDEFLATE_H
17167
17168 # error "lib_common.h must always be included before libdeflate.h"
17169 #endif
17170  
17171 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
17172 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
17173 #elif defined(__GNUC__)
17174 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
17175 #else
17176 # define LIBDEFLATE_EXPORT_SYM
17177 #endif
17178  
17179  
17180 #if defined(__GNUC__) && defined(__i386__)
17181 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
17182 #else
17183 # define LIBDEFLATE_ALIGN_STACK
17184 #endif
17185  
17186 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
17187  
17188 /* #include "../common_defs.h" */
17189  
17190  
17191 #ifndef COMMON_DEFS_H
17192 #define COMMON_DEFS_H
17193  
17194 /* #include "libdeflate.h" */
17195  
17196  
17197 #ifndef LIBDEFLATE_H
17198 #define LIBDEFLATE_H
17199  
17200 #include <stddef.h>
17201 #include <stdint.h>
17202  
17203 #ifdef __cplusplus
17204 extern "C" {
17205 #endif
17206  
17207 #define LIBDEFLATE_VERSION_MAJOR 1
17208 #define LIBDEFLATE_VERSION_MINOR 26
17209 #define LIBDEFLATE_VERSION_STRING "1.26"
17210  
17211  
17212 #ifndef LIBDEFLATEAPI
17213 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
17214 # define LIBDEFLATEAPI __declspec(dllimport)
17215 # else
17216 # define LIBDEFLATEAPI
17217 # endif
17218 #endif
17219  
17220  
17221  
17222  
17223  
17224 struct libdeflate_compressor;
17225 struct libdeflate_options;
17226  
17227  
17228 LIBDEFLATEAPI struct libdeflate_compressor *
17229 libdeflate_alloc_compressor(int compression_level);
17230  
17231  
17232 LIBDEFLATEAPI struct libdeflate_compressor *
17233 libdeflate_alloc_compressor_ex(int compression_level,
17234 const struct libdeflate_options *options);
17235  
17236  
17237 LIBDEFLATEAPI size_t
17238 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
17239 const void *in, size_t in_nbytes,
17240 void *out, size_t out_nbytes_avail);
17241  
17242  
17243 LIBDEFLATEAPI size_t
17244 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
17245 size_t in_nbytes);
17246  
17247  
17248 LIBDEFLATEAPI size_t
17249 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
17250 const void *in, size_t in_nbytes,
17251 void *out, size_t out_nbytes_avail);
17252  
17253  
17254 LIBDEFLATEAPI size_t
17255 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
17256 size_t in_nbytes);
17257  
17258  
17259 LIBDEFLATEAPI size_t
17260 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
17261 const void *in, size_t in_nbytes,
17262 void *out, size_t out_nbytes_avail);
17263  
17264  
17265 LIBDEFLATEAPI size_t
17266 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
17267 size_t in_nbytes);
17268  
17269  
17270 LIBDEFLATEAPI void
17271 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
17272  
17273  
17274  
17275  
17276  
17277 struct libdeflate_decompressor;
17278 struct libdeflate_options;
17279  
17280  
17281 LIBDEFLATEAPI struct libdeflate_decompressor *
17282 libdeflate_alloc_decompressor(void);
17283  
17284  
17285 LIBDEFLATEAPI struct libdeflate_decompressor *
17286 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
17287  
17288  
17289 enum libdeflate_result {
17290
17291 LIBDEFLATE_SUCCESS = 0,
17292  
17293
17294 LIBDEFLATE_BAD_DATA = 1,
17295  
17296
17297 LIBDEFLATE_SHORT_OUTPUT = 2,
17298  
17299
17300 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
17301 };
17302  
17303  
17304 LIBDEFLATEAPI enum libdeflate_result
17305 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
17306 const void *in, size_t in_nbytes,
17307 void *out, size_t out_nbytes_avail,
17308 size_t *actual_out_nbytes_ret);
17309  
17310  
17311 LIBDEFLATEAPI enum libdeflate_result
17312 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
17313 const void *in, size_t in_nbytes,
17314 void *out, size_t out_nbytes_avail,
17315 size_t *actual_in_nbytes_ret,
17316 size_t *actual_out_nbytes_ret);
17317  
17318  
17319 LIBDEFLATEAPI enum libdeflate_result
17320 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
17321 const void *in, size_t in_nbytes,
17322 void *out, size_t out_nbytes_avail,
17323 size_t *actual_out_nbytes_ret);
17324  
17325  
17326 LIBDEFLATEAPI enum libdeflate_result
17327 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
17328 const void *in, size_t in_nbytes,
17329 void *out, size_t out_nbytes_avail,
17330 size_t *actual_in_nbytes_ret,
17331 size_t *actual_out_nbytes_ret);
17332  
17333  
17334 LIBDEFLATEAPI enum libdeflate_result
17335 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
17336 const void *in, size_t in_nbytes,
17337 void *out, size_t out_nbytes_avail,
17338 size_t *actual_out_nbytes_ret);
17339  
17340  
17341 LIBDEFLATEAPI enum libdeflate_result
17342 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
17343 const void *in, size_t in_nbytes,
17344 void *out, size_t out_nbytes_avail,
17345 size_t *actual_in_nbytes_ret,
17346 size_t *actual_out_nbytes_ret);
17347  
17348  
17349 LIBDEFLATEAPI void
17350 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
17351  
17352  
17353  
17354  
17355  
17356  
17357 LIBDEFLATEAPI uint32_t
17358 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
17359  
17360  
17361  
17362 LIBDEFLATEAPI uint32_t
17363 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
17364  
17365  
17366  
17367  
17368  
17369  
17370 LIBDEFLATEAPI void
17371 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
17372 void (*free_func)(void *));
17373  
17374  
17375 struct libdeflate_options {
17376  
17377
17378 size_t sizeof_options;
17379  
17380
17381 void *(*malloc_func)(size_t);
17382 void (*free_func)(void *);
17383 };
17384  
17385 #ifdef __cplusplus
17386 }
17387 #endif
17388  
17389 #endif
17390  
17391  
17392 #include <stdbool.h>
17393 #include <stddef.h>
17394 #include <stdint.h>
17395 #ifdef _MSC_VER
17396 # include <intrin.h>
17397 # include <stdlib.h>
17398
17399
17400 # pragma warning(disable : 4146)
17401
17402 # pragma warning(disable : 4018)
17403 # pragma warning(disable : 4244)
17404 # pragma warning(disable : 4267)
17405 # pragma warning(disable : 4310)
17406
17407 # pragma warning(disable : 4100)
17408 # pragma warning(disable : 4127)
17409 # pragma warning(disable : 4189)
17410 # pragma warning(disable : 4232)
17411 # pragma warning(disable : 4245)
17412 # pragma warning(disable : 4295)
17413 #endif
17414 #ifndef FREESTANDING
17415 # include <string.h>
17416 #endif
17417  
17418  
17419  
17420  
17421  
17422  
17423 #undef ARCH_X86_64
17424 #undef ARCH_X86_32
17425 #undef ARCH_ARM64
17426 #undef ARCH_ARM32
17427 #undef ARCH_RISCV
17428 #ifdef _MSC_VER
17429
17430 # if defined(_M_X64) && !defined(_M_ARM64EC)
17431 # define ARCH_X86_64
17432 # elif defined(_M_IX86)
17433 # define ARCH_X86_32
17434 # elif defined(_M_ARM64)
17435 # define ARCH_ARM64
17436 # elif defined(_M_ARM)
17437 # define ARCH_ARM32
17438 # endif
17439 #else
17440 # if defined(__x86_64__)
17441 # define ARCH_X86_64
17442 # elif defined(__i386__)
17443 # define ARCH_X86_32
17444 # elif defined(__aarch64__)
17445 # define ARCH_ARM64
17446 # elif defined(__arm__)
17447 # define ARCH_ARM32
17448 # elif defined(__riscv)
17449 # define ARCH_RISCV
17450 # endif
17451 #endif
17452  
17453  
17454  
17455  
17456  
17457  
17458 typedef uint8_t u8;
17459 typedef uint16_t u16;
17460 typedef uint32_t u32;
17461 typedef uint64_t u64;
17462 typedef int8_t s8;
17463 typedef int16_t s16;
17464 typedef int32_t s32;
17465 typedef int64_t s64;
17466  
17467  
17468 #ifdef _MSC_VER
17469 # ifdef _WIN64
17470 typedef long long ssize_t;
17471 # else
17472 typedef long ssize_t;
17473 # endif
17474 #endif
17475  
17476  
17477 typedef size_t machine_word_t;
17478  
17479  
17480 #define WORDBYTES ((int)sizeof(machine_word_t))
17481  
17482  
17483 #define WORDBITS (8 * WORDBYTES)
17484  
17485  
17486  
17487  
17488  
17489  
17490 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
17491 # define GCC_PREREQ(major, minor) \
17492 (__GNUC__ > (major) || \
17493 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
17494 # if !GCC_PREREQ(4, 9)
17495 # error "gcc versions older than 4.9 are no longer supported"
17496 # endif
17497 #else
17498 # define GCC_PREREQ(major, minor) 0
17499 #endif
17500 #ifdef __clang__
17501 # ifdef __apple_build_version__
17502 # define CLANG_PREREQ(major, minor, apple_version) \
17503 (__apple_build_version__ >= (apple_version))
17504 # else
17505 # define CLANG_PREREQ(major, minor, apple_version) \
17506 (__clang_major__ > (major) || \
17507 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
17508 # endif
17509 # if !CLANG_PREREQ(3, 9, 8000000)
17510 # error "clang versions older than 3.9 are no longer supported"
17511 # endif
17512 #else
17513 # define CLANG_PREREQ(major, minor, apple_version) 0
17514 #endif
17515 #ifdef _MSC_VER
17516 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
17517 # if !MSVC_PREREQ(1928)
17518 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
17519 # endif
17520 #else
17521 # define MSVC_PREREQ(version) 0
17522 #endif
17523  
17524  
17525 #ifndef __has_attribute
17526 # define __has_attribute(attribute) 0
17527 #endif
17528  
17529  
17530 #ifndef __has_builtin
17531 # define __has_builtin(builtin) 0
17532 #endif
17533  
17534  
17535 #ifdef _MSC_VER
17536 # define inline __inline
17537 #endif
17538  
17539  
17540 #if defined(__GNUC__) || __has_attribute(always_inline)
17541 # define forceinline inline __attribute__((always_inline))
17542 #elif defined(_MSC_VER)
17543 # define forceinline __forceinline
17544 #else
17545 # define forceinline inline
17546 #endif
17547  
17548  
17549 #if defined(__GNUC__) || __has_attribute(unused)
17550 # define MAYBE_UNUSED __attribute__((unused))
17551 #else
17552 # define MAYBE_UNUSED
17553 #endif
17554  
17555  
17556 #if defined(__GNUC__) || __has_attribute(noreturn)
17557 # define NORETURN __attribute__((noreturn))
17558 #else
17559 # define NORETURN
17560 #endif
17561  
17562  
17563 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
17564 # define likely(expr) __builtin_expect(!!(expr), 1)
17565 #else
17566 # define likely(expr) (expr)
17567 #endif
17568  
17569  
17570 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
17571 # define unlikely(expr) __builtin_expect(!!(expr), 0)
17572 #else
17573 # define unlikely(expr) (expr)
17574 #endif
17575  
17576  
17577 #undef prefetchr
17578 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
17579 # define prefetchr(addr) __builtin_prefetch((addr), 0)
17580 #elif defined(_MSC_VER)
17581 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
17582 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
17583 # elif defined(ARCH_ARM64)
17584 # define prefetchr(addr) __prefetch2((addr), 0x00 )
17585 # elif defined(ARCH_ARM32)
17586 # define prefetchr(addr) __prefetch(addr)
17587 # endif
17588 #endif
17589 #ifndef prefetchr
17590 # define prefetchr(addr)
17591 #endif
17592  
17593  
17594 #undef prefetchw
17595 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
17596 # define prefetchw(addr) __builtin_prefetch((addr), 1)
17597 #elif defined(_MSC_VER)
17598 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
17599 # define prefetchw(addr) _m_prefetchw(addr)
17600 # elif defined(ARCH_ARM64)
17601 # define prefetchw(addr) __prefetch2((addr), 0x10 )
17602 # elif defined(ARCH_ARM32)
17603 # define prefetchw(addr) __prefetchw(addr)
17604 # endif
17605 #endif
17606 #ifndef prefetchw
17607 # define prefetchw(addr)
17608 #endif
17609  
17610  
17611 #undef _aligned_attribute
17612 #if defined(__GNUC__) || __has_attribute(aligned)
17613 # define _aligned_attribute(n) __attribute__((aligned(n)))
17614 #elif defined(_MSC_VER)
17615 # define _aligned_attribute(n) __declspec(align(n))
17616 #endif
17617  
17618  
17619 #if defined(__GNUC__) || __has_attribute(target)
17620 # define _target_attribute(attrs) __attribute__((target(attrs)))
17621 #else
17622 # define _target_attribute(attrs)
17623 #endif
17624  
17625  
17626  
17627  
17628  
17629 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
17630 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
17631 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
17632 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
17633 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
17634 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
17635 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
17636  
17637  
17638  
17639  
17640  
17641  
17642 #if defined(__BYTE_ORDER__)
17643 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
17644 #elif defined(_MSC_VER)
17645 # define CPU_IS_LITTLE_ENDIAN() true
17646 #else
17647 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
17648 {
17649 union {
17650 u32 w;
17651 u8 b;
17652 } u;
17653  
17654 u.w = 1;
17655 return u.b;
17656 }
17657 #endif
17658  
17659  
17660 static forceinline u16 bswap16(u16 v)
17661 {
17662 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
17663 return __builtin_bswap16(v);
17664 #elif defined(_MSC_VER)
17665 return _byteswap_ushort(v);
17666 #else
17667 return (v << 8) | (v >> 8);
17668 #endif
17669 }
17670  
17671  
17672 static forceinline u32 bswap32(u32 v)
17673 {
17674 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
17675 return __builtin_bswap32(v);
17676 #elif defined(_MSC_VER)
17677 return _byteswap_ulong(v);
17678 #else
17679 return ((v & 0x000000FF) << 24) |
17680 ((v & 0x0000FF00) << 8) |
17681 ((v & 0x00FF0000) >> 8) |
17682 ((v & 0xFF000000) >> 24);
17683 #endif
17684 }
17685  
17686  
17687 static forceinline u64 bswap64(u64 v)
17688 {
17689 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
17690 return __builtin_bswap64(v);
17691 #elif defined(_MSC_VER)
17692 return _byteswap_uint64(v);
17693 #else
17694 return ((v & 0x00000000000000FF) << 56) |
17695 ((v & 0x000000000000FF00) << 40) |
17696 ((v & 0x0000000000FF0000) << 24) |
17697 ((v & 0x00000000FF000000) << 8) |
17698 ((v & 0x000000FF00000000) >> 8) |
17699 ((v & 0x0000FF0000000000) >> 24) |
17700 ((v & 0x00FF000000000000) >> 40) |
17701 ((v & 0xFF00000000000000) >> 56);
17702 #endif
17703 }
17704  
17705 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
17706 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
17707 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
17708 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
17709 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
17710 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
17711  
17712  
17713  
17714  
17715  
17716  
17717 #if (defined(__GNUC__) || defined(__clang__)) && \
17718 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
17719 defined(__ARM_FEATURE_UNALIGNED) || \
17720 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
17721 defined(__riscv_misaligned_fast) || \
17722 defined(__wasm__))
17723 # define UNALIGNED_ACCESS_IS_FAST 1
17724 #elif defined(_MSC_VER)
17725 # define UNALIGNED_ACCESS_IS_FAST 1
17726 #else
17727 # define UNALIGNED_ACCESS_IS_FAST 0
17728 #endif
17729  
17730  
17731  
17732 #ifdef FREESTANDING
17733 # define MEMCOPY __builtin_memcpy
17734 #else
17735 # define MEMCOPY memcpy
17736 #endif
17737  
17738  
17739  
17740 #define DEFINE_UNALIGNED_TYPE(type) \
17741 static forceinline type \
17742 load_##type##_unaligned(const void *p) \
17743 { \
17744 type v; \
17745 \
17746 MEMCOPY(&v, p, sizeof(v)); \
17747 return v; \
17748 } \
17749 \
17750 static forceinline void \
17751 store_##type##_unaligned(type v, void *p) \
17752 { \
17753 MEMCOPY(p, &v, sizeof(v)); \
17754 }
17755  
17756 DEFINE_UNALIGNED_TYPE(u16)
17757 DEFINE_UNALIGNED_TYPE(u32)
17758 DEFINE_UNALIGNED_TYPE(u64)
17759 DEFINE_UNALIGNED_TYPE(machine_word_t)
17760  
17761 #undef MEMCOPY
17762  
17763 #define load_word_unaligned load_machine_word_t_unaligned
17764 #define store_word_unaligned store_machine_word_t_unaligned
17765  
17766  
17767  
17768 static forceinline u16
17769 get_unaligned_le16(const u8 *p)
17770 {
17771 if (UNALIGNED_ACCESS_IS_FAST)
17772 return le16_bswap(load_u16_unaligned(p));
17773 else
17774 return ((u16)p[1] << 8) | p[0];
17775 }
17776  
17777 static forceinline u16
17778 get_unaligned_be16(const u8 *p)
17779 {
17780 if (UNALIGNED_ACCESS_IS_FAST)
17781 return be16_bswap(load_u16_unaligned(p));
17782 else
17783 return ((u16)p[0] << 8) | p[1];
17784 }
17785  
17786 static forceinline u32
17787 get_unaligned_le32(const u8 *p)
17788 {
17789 if (UNALIGNED_ACCESS_IS_FAST)
17790 return le32_bswap(load_u32_unaligned(p));
17791 else
17792 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
17793 ((u32)p[1] << 8) | p[0];
17794 }
17795  
17796 static forceinline u32
17797 get_unaligned_be32(const u8 *p)
17798 {
17799 if (UNALIGNED_ACCESS_IS_FAST)
17800 return be32_bswap(load_u32_unaligned(p));
17801 else
17802 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
17803 ((u32)p[2] << 8) | p[3];
17804 }
17805  
17806 static forceinline u64
17807 get_unaligned_le64(const u8 *p)
17808 {
17809 if (UNALIGNED_ACCESS_IS_FAST)
17810 return le64_bswap(load_u64_unaligned(p));
17811 else
17812 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
17813 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
17814 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
17815 ((u64)p[1] << 8) | p[0];
17816 }
17817  
17818 static forceinline machine_word_t
17819 get_unaligned_leword(const u8 *p)
17820 {
17821 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
17822 if (WORDBITS == 32)
17823 return get_unaligned_le32(p);
17824 else
17825 return get_unaligned_le64(p);
17826 }
17827  
17828  
17829  
17830 static forceinline void
17831 put_unaligned_le16(u16 v, u8 *p)
17832 {
17833 if (UNALIGNED_ACCESS_IS_FAST) {
17834 store_u16_unaligned(le16_bswap(v), p);
17835 } else {
17836 p[0] = (u8)(v >> 0);
17837 p[1] = (u8)(v >> 8);
17838 }
17839 }
17840  
17841 static forceinline void
17842 put_unaligned_be16(u16 v, u8 *p)
17843 {
17844 if (UNALIGNED_ACCESS_IS_FAST) {
17845 store_u16_unaligned(be16_bswap(v), p);
17846 } else {
17847 p[0] = (u8)(v >> 8);
17848 p[1] = (u8)(v >> 0);
17849 }
17850 }
17851  
17852 static forceinline void
17853 put_unaligned_le32(u32 v, u8 *p)
17854 {
17855 if (UNALIGNED_ACCESS_IS_FAST) {
17856 store_u32_unaligned(le32_bswap(v), p);
17857 } else {
17858 p[0] = (u8)(v >> 0);
17859 p[1] = (u8)(v >> 8);
17860 p[2] = (u8)(v >> 16);
17861 p[3] = (u8)(v >> 24);
17862 }
17863 }
17864  
17865 static forceinline void
17866 put_unaligned_be32(u32 v, u8 *p)
17867 {
17868 if (UNALIGNED_ACCESS_IS_FAST) {
17869 store_u32_unaligned(be32_bswap(v), p);
17870 } else {
17871 p[0] = (u8)(v >> 24);
17872 p[1] = (u8)(v >> 16);
17873 p[2] = (u8)(v >> 8);
17874 p[3] = (u8)(v >> 0);
17875 }
17876 }
17877  
17878 static forceinline void
17879 put_unaligned_le64(u64 v, u8 *p)
17880 {
17881 if (UNALIGNED_ACCESS_IS_FAST) {
17882 store_u64_unaligned(le64_bswap(v), p);
17883 } else {
17884 p[0] = (u8)(v >> 0);
17885 p[1] = (u8)(v >> 8);
17886 p[2] = (u8)(v >> 16);
17887 p[3] = (u8)(v >> 24);
17888 p[4] = (u8)(v >> 32);
17889 p[5] = (u8)(v >> 40);
17890 p[6] = (u8)(v >> 48);
17891 p[7] = (u8)(v >> 56);
17892 }
17893 }
17894  
17895 static forceinline void
17896 put_unaligned_leword(machine_word_t v, u8 *p)
17897 {
17898 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
17899 if (WORDBITS == 32)
17900 put_unaligned_le32(v, p);
17901 else
17902 put_unaligned_le64(v, p);
17903 }
17904  
17905  
17906  
17907  
17908  
17909  
17910  
17911 static forceinline unsigned
17912 bsr32(u32 v)
17913 {
17914 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
17915 return 31 - __builtin_clz(v);
17916 #elif defined(_MSC_VER)
17917 unsigned long i;
17918  
17919 _BitScanReverse(&i, v);
17920 return i;
17921 #else
17922 unsigned i = 0;
17923  
17924 while ((v >>= 1) != 0)
17925 i++;
17926 return i;
17927 #endif
17928 }
17929  
17930 static forceinline unsigned
17931 bsr64(u64 v)
17932 {
17933 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
17934 return 63 - __builtin_clzll(v);
17935 #elif defined(_MSC_VER) && defined(_WIN64)
17936 unsigned long i;
17937  
17938 _BitScanReverse64(&i, v);
17939 return i;
17940 #else
17941 unsigned i = 0;
17942  
17943 while ((v >>= 1) != 0)
17944 i++;
17945 return i;
17946 #endif
17947 }
17948  
17949 static forceinline unsigned
17950 bsrw(machine_word_t v)
17951 {
17952 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
17953 if (WORDBITS == 32)
17954 return bsr32(v);
17955 else
17956 return bsr64(v);
17957 }
17958  
17959  
17960  
17961 static forceinline unsigned
17962 bsf32(u32 v)
17963 {
17964 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
17965 return __builtin_ctz(v);
17966 #elif defined(_MSC_VER)
17967 unsigned long i;
17968  
17969 _BitScanForward(&i, v);
17970 return i;
17971 #else
17972 unsigned i = 0;
17973  
17974 for (; (v & 1) == 0; v >>= 1)
17975 i++;
17976 return i;
17977 #endif
17978 }
17979  
17980 static forceinline unsigned
17981 bsf64(u64 v)
17982 {
17983 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
17984 return __builtin_ctzll(v);
17985 #elif defined(_MSC_VER) && defined(_WIN64)
17986 unsigned long i;
17987  
17988 _BitScanForward64(&i, v);
17989 return i;
17990 #else
17991 unsigned i = 0;
17992  
17993 for (; (v & 1) == 0; v >>= 1)
17994 i++;
17995 return i;
17996 #endif
17997 }
17998  
17999 static forceinline unsigned
18000 bsfw(machine_word_t v)
18001 {
18002 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
18003 if (WORDBITS == 32)
18004 return bsf32(v);
18005 else
18006 return bsf64(v);
18007 }
18008  
18009  
18010 #undef rbit32
18011 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
18012 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
18013 static forceinline u32
18014 rbit32(u32 v)
18015 {
18016 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
18017 return v;
18018 }
18019 #define rbit32 rbit32
18020 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
18021 static forceinline u32
18022 rbit32(u32 v)
18023 {
18024 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
18025 return v;
18026 }
18027 #define rbit32 rbit32
18028 #endif
18029  
18030 #endif
18031  
18032  
18033 typedef void *(*malloc_func_t)(size_t);
18034 typedef void (*free_func_t)(void *);
18035  
18036 extern malloc_func_t libdeflate_default_malloc_func;
18037 extern free_func_t libdeflate_default_free_func;
18038  
18039 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
18040 size_t alignment, size_t size);
18041 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
18042  
18043 #ifdef FREESTANDING
18044  
18045 void *memset(void *s, int c, size_t n);
18046 #define memset(s, c, n) __builtin_memset((s), (c), (n))
18047  
18048 void *memcpy(void *dest, const void *src, size_t n);
18049 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
18050  
18051 void *memmove(void *dest, const void *src, size_t n);
18052 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
18053  
18054 int memcmp(const void *s1, const void *s2, size_t n);
18055 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
18056  
18057 #undef LIBDEFLATE_ENABLE_ASSERTIONS
18058 #else
18059 # include <string.h>
18060
18061 # ifdef __clang_analyzer__
18062 # define LIBDEFLATE_ENABLE_ASSERTIONS
18063 # endif
18064 #endif
18065  
18066  
18067 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
18068 NORETURN void
18069 libdeflate_assertion_failed(const char *expr, const char *file, int line);
18070 #define ASSERT(expr) { if (unlikely(!(expr))) \
18071 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
18072 #else
18073 #define ASSERT(expr) (void)(expr)
18074 #endif
18075  
18076 #define CONCAT_IMPL(a, b) a##b
18077 #define CONCAT(a, b) CONCAT_IMPL(a, b)
18078 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
18079  
18080 #endif
18081  
18082  
18083 #ifndef MATCHFINDER_WINDOW_ORDER
18084 # error "MATCHFINDER_WINDOW_ORDER must be defined!"
18085 #endif
18086  
18087  
18088 static forceinline u32
18089 loaded_u32_to_u24(u32 v)
18090 {
18091 if (CPU_IS_LITTLE_ENDIAN())
18092 return v & 0xFFFFFF;
18093 else
18094 return v >> 8;
18095 }
18096  
18097  
18098 static forceinline u32
18099 load_u24_unaligned(const u8 *p)
18100 {
18101 #if UNALIGNED_ACCESS_IS_FAST
18102 return loaded_u32_to_u24(load_u32_unaligned(p));
18103 #else
18104 if (CPU_IS_LITTLE_ENDIAN())
18105 return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
18106 else
18107 return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
18108 #endif
18109 }
18110  
18111 #define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER)
18112  
18113 typedef s16 mf_pos_t;
18114  
18115 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
18116  
18117  
18118 #define MATCHFINDER_MEM_ALIGNMENT 32
18119  
18120  
18121 #define MATCHFINDER_SIZE_ALIGNMENT 1024
18122  
18123 #undef matchfinder_init
18124 #undef matchfinder_rebase
18125 #ifdef _aligned_attribute
18126 # define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
18127 # if defined(ARCH_ARM32) || defined(ARCH_ARM64)
18128 /* # include "arm/matchfinder_impl.h" */
18129  
18130  
18131 #ifndef LIB_ARM_MATCHFINDER_IMPL_H
18132 #define LIB_ARM_MATCHFINDER_IMPL_H
18133  
18134 /* #include "arm-cpu_features.h" */
18135  
18136  
18137 #ifndef LIB_ARM_CPU_FEATURES_H
18138 #define LIB_ARM_CPU_FEATURES_H
18139  
18140 /* #include "lib_common.h" */
18141  
18142  
18143 #ifndef LIB_LIB_COMMON_H
18144 #define LIB_LIB_COMMON_H
18145  
18146 #ifdef LIBDEFLATE_H
18147
18148 # error "lib_common.h must always be included before libdeflate.h"
18149 #endif
18150  
18151 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
18152 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
18153 #elif defined(__GNUC__)
18154 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
18155 #else
18156 # define LIBDEFLATE_EXPORT_SYM
18157 #endif
18158  
18159  
18160 #if defined(__GNUC__) && defined(__i386__)
18161 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
18162 #else
18163 # define LIBDEFLATE_ALIGN_STACK
18164 #endif
18165  
18166 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
18167  
18168 /* #include "../common_defs.h" */
18169  
18170  
18171 #ifndef COMMON_DEFS_H
18172 #define COMMON_DEFS_H
18173  
18174 /* #include "libdeflate.h" */
18175  
18176  
18177 #ifndef LIBDEFLATE_H
18178 #define LIBDEFLATE_H
18179  
18180 #include <stddef.h>
18181 #include <stdint.h>
18182  
18183 #ifdef __cplusplus
18184 extern "C" {
18185 #endif
18186  
18187 #define LIBDEFLATE_VERSION_MAJOR 1
18188 #define LIBDEFLATE_VERSION_MINOR 26
18189 #define LIBDEFLATE_VERSION_STRING "1.26"
18190  
18191  
18192 #ifndef LIBDEFLATEAPI
18193 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
18194 # define LIBDEFLATEAPI __declspec(dllimport)
18195 # else
18196 # define LIBDEFLATEAPI
18197 # endif
18198 #endif
18199  
18200  
18201  
18202  
18203  
18204 struct libdeflate_compressor;
18205 struct libdeflate_options;
18206  
18207  
18208 LIBDEFLATEAPI struct libdeflate_compressor *
18209 libdeflate_alloc_compressor(int compression_level);
18210  
18211  
18212 LIBDEFLATEAPI struct libdeflate_compressor *
18213 libdeflate_alloc_compressor_ex(int compression_level,
18214 const struct libdeflate_options *options);
18215  
18216  
18217 LIBDEFLATEAPI size_t
18218 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
18219 const void *in, size_t in_nbytes,
18220 void *out, size_t out_nbytes_avail);
18221  
18222  
18223 LIBDEFLATEAPI size_t
18224 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
18225 size_t in_nbytes);
18226  
18227  
18228 LIBDEFLATEAPI size_t
18229 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
18230 const void *in, size_t in_nbytes,
18231 void *out, size_t out_nbytes_avail);
18232  
18233  
18234 LIBDEFLATEAPI size_t
18235 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
18236 size_t in_nbytes);
18237  
18238  
18239 LIBDEFLATEAPI size_t
18240 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
18241 const void *in, size_t in_nbytes,
18242 void *out, size_t out_nbytes_avail);
18243  
18244  
18245 LIBDEFLATEAPI size_t
18246 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
18247 size_t in_nbytes);
18248  
18249  
18250 LIBDEFLATEAPI void
18251 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
18252  
18253  
18254  
18255  
18256  
18257 struct libdeflate_decompressor;
18258 struct libdeflate_options;
18259  
18260  
18261 LIBDEFLATEAPI struct libdeflate_decompressor *
18262 libdeflate_alloc_decompressor(void);
18263  
18264  
18265 LIBDEFLATEAPI struct libdeflate_decompressor *
18266 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
18267  
18268  
18269 enum libdeflate_result {
18270
18271 LIBDEFLATE_SUCCESS = 0,
18272  
18273
18274 LIBDEFLATE_BAD_DATA = 1,
18275  
18276
18277 LIBDEFLATE_SHORT_OUTPUT = 2,
18278  
18279
18280 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
18281 };
18282  
18283  
18284 LIBDEFLATEAPI enum libdeflate_result
18285 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
18286 const void *in, size_t in_nbytes,
18287 void *out, size_t out_nbytes_avail,
18288 size_t *actual_out_nbytes_ret);
18289  
18290  
18291 LIBDEFLATEAPI enum libdeflate_result
18292 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
18293 const void *in, size_t in_nbytes,
18294 void *out, size_t out_nbytes_avail,
18295 size_t *actual_in_nbytes_ret,
18296 size_t *actual_out_nbytes_ret);
18297  
18298  
18299 LIBDEFLATEAPI enum libdeflate_result
18300 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
18301 const void *in, size_t in_nbytes,
18302 void *out, size_t out_nbytes_avail,
18303 size_t *actual_out_nbytes_ret);
18304  
18305  
18306 LIBDEFLATEAPI enum libdeflate_result
18307 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
18308 const void *in, size_t in_nbytes,
18309 void *out, size_t out_nbytes_avail,
18310 size_t *actual_in_nbytes_ret,
18311 size_t *actual_out_nbytes_ret);
18312  
18313  
18314 LIBDEFLATEAPI enum libdeflate_result
18315 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
18316 const void *in, size_t in_nbytes,
18317 void *out, size_t out_nbytes_avail,
18318 size_t *actual_out_nbytes_ret);
18319  
18320  
18321 LIBDEFLATEAPI enum libdeflate_result
18322 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
18323 const void *in, size_t in_nbytes,
18324 void *out, size_t out_nbytes_avail,
18325 size_t *actual_in_nbytes_ret,
18326 size_t *actual_out_nbytes_ret);
18327  
18328  
18329 LIBDEFLATEAPI void
18330 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
18331  
18332  
18333  
18334  
18335  
18336  
18337 LIBDEFLATEAPI uint32_t
18338 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
18339  
18340  
18341  
18342 LIBDEFLATEAPI uint32_t
18343 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
18344  
18345  
18346  
18347  
18348  
18349  
18350 LIBDEFLATEAPI void
18351 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
18352 void (*free_func)(void *));
18353  
18354  
18355 struct libdeflate_options {
18356  
18357
18358 size_t sizeof_options;
18359  
18360
18361 void *(*malloc_func)(size_t);
18362 void (*free_func)(void *);
18363 };
18364  
18365 #ifdef __cplusplus
18366 }
18367 #endif
18368  
18369 #endif
18370  
18371  
18372 #include <stdbool.h>
18373 #include <stddef.h>
18374 #include <stdint.h>
18375 #ifdef _MSC_VER
18376 # include <intrin.h>
18377 # include <stdlib.h>
18378
18379
18380 # pragma warning(disable : 4146)
18381
18382 # pragma warning(disable : 4018)
18383 # pragma warning(disable : 4244)
18384 # pragma warning(disable : 4267)
18385 # pragma warning(disable : 4310)
18386
18387 # pragma warning(disable : 4100)
18388 # pragma warning(disable : 4127)
18389 # pragma warning(disable : 4189)
18390 # pragma warning(disable : 4232)
18391 # pragma warning(disable : 4245)
18392 # pragma warning(disable : 4295)
18393 #endif
18394 #ifndef FREESTANDING
18395 # include <string.h>
18396 #endif
18397  
18398  
18399  
18400  
18401  
18402  
18403 #undef ARCH_X86_64
18404 #undef ARCH_X86_32
18405 #undef ARCH_ARM64
18406 #undef ARCH_ARM32
18407 #undef ARCH_RISCV
18408 #ifdef _MSC_VER
18409
18410 # if defined(_M_X64) && !defined(_M_ARM64EC)
18411 # define ARCH_X86_64
18412 # elif defined(_M_IX86)
18413 # define ARCH_X86_32
18414 # elif defined(_M_ARM64)
18415 # define ARCH_ARM64
18416 # elif defined(_M_ARM)
18417 # define ARCH_ARM32
18418 # endif
18419 #else
18420 # if defined(__x86_64__)
18421 # define ARCH_X86_64
18422 # elif defined(__i386__)
18423 # define ARCH_X86_32
18424 # elif defined(__aarch64__)
18425 # define ARCH_ARM64
18426 # elif defined(__arm__)
18427 # define ARCH_ARM32
18428 # elif defined(__riscv)
18429 # define ARCH_RISCV
18430 # endif
18431 #endif
18432  
18433  
18434  
18435  
18436  
18437  
18438 typedef uint8_t u8;
18439 typedef uint16_t u16;
18440 typedef uint32_t u32;
18441 typedef uint64_t u64;
18442 typedef int8_t s8;
18443 typedef int16_t s16;
18444 typedef int32_t s32;
18445 typedef int64_t s64;
18446  
18447  
18448 #ifdef _MSC_VER
18449 # ifdef _WIN64
18450 typedef long long ssize_t;
18451 # else
18452 typedef long ssize_t;
18453 # endif
18454 #endif
18455  
18456  
18457 typedef size_t machine_word_t;
18458  
18459  
18460 #define WORDBYTES ((int)sizeof(machine_word_t))
18461  
18462  
18463 #define WORDBITS (8 * WORDBYTES)
18464  
18465  
18466  
18467  
18468  
18469  
18470 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
18471 # define GCC_PREREQ(major, minor) \
18472 (__GNUC__ > (major) || \
18473 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
18474 # if !GCC_PREREQ(4, 9)
18475 # error "gcc versions older than 4.9 are no longer supported"
18476 # endif
18477 #else
18478 # define GCC_PREREQ(major, minor) 0
18479 #endif
18480 #ifdef __clang__
18481 # ifdef __apple_build_version__
18482 # define CLANG_PREREQ(major, minor, apple_version) \
18483 (__apple_build_version__ >= (apple_version))
18484 # else
18485 # define CLANG_PREREQ(major, minor, apple_version) \
18486 (__clang_major__ > (major) || \
18487 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
18488 # endif
18489 # if !CLANG_PREREQ(3, 9, 8000000)
18490 # error "clang versions older than 3.9 are no longer supported"
18491 # endif
18492 #else
18493 # define CLANG_PREREQ(major, minor, apple_version) 0
18494 #endif
18495 #ifdef _MSC_VER
18496 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
18497 # if !MSVC_PREREQ(1928)
18498 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
18499 # endif
18500 #else
18501 # define MSVC_PREREQ(version) 0
18502 #endif
18503  
18504  
18505 #ifndef __has_attribute
18506 # define __has_attribute(attribute) 0
18507 #endif
18508  
18509  
18510 #ifndef __has_builtin
18511 # define __has_builtin(builtin) 0
18512 #endif
18513  
18514  
18515 #ifdef _MSC_VER
18516 # define inline __inline
18517 #endif
18518  
18519  
18520 #if defined(__GNUC__) || __has_attribute(always_inline)
18521 # define forceinline inline __attribute__((always_inline))
18522 #elif defined(_MSC_VER)
18523 # define forceinline __forceinline
18524 #else
18525 # define forceinline inline
18526 #endif
18527  
18528  
18529 #if defined(__GNUC__) || __has_attribute(unused)
18530 # define MAYBE_UNUSED __attribute__((unused))
18531 #else
18532 # define MAYBE_UNUSED
18533 #endif
18534  
18535  
18536 #if defined(__GNUC__) || __has_attribute(noreturn)
18537 # define NORETURN __attribute__((noreturn))
18538 #else
18539 # define NORETURN
18540 #endif
18541  
18542  
18543 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
18544 # define likely(expr) __builtin_expect(!!(expr), 1)
18545 #else
18546 # define likely(expr) (expr)
18547 #endif
18548  
18549  
18550 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
18551 # define unlikely(expr) __builtin_expect(!!(expr), 0)
18552 #else
18553 # define unlikely(expr) (expr)
18554 #endif
18555  
18556  
18557 #undef prefetchr
18558 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
18559 # define prefetchr(addr) __builtin_prefetch((addr), 0)
18560 #elif defined(_MSC_VER)
18561 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
18562 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
18563 # elif defined(ARCH_ARM64)
18564 # define prefetchr(addr) __prefetch2((addr), 0x00 )
18565 # elif defined(ARCH_ARM32)
18566 # define prefetchr(addr) __prefetch(addr)
18567 # endif
18568 #endif
18569 #ifndef prefetchr
18570 # define prefetchr(addr)
18571 #endif
18572  
18573  
18574 #undef prefetchw
18575 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
18576 # define prefetchw(addr) __builtin_prefetch((addr), 1)
18577 #elif defined(_MSC_VER)
18578 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
18579 # define prefetchw(addr) _m_prefetchw(addr)
18580 # elif defined(ARCH_ARM64)
18581 # define prefetchw(addr) __prefetch2((addr), 0x10 )
18582 # elif defined(ARCH_ARM32)
18583 # define prefetchw(addr) __prefetchw(addr)
18584 # endif
18585 #endif
18586 #ifndef prefetchw
18587 # define prefetchw(addr)
18588 #endif
18589  
18590  
18591 #undef _aligned_attribute
18592 #if defined(__GNUC__) || __has_attribute(aligned)
18593 # define _aligned_attribute(n) __attribute__((aligned(n)))
18594 #elif defined(_MSC_VER)
18595 # define _aligned_attribute(n) __declspec(align(n))
18596 #endif
18597  
18598  
18599 #if defined(__GNUC__) || __has_attribute(target)
18600 # define _target_attribute(attrs) __attribute__((target(attrs)))
18601 #else
18602 # define _target_attribute(attrs)
18603 #endif
18604  
18605  
18606  
18607  
18608  
18609 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
18610 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
18611 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
18612 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
18613 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
18614 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
18615 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
18616  
18617  
18618  
18619  
18620  
18621  
18622 #if defined(__BYTE_ORDER__)
18623 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
18624 #elif defined(_MSC_VER)
18625 # define CPU_IS_LITTLE_ENDIAN() true
18626 #else
18627 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
18628 {
18629 union {
18630 u32 w;
18631 u8 b;
18632 } u;
18633  
18634 u.w = 1;
18635 return u.b;
18636 }
18637 #endif
18638  
18639  
18640 static forceinline u16 bswap16(u16 v)
18641 {
18642 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
18643 return __builtin_bswap16(v);
18644 #elif defined(_MSC_VER)
18645 return _byteswap_ushort(v);
18646 #else
18647 return (v << 8) | (v >> 8);
18648 #endif
18649 }
18650  
18651  
18652 static forceinline u32 bswap32(u32 v)
18653 {
18654 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
18655 return __builtin_bswap32(v);
18656 #elif defined(_MSC_VER)
18657 return _byteswap_ulong(v);
18658 #else
18659 return ((v & 0x000000FF) << 24) |
18660 ((v & 0x0000FF00) << 8) |
18661 ((v & 0x00FF0000) >> 8) |
18662 ((v & 0xFF000000) >> 24);
18663 #endif
18664 }
18665  
18666  
18667 static forceinline u64 bswap64(u64 v)
18668 {
18669 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
18670 return __builtin_bswap64(v);
18671 #elif defined(_MSC_VER)
18672 return _byteswap_uint64(v);
18673 #else
18674 return ((v & 0x00000000000000FF) << 56) |
18675 ((v & 0x000000000000FF00) << 40) |
18676 ((v & 0x0000000000FF0000) << 24) |
18677 ((v & 0x00000000FF000000) << 8) |
18678 ((v & 0x000000FF00000000) >> 8) |
18679 ((v & 0x0000FF0000000000) >> 24) |
18680 ((v & 0x00FF000000000000) >> 40) |
18681 ((v & 0xFF00000000000000) >> 56);
18682 #endif
18683 }
18684  
18685 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
18686 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
18687 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
18688 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
18689 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
18690 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
18691  
18692  
18693  
18694  
18695  
18696  
18697 #if (defined(__GNUC__) || defined(__clang__)) && \
18698 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
18699 defined(__ARM_FEATURE_UNALIGNED) || \
18700 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
18701 defined(__riscv_misaligned_fast) || \
18702 defined(__wasm__))
18703 # define UNALIGNED_ACCESS_IS_FAST 1
18704 #elif defined(_MSC_VER)
18705 # define UNALIGNED_ACCESS_IS_FAST 1
18706 #else
18707 # define UNALIGNED_ACCESS_IS_FAST 0
18708 #endif
18709  
18710  
18711  
18712 #ifdef FREESTANDING
18713 # define MEMCOPY __builtin_memcpy
18714 #else
18715 # define MEMCOPY memcpy
18716 #endif
18717  
18718  
18719  
18720 #define DEFINE_UNALIGNED_TYPE(type) \
18721 static forceinline type \
18722 load_##type##_unaligned(const void *p) \
18723 { \
18724 type v; \
18725 \
18726 MEMCOPY(&v, p, sizeof(v)); \
18727 return v; \
18728 } \
18729 \
18730 static forceinline void \
18731 store_##type##_unaligned(type v, void *p) \
18732 { \
18733 MEMCOPY(p, &v, sizeof(v)); \
18734 }
18735  
18736 DEFINE_UNALIGNED_TYPE(u16)
18737 DEFINE_UNALIGNED_TYPE(u32)
18738 DEFINE_UNALIGNED_TYPE(u64)
18739 DEFINE_UNALIGNED_TYPE(machine_word_t)
18740  
18741 #undef MEMCOPY
18742  
18743 #define load_word_unaligned load_machine_word_t_unaligned
18744 #define store_word_unaligned store_machine_word_t_unaligned
18745  
18746  
18747  
18748 static forceinline u16
18749 get_unaligned_le16(const u8 *p)
18750 {
18751 if (UNALIGNED_ACCESS_IS_FAST)
18752 return le16_bswap(load_u16_unaligned(p));
18753 else
18754 return ((u16)p[1] << 8) | p[0];
18755 }
18756  
18757 static forceinline u16
18758 get_unaligned_be16(const u8 *p)
18759 {
18760 if (UNALIGNED_ACCESS_IS_FAST)
18761 return be16_bswap(load_u16_unaligned(p));
18762 else
18763 return ((u16)p[0] << 8) | p[1];
18764 }
18765  
18766 static forceinline u32
18767 get_unaligned_le32(const u8 *p)
18768 {
18769 if (UNALIGNED_ACCESS_IS_FAST)
18770 return le32_bswap(load_u32_unaligned(p));
18771 else
18772 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
18773 ((u32)p[1] << 8) | p[0];
18774 }
18775  
18776 static forceinline u32
18777 get_unaligned_be32(const u8 *p)
18778 {
18779 if (UNALIGNED_ACCESS_IS_FAST)
18780 return be32_bswap(load_u32_unaligned(p));
18781 else
18782 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
18783 ((u32)p[2] << 8) | p[3];
18784 }
18785  
18786 static forceinline u64
18787 get_unaligned_le64(const u8 *p)
18788 {
18789 if (UNALIGNED_ACCESS_IS_FAST)
18790 return le64_bswap(load_u64_unaligned(p));
18791 else
18792 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
18793 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
18794 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
18795 ((u64)p[1] << 8) | p[0];
18796 }
18797  
18798 static forceinline machine_word_t
18799 get_unaligned_leword(const u8 *p)
18800 {
18801 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
18802 if (WORDBITS == 32)
18803 return get_unaligned_le32(p);
18804 else
18805 return get_unaligned_le64(p);
18806 }
18807  
18808  
18809  
18810 static forceinline void
18811 put_unaligned_le16(u16 v, u8 *p)
18812 {
18813 if (UNALIGNED_ACCESS_IS_FAST) {
18814 store_u16_unaligned(le16_bswap(v), p);
18815 } else {
18816 p[0] = (u8)(v >> 0);
18817 p[1] = (u8)(v >> 8);
18818 }
18819 }
18820  
18821 static forceinline void
18822 put_unaligned_be16(u16 v, u8 *p)
18823 {
18824 if (UNALIGNED_ACCESS_IS_FAST) {
18825 store_u16_unaligned(be16_bswap(v), p);
18826 } else {
18827 p[0] = (u8)(v >> 8);
18828 p[1] = (u8)(v >> 0);
18829 }
18830 }
18831  
18832 static forceinline void
18833 put_unaligned_le32(u32 v, u8 *p)
18834 {
18835 if (UNALIGNED_ACCESS_IS_FAST) {
18836 store_u32_unaligned(le32_bswap(v), p);
18837 } else {
18838 p[0] = (u8)(v >> 0);
18839 p[1] = (u8)(v >> 8);
18840 p[2] = (u8)(v >> 16);
18841 p[3] = (u8)(v >> 24);
18842 }
18843 }
18844  
18845 static forceinline void
18846 put_unaligned_be32(u32 v, u8 *p)
18847 {
18848 if (UNALIGNED_ACCESS_IS_FAST) {
18849 store_u32_unaligned(be32_bswap(v), p);
18850 } else {
18851 p[0] = (u8)(v >> 24);
18852 p[1] = (u8)(v >> 16);
18853 p[2] = (u8)(v >> 8);
18854 p[3] = (u8)(v >> 0);
18855 }
18856 }
18857  
18858 static forceinline void
18859 put_unaligned_le64(u64 v, u8 *p)
18860 {
18861 if (UNALIGNED_ACCESS_IS_FAST) {
18862 store_u64_unaligned(le64_bswap(v), p);
18863 } else {
18864 p[0] = (u8)(v >> 0);
18865 p[1] = (u8)(v >> 8);
18866 p[2] = (u8)(v >> 16);
18867 p[3] = (u8)(v >> 24);
18868 p[4] = (u8)(v >> 32);
18869 p[5] = (u8)(v >> 40);
18870 p[6] = (u8)(v >> 48);
18871 p[7] = (u8)(v >> 56);
18872 }
18873 }
18874  
18875 static forceinline void
18876 put_unaligned_leword(machine_word_t v, u8 *p)
18877 {
18878 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
18879 if (WORDBITS == 32)
18880 put_unaligned_le32(v, p);
18881 else
18882 put_unaligned_le64(v, p);
18883 }
18884  
18885  
18886  
18887  
18888  
18889  
18890  
18891 static forceinline unsigned
18892 bsr32(u32 v)
18893 {
18894 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
18895 return 31 - __builtin_clz(v);
18896 #elif defined(_MSC_VER)
18897 unsigned long i;
18898  
18899 _BitScanReverse(&i, v);
18900 return i;
18901 #else
18902 unsigned i = 0;
18903  
18904 while ((v >>= 1) != 0)
18905 i++;
18906 return i;
18907 #endif
18908 }
18909  
18910 static forceinline unsigned
18911 bsr64(u64 v)
18912 {
18913 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
18914 return 63 - __builtin_clzll(v);
18915 #elif defined(_MSC_VER) && defined(_WIN64)
18916 unsigned long i;
18917  
18918 _BitScanReverse64(&i, v);
18919 return i;
18920 #else
18921 unsigned i = 0;
18922  
18923 while ((v >>= 1) != 0)
18924 i++;
18925 return i;
18926 #endif
18927 }
18928  
18929 static forceinline unsigned
18930 bsrw(machine_word_t v)
18931 {
18932 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
18933 if (WORDBITS == 32)
18934 return bsr32(v);
18935 else
18936 return bsr64(v);
18937 }
18938  
18939  
18940  
18941 static forceinline unsigned
18942 bsf32(u32 v)
18943 {
18944 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
18945 return __builtin_ctz(v);
18946 #elif defined(_MSC_VER)
18947 unsigned long i;
18948  
18949 _BitScanForward(&i, v);
18950 return i;
18951 #else
18952 unsigned i = 0;
18953  
18954 for (; (v & 1) == 0; v >>= 1)
18955 i++;
18956 return i;
18957 #endif
18958 }
18959  
18960 static forceinline unsigned
18961 bsf64(u64 v)
18962 {
18963 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
18964 return __builtin_ctzll(v);
18965 #elif defined(_MSC_VER) && defined(_WIN64)
18966 unsigned long i;
18967  
18968 _BitScanForward64(&i, v);
18969 return i;
18970 #else
18971 unsigned i = 0;
18972  
18973 for (; (v & 1) == 0; v >>= 1)
18974 i++;
18975 return i;
18976 #endif
18977 }
18978  
18979 static forceinline unsigned
18980 bsfw(machine_word_t v)
18981 {
18982 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
18983 if (WORDBITS == 32)
18984 return bsf32(v);
18985 else
18986 return bsf64(v);
18987 }
18988  
18989  
18990 #undef rbit32
18991 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
18992 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
18993 static forceinline u32
18994 rbit32(u32 v)
18995 {
18996 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
18997 return v;
18998 }
18999 #define rbit32 rbit32
19000 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
19001 static forceinline u32
19002 rbit32(u32 v)
19003 {
19004 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
19005 return v;
19006 }
19007 #define rbit32 rbit32
19008 #endif
19009  
19010 #endif
19011  
19012  
19013 typedef void *(*malloc_func_t)(size_t);
19014 typedef void (*free_func_t)(void *);
19015  
19016 extern malloc_func_t libdeflate_default_malloc_func;
19017 extern free_func_t libdeflate_default_free_func;
19018  
19019 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
19020 size_t alignment, size_t size);
19021 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
19022  
19023 #ifdef FREESTANDING
19024  
19025 void *memset(void *s, int c, size_t n);
19026 #define memset(s, c, n) __builtin_memset((s), (c), (n))
19027  
19028 void *memcpy(void *dest, const void *src, size_t n);
19029 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
19030  
19031 void *memmove(void *dest, const void *src, size_t n);
19032 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
19033  
19034 int memcmp(const void *s1, const void *s2, size_t n);
19035 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
19036  
19037 #undef LIBDEFLATE_ENABLE_ASSERTIONS
19038 #else
19039 # include <string.h>
19040
19041 # ifdef __clang_analyzer__
19042 # define LIBDEFLATE_ENABLE_ASSERTIONS
19043 # endif
19044 #endif
19045  
19046  
19047 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
19048 NORETURN void
19049 libdeflate_assertion_failed(const char *expr, const char *file, int line);
19050 #define ASSERT(expr) { if (unlikely(!(expr))) \
19051 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
19052 #else
19053 #define ASSERT(expr) (void)(expr)
19054 #endif
19055  
19056 #define CONCAT_IMPL(a, b) a##b
19057 #define CONCAT(a, b) CONCAT_IMPL(a, b)
19058 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
19059  
19060 #endif
19061  
19062  
19063 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
19064  
19065 #define ARM_CPU_FEATURE_NEON (1 << 0)
19066 #define ARM_CPU_FEATURE_PMULL (1 << 1)
19067  
19068 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
19069 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
19070 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
19071 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
19072  
19073 #if !defined(FREESTANDING) && \
19074 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
19075 (defined(__linux__) || \
19076 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
19077 (defined(_WIN32) && defined(ARCH_ARM64)))
19078  
19079 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
19080 extern volatile u32 libdeflate_arm_cpu_features;
19081  
19082 void libdeflate_init_arm_cpu_features(void);
19083  
19084 static inline u32 get_arm_cpu_features(void)
19085 {
19086 if (libdeflate_arm_cpu_features == 0)
19087 libdeflate_init_arm_cpu_features();
19088 return libdeflate_arm_cpu_features;
19089 }
19090 #else
19091 static inline u32 get_arm_cpu_features(void) { return 0; }
19092 #endif
19093  
19094  
19095 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
19096 # define HAVE_NEON(features) 1
19097 # define HAVE_NEON_NATIVE 1
19098 #else
19099 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
19100 # define HAVE_NEON_NATIVE 0
19101 #endif
19102  
19103 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
19104 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
19105 # define HAVE_NEON_INTRIN 1
19106 # include <arm_neon.h>
19107 #else
19108 # define HAVE_NEON_INTRIN 0
19109 #endif
19110  
19111  
19112 #ifdef __ARM_FEATURE_CRYPTO
19113 # define HAVE_PMULL(features) 1
19114 #else
19115 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
19116 #endif
19117 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
19118 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
19119 CPU_IS_LITTLE_ENDIAN()
19120 # define HAVE_PMULL_INTRIN 1
19121
19122 # ifdef _MSC_VER
19123 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
19124 # else
19125 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
19126 # endif
19127 #else
19128 # define HAVE_PMULL_INTRIN 0
19129 #endif
19130  
19131  
19132 #ifdef __ARM_FEATURE_CRC32
19133 # define HAVE_CRC32(features) 1
19134 #else
19135 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
19136 #endif
19137 #if defined(ARCH_ARM64) && \
19138 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
19139 # define HAVE_CRC32_INTRIN 1
19140 # if defined(__GNUC__) || defined(__clang__)
19141 # include <arm_acle.h>
19142 # endif
19143
19144 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
19145 !defined(__ARM_FEATURE_CRC32)
19146 # undef __crc32b
19147 # define __crc32b(a, b) \
19148 ({ uint32_t res; \
19149 __asm__("crc32b %w0, %w1, %w2" \
19150 : "=r" (res) : "r" (a), "r" (b)); \
19151 res; })
19152 # undef __crc32h
19153 # define __crc32h(a, b) \
19154 ({ uint32_t res; \
19155 __asm__("crc32h %w0, %w1, %w2" \
19156 : "=r" (res) : "r" (a), "r" (b)); \
19157 res; })
19158 # undef __crc32w
19159 # define __crc32w(a, b) \
19160 ({ uint32_t res; \
19161 __asm__("crc32w %w0, %w1, %w2" \
19162 : "=r" (res) : "r" (a), "r" (b)); \
19163 res; })
19164 # undef __crc32d
19165 # define __crc32d(a, b) \
19166 ({ uint32_t res; \
19167 __asm__("crc32x %w0, %w1, %2" \
19168 : "=r" (res) : "r" (a), "r" (b)); \
19169 res; })
19170 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
19171 # endif
19172 #else
19173 # define HAVE_CRC32_INTRIN 0
19174 #endif
19175  
19176  
19177 #ifdef __ARM_FEATURE_SHA3
19178 # define HAVE_SHA3(features) 1
19179 #else
19180 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
19181 #endif
19182 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
19183 (GCC_PREREQ(9, 1) || \
19184 CLANG_PREREQ(7, 0, 10010463) )
19185 # define HAVE_SHA3_INTRIN 1
19186
19187 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
19188 !defined(__ARM_FEATURE_SHA3)
19189 # undef veor3q_u8
19190 # define veor3q_u8(a, b, c) \
19191 ({ uint8x16_t res; \
19192 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
19193 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
19194 res; })
19195 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
19196 # endif
19197 #else
19198 # define HAVE_SHA3_INTRIN 0
19199 #endif
19200  
19201  
19202 #ifdef __ARM_FEATURE_DOTPROD
19203 # define HAVE_DOTPROD(features) 1
19204 #else
19205 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
19206 #endif
19207 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
19208 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
19209 # define HAVE_DOTPROD_INTRIN 1
19210
19211 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
19212 !defined(__ARM_FEATURE_DOTPROD)
19213 # undef vdotq_u32
19214 # define vdotq_u32(a, b, c) \
19215 ({ uint32x4_t res = (a); \
19216 __asm__("udot %0.4s, %1.16b, %2.16b" \
19217 : "+w" (res) : "w" (b), "w" (c)); \
19218 res; })
19219 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
19220 # endif
19221 #else
19222 # define HAVE_DOTPROD_INTRIN 0
19223 #endif
19224  
19225 #endif
19226  
19227 #endif
19228  
19229  
19230 #if HAVE_NEON_NATIVE
19231 static forceinline void
19232 matchfinder_init_neon(mf_pos_t *data, size_t size)
19233 {
19234 int16x8_t *p = (int16x8_t *)data;
19235 int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);
19236  
19237 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
19238 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
19239 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
19240  
19241 do {
19242 p[0] = v;
19243 p[1] = v;
19244 p[2] = v;
19245 p[3] = v;
19246 p += 4;
19247 size -= 4 * sizeof(*p);
19248 } while (size != 0);
19249 }
19250 #define matchfinder_init matchfinder_init_neon
19251  
19252 static forceinline void
19253 matchfinder_rebase_neon(mf_pos_t *data, size_t size)
19254 {
19255 int16x8_t *p = (int16x8_t *)data;
19256 int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);
19257  
19258 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
19259 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
19260 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
19261  
19262 do {
19263 p[0] = vqaddq_s16(p[0], v);
19264 p[1] = vqaddq_s16(p[1], v);
19265 p[2] = vqaddq_s16(p[2], v);
19266 p[3] = vqaddq_s16(p[3], v);
19267 p += 4;
19268 size -= 4 * sizeof(*p);
19269 } while (size != 0);
19270 }
19271 #define matchfinder_rebase matchfinder_rebase_neon
19272  
19273 #endif
19274  
19275 #endif
19276  
19277 # elif defined(ARCH_RISCV)
19278 # include "riscv/matchfinder_impl.h"
19279 # elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
19280 /* # include "x86/matchfinder_impl.h" */
19281  
19282  
19283 #ifndef LIB_X86_MATCHFINDER_IMPL_H
19284 #define LIB_X86_MATCHFINDER_IMPL_H
19285  
19286 /* #include "x86-cpu_features.h" */
19287  
19288  
19289 #ifndef LIB_X86_CPU_FEATURES_H
19290 #define LIB_X86_CPU_FEATURES_H
19291  
19292 /* #include "lib_common.h" */
19293  
19294  
19295 #ifndef LIB_LIB_COMMON_H
19296 #define LIB_LIB_COMMON_H
19297  
19298 #ifdef LIBDEFLATE_H
19299
19300 # error "lib_common.h must always be included before libdeflate.h"
19301 #endif
19302  
19303 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
19304 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
19305 #elif defined(__GNUC__)
19306 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
19307 #else
19308 # define LIBDEFLATE_EXPORT_SYM
19309 #endif
19310  
19311  
19312 #if defined(__GNUC__) && defined(__i386__)
19313 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
19314 #else
19315 # define LIBDEFLATE_ALIGN_STACK
19316 #endif
19317  
19318 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
19319  
19320 /* #include "../common_defs.h" */
19321  
19322  
19323 #ifndef COMMON_DEFS_H
19324 #define COMMON_DEFS_H
19325  
19326 /* #include "libdeflate.h" */
19327  
19328  
19329 #ifndef LIBDEFLATE_H
19330 #define LIBDEFLATE_H
19331  
19332 #include <stddef.h>
19333 #include <stdint.h>
19334  
19335 #ifdef __cplusplus
19336 extern "C" {
19337 #endif
19338  
19339 #define LIBDEFLATE_VERSION_MAJOR 1
19340 #define LIBDEFLATE_VERSION_MINOR 26
19341 #define LIBDEFLATE_VERSION_STRING "1.26"
19342  
19343  
19344 #ifndef LIBDEFLATEAPI
19345 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
19346 # define LIBDEFLATEAPI __declspec(dllimport)
19347 # else
19348 # define LIBDEFLATEAPI
19349 # endif
19350 #endif
19351  
19352  
19353  
19354  
19355  
19356 struct libdeflate_compressor;
19357 struct libdeflate_options;
19358  
19359  
19360 LIBDEFLATEAPI struct libdeflate_compressor *
19361 libdeflate_alloc_compressor(int compression_level);
19362  
19363  
19364 LIBDEFLATEAPI struct libdeflate_compressor *
19365 libdeflate_alloc_compressor_ex(int compression_level,
19366 const struct libdeflate_options *options);
19367  
19368  
19369 LIBDEFLATEAPI size_t
19370 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
19371 const void *in, size_t in_nbytes,
19372 void *out, size_t out_nbytes_avail);
19373  
19374  
19375 LIBDEFLATEAPI size_t
19376 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
19377 size_t in_nbytes);
19378  
19379  
19380 LIBDEFLATEAPI size_t
19381 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
19382 const void *in, size_t in_nbytes,
19383 void *out, size_t out_nbytes_avail);
19384  
19385  
19386 LIBDEFLATEAPI size_t
19387 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
19388 size_t in_nbytes);
19389  
19390  
19391 LIBDEFLATEAPI size_t
19392 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
19393 const void *in, size_t in_nbytes,
19394 void *out, size_t out_nbytes_avail);
19395  
19396  
19397 LIBDEFLATEAPI size_t
19398 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
19399 size_t in_nbytes);
19400  
19401  
19402 LIBDEFLATEAPI void
19403 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
19404  
19405  
19406  
19407  
19408  
19409 struct libdeflate_decompressor;
19410 struct libdeflate_options;
19411  
19412  
19413 LIBDEFLATEAPI struct libdeflate_decompressor *
19414 libdeflate_alloc_decompressor(void);
19415  
19416  
19417 LIBDEFLATEAPI struct libdeflate_decompressor *
19418 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
19419  
19420  
19421 enum libdeflate_result {
19422
19423 LIBDEFLATE_SUCCESS = 0,
19424  
19425
19426 LIBDEFLATE_BAD_DATA = 1,
19427  
19428
19429 LIBDEFLATE_SHORT_OUTPUT = 2,
19430  
19431
19432 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
19433 };
19434  
19435  
19436 LIBDEFLATEAPI enum libdeflate_result
19437 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
19438 const void *in, size_t in_nbytes,
19439 void *out, size_t out_nbytes_avail,
19440 size_t *actual_out_nbytes_ret);
19441  
19442  
19443 LIBDEFLATEAPI enum libdeflate_result
19444 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
19445 const void *in, size_t in_nbytes,
19446 void *out, size_t out_nbytes_avail,
19447 size_t *actual_in_nbytes_ret,
19448 size_t *actual_out_nbytes_ret);
19449  
19450  
19451 LIBDEFLATEAPI enum libdeflate_result
19452 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
19453 const void *in, size_t in_nbytes,
19454 void *out, size_t out_nbytes_avail,
19455 size_t *actual_out_nbytes_ret);
19456  
19457  
19458 LIBDEFLATEAPI enum libdeflate_result
19459 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
19460 const void *in, size_t in_nbytes,
19461 void *out, size_t out_nbytes_avail,
19462 size_t *actual_in_nbytes_ret,
19463 size_t *actual_out_nbytes_ret);
19464  
19465  
19466 LIBDEFLATEAPI enum libdeflate_result
19467 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
19468 const void *in, size_t in_nbytes,
19469 void *out, size_t out_nbytes_avail,
19470 size_t *actual_out_nbytes_ret);
19471  
19472  
19473 LIBDEFLATEAPI enum libdeflate_result
19474 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
19475 const void *in, size_t in_nbytes,
19476 void *out, size_t out_nbytes_avail,
19477 size_t *actual_in_nbytes_ret,
19478 size_t *actual_out_nbytes_ret);
19479  
19480  
19481 LIBDEFLATEAPI void
19482 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
19483  
19484  
19485  
19486  
19487  
19488  
19489 LIBDEFLATEAPI uint32_t
19490 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
19491  
19492  
19493  
19494 LIBDEFLATEAPI uint32_t
19495 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
19496  
19497  
19498  
19499  
19500  
19501  
19502 LIBDEFLATEAPI void
19503 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
19504 void (*free_func)(void *));
19505  
19506  
19507 struct libdeflate_options {
19508  
19509
19510 size_t sizeof_options;
19511  
19512
19513 void *(*malloc_func)(size_t);
19514 void (*free_func)(void *);
19515 };
19516  
19517 #ifdef __cplusplus
19518 }
19519 #endif
19520  
19521 #endif
19522  
19523  
19524 #include <stdbool.h>
19525 #include <stddef.h>
19526 #include <stdint.h>
19527 #ifdef _MSC_VER
19528 # include <intrin.h>
19529 # include <stdlib.h>
19530
19531
19532 # pragma warning(disable : 4146)
19533
19534 # pragma warning(disable : 4018)
19535 # pragma warning(disable : 4244)
19536 # pragma warning(disable : 4267)
19537 # pragma warning(disable : 4310)
19538
19539 # pragma warning(disable : 4100)
19540 # pragma warning(disable : 4127)
19541 # pragma warning(disable : 4189)
19542 # pragma warning(disable : 4232)
19543 # pragma warning(disable : 4245)
19544 # pragma warning(disable : 4295)
19545 #endif
19546 #ifndef FREESTANDING
19547 # include <string.h>
19548 #endif
19549  
19550  
19551  
19552  
19553  
19554  
19555 #undef ARCH_X86_64
19556 #undef ARCH_X86_32
19557 #undef ARCH_ARM64
19558 #undef ARCH_ARM32
19559 #undef ARCH_RISCV
19560 #ifdef _MSC_VER
19561
19562 # if defined(_M_X64) && !defined(_M_ARM64EC)
19563 # define ARCH_X86_64
19564 # elif defined(_M_IX86)
19565 # define ARCH_X86_32
19566 # elif defined(_M_ARM64)
19567 # define ARCH_ARM64
19568 # elif defined(_M_ARM)
19569 # define ARCH_ARM32
19570 # endif
19571 #else
19572 # if defined(__x86_64__)
19573 # define ARCH_X86_64
19574 # elif defined(__i386__)
19575 # define ARCH_X86_32
19576 # elif defined(__aarch64__)
19577 # define ARCH_ARM64
19578 # elif defined(__arm__)
19579 # define ARCH_ARM32
19580 # elif defined(__riscv)
19581 # define ARCH_RISCV
19582 # endif
19583 #endif
19584  
19585  
19586  
19587  
19588  
19589  
19590 typedef uint8_t u8;
19591 typedef uint16_t u16;
19592 typedef uint32_t u32;
19593 typedef uint64_t u64;
19594 typedef int8_t s8;
19595 typedef int16_t s16;
19596 typedef int32_t s32;
19597 typedef int64_t s64;
19598  
19599  
19600 #ifdef _MSC_VER
19601 # ifdef _WIN64
19602 typedef long long ssize_t;
19603 # else
19604 typedef long ssize_t;
19605 # endif
19606 #endif
19607  
19608  
19609 typedef size_t machine_word_t;
19610  
19611  
19612 #define WORDBYTES ((int)sizeof(machine_word_t))
19613  
19614  
19615 #define WORDBITS (8 * WORDBYTES)
19616  
19617  
19618  
19619  
19620  
19621  
19622 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
19623 # define GCC_PREREQ(major, minor) \
19624 (__GNUC__ > (major) || \
19625 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
19626 # if !GCC_PREREQ(4, 9)
19627 # error "gcc versions older than 4.9 are no longer supported"
19628 # endif
19629 #else
19630 # define GCC_PREREQ(major, minor) 0
19631 #endif
19632 #ifdef __clang__
19633 # ifdef __apple_build_version__
19634 # define CLANG_PREREQ(major, minor, apple_version) \
19635 (__apple_build_version__ >= (apple_version))
19636 # else
19637 # define CLANG_PREREQ(major, minor, apple_version) \
19638 (__clang_major__ > (major) || \
19639 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
19640 # endif
19641 # if !CLANG_PREREQ(3, 9, 8000000)
19642 # error "clang versions older than 3.9 are no longer supported"
19643 # endif
19644 #else
19645 # define CLANG_PREREQ(major, minor, apple_version) 0
19646 #endif
19647 #ifdef _MSC_VER
19648 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
19649 # if !MSVC_PREREQ(1928)
19650 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
19651 # endif
19652 #else
19653 # define MSVC_PREREQ(version) 0
19654 #endif
19655  
19656  
19657 #ifndef __has_attribute
19658 # define __has_attribute(attribute) 0
19659 #endif
19660  
19661  
19662 #ifndef __has_builtin
19663 # define __has_builtin(builtin) 0
19664 #endif
19665  
19666  
19667 #ifdef _MSC_VER
19668 # define inline __inline
19669 #endif
19670  
19671  
19672 #if defined(__GNUC__) || __has_attribute(always_inline)
19673 # define forceinline inline __attribute__((always_inline))
19674 #elif defined(_MSC_VER)
19675 # define forceinline __forceinline
19676 #else
19677 # define forceinline inline
19678 #endif
19679  
19680  
19681 #if defined(__GNUC__) || __has_attribute(unused)
19682 # define MAYBE_UNUSED __attribute__((unused))
19683 #else
19684 # define MAYBE_UNUSED
19685 #endif
19686  
19687  
19688 #if defined(__GNUC__) || __has_attribute(noreturn)
19689 # define NORETURN __attribute__((noreturn))
19690 #else
19691 # define NORETURN
19692 #endif
19693  
19694  
19695 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
19696 # define likely(expr) __builtin_expect(!!(expr), 1)
19697 #else
19698 # define likely(expr) (expr)
19699 #endif
19700  
19701  
19702 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
19703 # define unlikely(expr) __builtin_expect(!!(expr), 0)
19704 #else
19705 # define unlikely(expr) (expr)
19706 #endif
19707  
19708  
19709 #undef prefetchr
19710 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
19711 # define prefetchr(addr) __builtin_prefetch((addr), 0)
19712 #elif defined(_MSC_VER)
19713 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
19714 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
19715 # elif defined(ARCH_ARM64)
19716 # define prefetchr(addr) __prefetch2((addr), 0x00 )
19717 # elif defined(ARCH_ARM32)
19718 # define prefetchr(addr) __prefetch(addr)
19719 # endif
19720 #endif
19721 #ifndef prefetchr
19722 # define prefetchr(addr)
19723 #endif
19724  
19725  
19726 #undef prefetchw
19727 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
19728 # define prefetchw(addr) __builtin_prefetch((addr), 1)
19729 #elif defined(_MSC_VER)
19730 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
19731 # define prefetchw(addr) _m_prefetchw(addr)
19732 # elif defined(ARCH_ARM64)
19733 # define prefetchw(addr) __prefetch2((addr), 0x10 )
19734 # elif defined(ARCH_ARM32)
19735 # define prefetchw(addr) __prefetchw(addr)
19736 # endif
19737 #endif
19738 #ifndef prefetchw
19739 # define prefetchw(addr)
19740 #endif
19741  
19742  
19743 #undef _aligned_attribute
19744 #if defined(__GNUC__) || __has_attribute(aligned)
19745 # define _aligned_attribute(n) __attribute__((aligned(n)))
19746 #elif defined(_MSC_VER)
19747 # define _aligned_attribute(n) __declspec(align(n))
19748 #endif
19749  
19750  
19751 #if defined(__GNUC__) || __has_attribute(target)
19752 # define _target_attribute(attrs) __attribute__((target(attrs)))
19753 #else
19754 # define _target_attribute(attrs)
19755 #endif
19756  
19757  
19758  
19759  
19760  
19761 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
19762 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
19763 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
19764 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
19765 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
19766 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
19767 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
19768  
19769  
19770  
19771  
19772  
19773  
19774 #if defined(__BYTE_ORDER__)
19775 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
19776 #elif defined(_MSC_VER)
19777 # define CPU_IS_LITTLE_ENDIAN() true
19778 #else
19779 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
19780 {
19781 union {
19782 u32 w;
19783 u8 b;
19784 } u;
19785  
19786 u.w = 1;
19787 return u.b;
19788 }
19789 #endif
19790  
19791  
19792 static forceinline u16 bswap16(u16 v)
19793 {
19794 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
19795 return __builtin_bswap16(v);
19796 #elif defined(_MSC_VER)
19797 return _byteswap_ushort(v);
19798 #else
19799 return (v << 8) | (v >> 8);
19800 #endif
19801 }
19802  
19803  
19804 static forceinline u32 bswap32(u32 v)
19805 {
19806 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
19807 return __builtin_bswap32(v);
19808 #elif defined(_MSC_VER)
19809 return _byteswap_ulong(v);
19810 #else
19811 return ((v & 0x000000FF) << 24) |
19812 ((v & 0x0000FF00) << 8) |
19813 ((v & 0x00FF0000) >> 8) |
19814 ((v & 0xFF000000) >> 24);
19815 #endif
19816 }
19817  
19818  
19819 static forceinline u64 bswap64(u64 v)
19820 {
19821 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
19822 return __builtin_bswap64(v);
19823 #elif defined(_MSC_VER)
19824 return _byteswap_uint64(v);
19825 #else
19826 return ((v & 0x00000000000000FF) << 56) |
19827 ((v & 0x000000000000FF00) << 40) |
19828 ((v & 0x0000000000FF0000) << 24) |
19829 ((v & 0x00000000FF000000) << 8) |
19830 ((v & 0x000000FF00000000) >> 8) |
19831 ((v & 0x0000FF0000000000) >> 24) |
19832 ((v & 0x00FF000000000000) >> 40) |
19833 ((v & 0xFF00000000000000) >> 56);
19834 #endif
19835 }
19836  
19837 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
19838 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
19839 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
19840 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
19841 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
19842 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
19843  
19844  
19845  
19846  
19847  
19848  
19849 #if (defined(__GNUC__) || defined(__clang__)) && \
19850 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
19851 defined(__ARM_FEATURE_UNALIGNED) || \
19852 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
19853 defined(__riscv_misaligned_fast) || \
19854 defined(__wasm__))
19855 # define UNALIGNED_ACCESS_IS_FAST 1
19856 #elif defined(_MSC_VER)
19857 # define UNALIGNED_ACCESS_IS_FAST 1
19858 #else
19859 # define UNALIGNED_ACCESS_IS_FAST 0
19860 #endif
19861  
19862  
19863  
19864 #ifdef FREESTANDING
19865 # define MEMCOPY __builtin_memcpy
19866 #else
19867 # define MEMCOPY memcpy
19868 #endif
19869  
19870  
19871  
19872 #define DEFINE_UNALIGNED_TYPE(type) \
19873 static forceinline type \
19874 load_##type##_unaligned(const void *p) \
19875 { \
19876 type v; \
19877 \
19878 MEMCOPY(&v, p, sizeof(v)); \
19879 return v; \
19880 } \
19881 \
19882 static forceinline void \
19883 store_##type##_unaligned(type v, void *p) \
19884 { \
19885 MEMCOPY(p, &v, sizeof(v)); \
19886 }
19887  
19888 DEFINE_UNALIGNED_TYPE(u16)
19889 DEFINE_UNALIGNED_TYPE(u32)
19890 DEFINE_UNALIGNED_TYPE(u64)
19891 DEFINE_UNALIGNED_TYPE(machine_word_t)
19892  
19893 #undef MEMCOPY
19894  
19895 #define load_word_unaligned load_machine_word_t_unaligned
19896 #define store_word_unaligned store_machine_word_t_unaligned
19897  
19898  
19899  
19900 static forceinline u16
19901 get_unaligned_le16(const u8 *p)
19902 {
19903 if (UNALIGNED_ACCESS_IS_FAST)
19904 return le16_bswap(load_u16_unaligned(p));
19905 else
19906 return ((u16)p[1] << 8) | p[0];
19907 }
19908  
19909 static forceinline u16
19910 get_unaligned_be16(const u8 *p)
19911 {
19912 if (UNALIGNED_ACCESS_IS_FAST)
19913 return be16_bswap(load_u16_unaligned(p));
19914 else
19915 return ((u16)p[0] << 8) | p[1];
19916 }
19917  
19918 static forceinline u32
19919 get_unaligned_le32(const u8 *p)
19920 {
19921 if (UNALIGNED_ACCESS_IS_FAST)
19922 return le32_bswap(load_u32_unaligned(p));
19923 else
19924 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
19925 ((u32)p[1] << 8) | p[0];
19926 }
19927  
19928 static forceinline u32
19929 get_unaligned_be32(const u8 *p)
19930 {
19931 if (UNALIGNED_ACCESS_IS_FAST)
19932 return be32_bswap(load_u32_unaligned(p));
19933 else
19934 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
19935 ((u32)p[2] << 8) | p[3];
19936 }
19937  
19938 static forceinline u64
19939 get_unaligned_le64(const u8 *p)
19940 {
19941 if (UNALIGNED_ACCESS_IS_FAST)
19942 return le64_bswap(load_u64_unaligned(p));
19943 else
19944 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
19945 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
19946 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
19947 ((u64)p[1] << 8) | p[0];
19948 }
19949  
19950 static forceinline machine_word_t
19951 get_unaligned_leword(const u8 *p)
19952 {
19953 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
19954 if (WORDBITS == 32)
19955 return get_unaligned_le32(p);
19956 else
19957 return get_unaligned_le64(p);
19958 }
19959  
19960  
19961  
19962 static forceinline void
19963 put_unaligned_le16(u16 v, u8 *p)
19964 {
19965 if (UNALIGNED_ACCESS_IS_FAST) {
19966 store_u16_unaligned(le16_bswap(v), p);
19967 } else {
19968 p[0] = (u8)(v >> 0);
19969 p[1] = (u8)(v >> 8);
19970 }
19971 }
19972  
19973 static forceinline void
19974 put_unaligned_be16(u16 v, u8 *p)
19975 {
19976 if (UNALIGNED_ACCESS_IS_FAST) {
19977 store_u16_unaligned(be16_bswap(v), p);
19978 } else {
19979 p[0] = (u8)(v >> 8);
19980 p[1] = (u8)(v >> 0);
19981 }
19982 }
19983  
19984 static forceinline void
19985 put_unaligned_le32(u32 v, u8 *p)
19986 {
19987 if (UNALIGNED_ACCESS_IS_FAST) {
19988 store_u32_unaligned(le32_bswap(v), p);
19989 } else {
19990 p[0] = (u8)(v >> 0);
19991 p[1] = (u8)(v >> 8);
19992 p[2] = (u8)(v >> 16);
19993 p[3] = (u8)(v >> 24);
19994 }
19995 }
19996  
19997 static forceinline void
19998 put_unaligned_be32(u32 v, u8 *p)
19999 {
20000 if (UNALIGNED_ACCESS_IS_FAST) {
20001 store_u32_unaligned(be32_bswap(v), p);
20002 } else {
20003 p[0] = (u8)(v >> 24);
20004 p[1] = (u8)(v >> 16);
20005 p[2] = (u8)(v >> 8);
20006 p[3] = (u8)(v >> 0);
20007 }
20008 }
20009  
20010 static forceinline void
20011 put_unaligned_le64(u64 v, u8 *p)
20012 {
20013 if (UNALIGNED_ACCESS_IS_FAST) {
20014 store_u64_unaligned(le64_bswap(v), p);
20015 } else {
20016 p[0] = (u8)(v >> 0);
20017 p[1] = (u8)(v >> 8);
20018 p[2] = (u8)(v >> 16);
20019 p[3] = (u8)(v >> 24);
20020 p[4] = (u8)(v >> 32);
20021 p[5] = (u8)(v >> 40);
20022 p[6] = (u8)(v >> 48);
20023 p[7] = (u8)(v >> 56);
20024 }
20025 }
20026  
20027 static forceinline void
20028 put_unaligned_leword(machine_word_t v, u8 *p)
20029 {
20030 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
20031 if (WORDBITS == 32)
20032 put_unaligned_le32(v, p);
20033 else
20034 put_unaligned_le64(v, p);
20035 }
20036  
20037  
20038  
20039  
20040  
20041  
20042  
20043 static forceinline unsigned
20044 bsr32(u32 v)
20045 {
20046 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
20047 return 31 - __builtin_clz(v);
20048 #elif defined(_MSC_VER)
20049 unsigned long i;
20050  
20051 _BitScanReverse(&i, v);
20052 return i;
20053 #else
20054 unsigned i = 0;
20055  
20056 while ((v >>= 1) != 0)
20057 i++;
20058 return i;
20059 #endif
20060 }
20061  
20062 static forceinline unsigned
20063 bsr64(u64 v)
20064 {
20065 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
20066 return 63 - __builtin_clzll(v);
20067 #elif defined(_MSC_VER) && defined(_WIN64)
20068 unsigned long i;
20069  
20070 _BitScanReverse64(&i, v);
20071 return i;
20072 #else
20073 unsigned i = 0;
20074  
20075 while ((v >>= 1) != 0)
20076 i++;
20077 return i;
20078 #endif
20079 }
20080  
20081 static forceinline unsigned
20082 bsrw(machine_word_t v)
20083 {
20084 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
20085 if (WORDBITS == 32)
20086 return bsr32(v);
20087 else
20088 return bsr64(v);
20089 }
20090  
20091  
20092  
20093 static forceinline unsigned
20094 bsf32(u32 v)
20095 {
20096 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
20097 return __builtin_ctz(v);
20098 #elif defined(_MSC_VER)
20099 unsigned long i;
20100  
20101 _BitScanForward(&i, v);
20102 return i;
20103 #else
20104 unsigned i = 0;
20105  
20106 for (; (v & 1) == 0; v >>= 1)
20107 i++;
20108 return i;
20109 #endif
20110 }
20111  
20112 static forceinline unsigned
20113 bsf64(u64 v)
20114 {
20115 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
20116 return __builtin_ctzll(v);
20117 #elif defined(_MSC_VER) && defined(_WIN64)
20118 unsigned long i;
20119  
20120 _BitScanForward64(&i, v);
20121 return i;
20122 #else
20123 unsigned i = 0;
20124  
20125 for (; (v & 1) == 0; v >>= 1)
20126 i++;
20127 return i;
20128 #endif
20129 }
20130  
20131 static forceinline unsigned
20132 bsfw(machine_word_t v)
20133 {
20134 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
20135 if (WORDBITS == 32)
20136 return bsf32(v);
20137 else
20138 return bsf64(v);
20139 }
20140  
20141  
20142 #undef rbit32
20143 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
20144 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
20145 static forceinline u32
20146 rbit32(u32 v)
20147 {
20148 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
20149 return v;
20150 }
20151 #define rbit32 rbit32
20152 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
20153 static forceinline u32
20154 rbit32(u32 v)
20155 {
20156 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
20157 return v;
20158 }
20159 #define rbit32 rbit32
20160 #endif
20161  
20162 #endif
20163  
20164  
20165 typedef void *(*malloc_func_t)(size_t);
20166 typedef void (*free_func_t)(void *);
20167  
20168 extern malloc_func_t libdeflate_default_malloc_func;
20169 extern free_func_t libdeflate_default_free_func;
20170  
20171 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
20172 size_t alignment, size_t size);
20173 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
20174  
20175 #ifdef FREESTANDING
20176  
20177 void *memset(void *s, int c, size_t n);
20178 #define memset(s, c, n) __builtin_memset((s), (c), (n))
20179  
20180 void *memcpy(void *dest, const void *src, size_t n);
20181 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
20182  
20183 void *memmove(void *dest, const void *src, size_t n);
20184 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
20185  
20186 int memcmp(const void *s1, const void *s2, size_t n);
20187 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
20188  
20189 #undef LIBDEFLATE_ENABLE_ASSERTIONS
20190 #else
20191 # include <string.h>
20192
20193 # ifdef __clang_analyzer__
20194 # define LIBDEFLATE_ENABLE_ASSERTIONS
20195 # endif
20196 #endif
20197  
20198  
20199 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
20200 NORETURN void
20201 libdeflate_assertion_failed(const char *expr, const char *file, int line);
20202 #define ASSERT(expr) { if (unlikely(!(expr))) \
20203 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
20204 #else
20205 #define ASSERT(expr) (void)(expr)
20206 #endif
20207  
20208 #define CONCAT_IMPL(a, b) a##b
20209 #define CONCAT(a, b) CONCAT_IMPL(a, b)
20210 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
20211  
20212 #endif
20213  
20214  
20215 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
20216  
20217 #define X86_CPU_FEATURE_SSE2 (1 << 0)
20218 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
20219 #define X86_CPU_FEATURE_AVX (1 << 2)
20220 #define X86_CPU_FEATURE_AVX2 (1 << 3)
20221 #define X86_CPU_FEATURE_BMI2 (1 << 4)
20222  
20223 #define X86_CPU_FEATURE_ZMM (1 << 5)
20224 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
20225 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
20226 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
20227 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
20228 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
20229  
20230 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
20231  
20232 # define X86_CPU_FEATURES_KNOWN (1U << 31)
20233 extern volatile u32 libdeflate_x86_cpu_features;
20234  
20235 void libdeflate_init_x86_cpu_features(void);
20236  
20237 static inline u32 get_x86_cpu_features(void)
20238 {
20239 if (libdeflate_x86_cpu_features == 0)
20240 libdeflate_init_x86_cpu_features();
20241 return libdeflate_x86_cpu_features;
20242 }
20243  
20244 # include <immintrin.h>
20245 # if defined(_MSC_VER) && defined(__clang__)
20246 # include <tmmintrin.h>
20247 # include <smmintrin.h>
20248 # include <wmmintrin.h>
20249 # include <avxintrin.h>
20250 # include <avx2intrin.h>
20251 # include <avx512fintrin.h>
20252 # include <avx512bwintrin.h>
20253 # include <avx512vlintrin.h>
20254 # if __has_include(<avx512vlbwintrin.h>)
20255 # include <avx512vlbwintrin.h>
20256 # endif
20257 # if __has_include(<vpclmulqdqintrin.h>)
20258 # include <vpclmulqdqintrin.h>
20259 # endif
20260 # if __has_include(<avx512vnniintrin.h>)
20261 # include <avx512vnniintrin.h>
20262 # endif
20263 # if __has_include(<avx512vlvnniintrin.h>)
20264 # include <avx512vlvnniintrin.h>
20265 # endif
20266 # if __has_include(<avxvnniintrin.h>)
20267 # include <avxvnniintrin.h>
20268 # endif
20269 # endif
20270 #else
20271 static inline u32 get_x86_cpu_features(void) { return 0; }
20272 #endif
20273  
20274 #if defined(__SSE2__) || \
20275 (defined(_MSC_VER) && \
20276 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
20277 # define HAVE_SSE2(features) 1
20278 # define HAVE_SSE2_NATIVE 1
20279 #else
20280 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
20281 # define HAVE_SSE2_NATIVE 0
20282 #endif
20283  
20284 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
20285 (defined(_MSC_VER) && defined(__AVX2__))
20286 # define HAVE_PCLMULQDQ(features) 1
20287 #else
20288 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
20289 #endif
20290  
20291 #ifdef __AVX__
20292 # define HAVE_AVX(features) 1
20293 #else
20294 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
20295 #endif
20296  
20297 #ifdef __AVX2__
20298 # define HAVE_AVX2(features) 1
20299 #else
20300 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
20301 #endif
20302  
20303 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
20304 # define HAVE_BMI2(features) 1
20305 # define HAVE_BMI2_NATIVE 1
20306 #else
20307 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
20308 # define HAVE_BMI2_NATIVE 0
20309 #endif
20310  
20311 #ifdef __AVX512BW__
20312 # define HAVE_AVX512BW(features) 1
20313 #else
20314 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
20315 #endif
20316  
20317 #ifdef __AVX512VL__
20318 # define HAVE_AVX512VL(features) 1
20319 #else
20320 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
20321 #endif
20322  
20323 #ifdef __VPCLMULQDQ__
20324 # define HAVE_VPCLMULQDQ(features) 1
20325 #else
20326 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
20327 #endif
20328  
20329 #ifdef __AVX512VNNI__
20330 # define HAVE_AVX512VNNI(features) 1
20331 #else
20332 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
20333 #endif
20334  
20335 #ifdef __AVXVNNI__
20336 # define HAVE_AVXVNNI(features) 1
20337 #else
20338 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
20339 #endif
20340  
20341 #endif
20342  
20343 #endif
20344  
20345  
20346 #ifdef __AVX2__
20347 static forceinline void
20348 matchfinder_init_avx2(mf_pos_t *data, size_t size)
20349 {
20350 __m256i *p = (__m256i *)data;
20351 __m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
20352  
20353 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
20354 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
20355 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
20356  
20357 do {
20358 p[0] = v;
20359 p[1] = v;
20360 p[2] = v;
20361 p[3] = v;
20362 p += 4;
20363 size -= 4 * sizeof(*p);
20364 } while (size != 0);
20365 }
20366 #define matchfinder_init matchfinder_init_avx2
20367  
20368 static forceinline void
20369 matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
20370 {
20371 __m256i *p = (__m256i *)data;
20372 __m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
20373  
20374 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
20375 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
20376 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
20377  
20378 do {
20379
20380 p[0] = _mm256_adds_epi16(p[0], v);
20381 p[1] = _mm256_adds_epi16(p[1], v);
20382 p[2] = _mm256_adds_epi16(p[2], v);
20383 p[3] = _mm256_adds_epi16(p[3], v);
20384 p += 4;
20385 size -= 4 * sizeof(*p);
20386 } while (size != 0);
20387 }
20388 #define matchfinder_rebase matchfinder_rebase_avx2
20389  
20390 #elif HAVE_SSE2_NATIVE
20391 static forceinline void
20392 matchfinder_init_sse2(mf_pos_t *data, size_t size)
20393 {
20394 __m128i *p = (__m128i *)data;
20395 __m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
20396  
20397 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
20398 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
20399 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
20400  
20401 do {
20402 p[0] = v;
20403 p[1] = v;
20404 p[2] = v;
20405 p[3] = v;
20406 p += 4;
20407 size -= 4 * sizeof(*p);
20408 } while (size != 0);
20409 }
20410 #define matchfinder_init matchfinder_init_sse2
20411  
20412 static forceinline void
20413 matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
20414 {
20415 __m128i *p = (__m128i *)data;
20416 __m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
20417  
20418 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
20419 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
20420 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
20421  
20422 do {
20423
20424 p[0] = _mm_adds_epi16(p[0], v);
20425 p[1] = _mm_adds_epi16(p[1], v);
20426 p[2] = _mm_adds_epi16(p[2], v);
20427 p[3] = _mm_adds_epi16(p[3], v);
20428 p += 4;
20429 size -= 4 * sizeof(*p);
20430 } while (size != 0);
20431 }
20432 #define matchfinder_rebase matchfinder_rebase_sse2
20433 #endif
20434  
20435 #endif
20436  
20437 # endif
20438 #else
20439 # define MATCHFINDER_ALIGNED
20440 #endif
20441  
20442  
20443 #ifndef matchfinder_init
20444 static forceinline void
20445 matchfinder_init(mf_pos_t *data, size_t size)
20446 {
20447 size_t num_entries = size / sizeof(*data);
20448 size_t i;
20449  
20450 for (i = 0; i < num_entries; i++)
20451 data[i] = MATCHFINDER_INITVAL;
20452 }
20453 #endif
20454  
20455  
20456 #ifndef matchfinder_rebase
20457 static forceinline void
20458 matchfinder_rebase(mf_pos_t *data, size_t size)
20459 {
20460 size_t num_entries = size / sizeof(*data);
20461 size_t i;
20462  
20463 if (MATCHFINDER_WINDOW_SIZE == 32768) {
20464
20465 for (i = 0; i < num_entries; i++)
20466 data[i] = 0x8000 | (data[i] & ~(data[i] >> 15));
20467 } else {
20468 for (i = 0; i < num_entries; i++) {
20469 if (data[i] >= 0)
20470 data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
20471 else
20472 data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
20473 }
20474 }
20475 }
20476 #endif
20477  
20478  
20479 static forceinline u32
20480 lz_hash(u32 seq, unsigned num_bits)
20481 {
20482 return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
20483 }
20484  
20485  
20486 static forceinline u32
20487 lz_extend(const u8 * const strptr, const u8 * const matchptr,
20488 const u32 start_len, const u32 max_len)
20489 {
20490 u32 len = start_len;
20491 machine_word_t v_word;
20492  
20493 if (UNALIGNED_ACCESS_IS_FAST) {
20494  
20495 if (likely(max_len - len >= 4 * WORDBYTES)) {
20496  
20497 #define COMPARE_WORD_STEP \
20498 v_word = load_word_unaligned(&matchptr[len]) ^ \
20499 load_word_unaligned(&strptr[len]); \
20500 if (v_word != 0) \
20501 goto word_differs; \
20502 len += WORDBYTES; \
20503  
20504 COMPARE_WORD_STEP
20505 COMPARE_WORD_STEP
20506 COMPARE_WORD_STEP
20507 COMPARE_WORD_STEP
20508 #undef COMPARE_WORD_STEP
20509 }
20510  
20511 while (len + WORDBYTES <= max_len) {
20512 v_word = load_word_unaligned(&matchptr[len]) ^
20513 load_word_unaligned(&strptr[len]);
20514 if (v_word != 0)
20515 goto word_differs;
20516 len += WORDBYTES;
20517 }
20518 }
20519  
20520 while (len < max_len && matchptr[len] == strptr[len])
20521 len++;
20522 return len;
20523  
20524 word_differs:
20525 if (CPU_IS_LITTLE_ENDIAN())
20526 len += (bsfw(v_word) >> 3);
20527 else
20528 len += (WORDBITS - 1 - bsrw(v_word)) >> 3;
20529 return len;
20530 }
20531  
20532 #endif
20533  
20534  
20535 #define HT_MATCHFINDER_HASH_ORDER 15
20536 #define HT_MATCHFINDER_BUCKET_SIZE 2
20537  
20538 #define HT_MATCHFINDER_MIN_MATCH_LEN 4
20539  
20540 #define HT_MATCHFINDER_REQUIRED_NBYTES 5
20541  
20542 struct MATCHFINDER_ALIGNED ht_matchfinder {
20543 mf_pos_t hash_tab[1UL << HT_MATCHFINDER_HASH_ORDER]
20544 [HT_MATCHFINDER_BUCKET_SIZE];
20545 };
20546  
20547 static forceinline void
20548 ht_matchfinder_init(struct ht_matchfinder *mf)
20549 {
20550 STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
20551  
20552 matchfinder_init((mf_pos_t *)mf, sizeof(*mf));
20553 3 }
20554  
20555 static forceinline void
20556 ht_matchfinder_slide_window(struct ht_matchfinder *mf)
20557 {
20558 matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
20559 0 }
20560  
20561  
20562 static forceinline u32
20563 ht_matchfinder_longest_match(struct ht_matchfinder * const mf,
20564 const u8 ** const in_base_p,
20565 const u8 * const in_next,
20566 const u32 max_len,
20567 const u32 nice_len,
20568 u32 * const next_hash,
20569 u32 * const offset_ret)
20570 {
20571 75 u32 best_len = 0;
20572 75 const u8 *best_matchptr = in_next;
20573 75 u32 cur_pos = in_next - *in_base_p;
20574 const u8 *in_base;
20575 mf_pos_t cutoff;
20576 u32 hash;
20577 u32 seq;
20578 mf_pos_t cur_node;
20579 const u8 *matchptr;
20580 #if HT_MATCHFINDER_BUCKET_SIZE > 1
20581 mf_pos_t to_insert;
20582 u32 len;
20583 #endif
20584 #if HT_MATCHFINDER_BUCKET_SIZE > 2
20585 int i;
20586 #endif
20587  
20588
20589 STATIC_ASSERT(HT_MATCHFINDER_MIN_MATCH_LEN == 4);
20590  
20591 75 if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
20592 ht_matchfinder_slide_window(mf);
20593 0 *in_base_p += MATCHFINDER_WINDOW_SIZE;
20594 0 cur_pos = 0;
20595 }
20596 75 in_base = *in_base_p;
20597 75 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
20598  
20599 75 hash = *next_hash;
20600 STATIC_ASSERT(HT_MATCHFINDER_REQUIRED_NBYTES == 5);
20601 225 *next_hash = lz_hash(get_unaligned_le32(in_next + 1),
20602 HT_MATCHFINDER_HASH_ORDER);
20603 75 seq = load_u32_unaligned(in_next);
20604 75 prefetchw(&mf->hash_tab[*next_hash]);
20605 #if HT_MATCHFINDER_BUCKET_SIZE == 1
20606
20607 cur_node = mf->hash_tab[hash][0];
20608 mf->hash_tab[hash][0] = cur_pos;
20609 if (cur_node <= cutoff)
20610 goto out;
20611 matchptr = &in_base[cur_node];
20612 if (load_u32_unaligned(matchptr) == seq) {
20613 best_len = lz_extend(in_next, matchptr, 4, max_len);
20614 best_matchptr = matchptr;
20615 }
20616 #elif HT_MATCHFINDER_BUCKET_SIZE == 2
20617
20618 75 cur_node = mf->hash_tab[hash][0];
20619 75 mf->hash_tab[hash][0] = cur_pos;
20620 75 ▶ if (cur_node <= cutoff)
Branch: 1 branch
BranchTrueFalseTotal
if (cur_node <= cutoff) 54 21 75
20621 54 goto out;
20622 21 matchptr = &in_base[cur_node];
20623  
20624 21 to_insert = cur_node;
20625 21 cur_node = mf->hash_tab[hash][1];
20626 21 mf->hash_tab[hash][1] = to_insert;
20627  
20628 21 ▶ if (load_u32_unaligned(matchptr) == seq) {
Branch: 1 branch
BranchTrueFalseTotal
if (load_u32_unaligned(matchptr) == seq) { 21 0 21
20629 21 best_len = lz_extend(in_next, matchptr, 4, max_len);
20630 21 best_matchptr = matchptr;
20631 21 ▶ if (cur_node <= cutoff || best_len >= nice_len)
Branch: 2 branches
BranchTrueFalseTotal
if (cur_node <= cutoff || best_len >= nice_len) 18 3 21
if (cur_node <= cutoff || best_len >= nice_len) 18 0 18
20632 21 goto out;
20633 0 matchptr = &in_base[cur_node];
20634 0 ▶ if (load_u32_unaligned(matchptr) == seq &&
Branch: 1 branch
BranchTrueFalseTotal
if (load_u32_unaligned(matchptr) == seq && 0 0 0
20635 0 load_u32_unaligned(matchptr + best_len - 3) ==
20636 0 ▶ load_u32_unaligned(in_next + best_len - 3)) {
Branch: 1 branch
BranchTrueFalseTotal
load_u32_unaligned(in_next + best_len - 3)) { 0 0 0
20637 0 len = lz_extend(in_next, matchptr, 4, max_len);
20638 0 ▶ if (len > best_len) {
Branch: 1 branch
BranchTrueFalseTotal
if (len > best_len) { 0 0 0
20639 0 best_len = len;
20640 0 best_matchptr = matchptr;
20641 }
20642 }
20643 } else {
20644 0 ▶ if (cur_node <= cutoff)
Branch: 1 branch
BranchTrueFalseTotal
if (cur_node <= cutoff) 0 0 0
20645 0 goto out;
20646 0 matchptr = &in_base[cur_node];
20647 0 ▶ if (load_u32_unaligned(matchptr) == seq) {
Branch: 1 branch
BranchTrueFalseTotal
if (load_u32_unaligned(matchptr) == seq) { 0 0 0
20648 0 best_len = lz_extend(in_next, matchptr, 4, max_len);
20649 0 best_matchptr = matchptr;
20650 }
20651 }
20652 #else
20653
20654 to_insert = cur_pos;
20655 for (i = 0; i < HT_MATCHFINDER_BUCKET_SIZE; i++) {
20656 cur_node = mf->hash_tab[hash][i];
20657 mf->hash_tab[hash][i] = to_insert;
20658 if (cur_node <= cutoff)
20659 goto out;
20660 matchptr = &in_base[cur_node];
20661 if (load_u32_unaligned(matchptr) == seq) {
20662 len = lz_extend(in_next, matchptr, 4, max_len);
20663 if (len > best_len) {
20664 best_len = len;
20665 best_matchptr = matchptr;
20666 if (best_len >= nice_len)
20667 goto out;
20668 }
20669 }
20670 to_insert = cur_node;
20671 }
20672 #endif
20673 0 out:
20674 75 *offset_ret = in_next - best_matchptr;
20675 75 return best_len;
20676 }
20677  
20678 static forceinline void
20679 ht_matchfinder_skip_bytes(struct ht_matchfinder * const mf,
20680 const u8 ** const in_base_p,
20681 const u8 *in_next,
20682 const u8 * const in_end,
20683 const u32 count,
20684 u32 * const next_hash)
20685 {
20686 21 s32 cur_pos = in_next - *in_base_p;
20687 u32 hash;
20688 21 u32 remaining = count;
20689 int i;
20690  
20691 21 if (unlikely(count + HT_MATCHFINDER_REQUIRED_NBYTES > in_end - in_next))
20692 3 return;
20693  
20694 18 ▶ if (cur_pos + count - 1 >= MATCHFINDER_WINDOW_SIZE) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur_pos + count - 1 >= MATCHFINDER_WINDOW_SIZE) { 0 18 18
20695 ht_matchfinder_slide_window(mf);
20696 0 *in_base_p += MATCHFINDER_WINDOW_SIZE;
20697 0 cur_pos -= MATCHFINDER_WINDOW_SIZE;
20698 }
20699  
20700 18 hash = *next_hash;
20701 do {
20702 9252 ▶ for (i = HT_MATCHFINDER_BUCKET_SIZE - 1; i > 0; i--)
Branch: 1 branch
BranchTrueFalseTotal
for (i = HT_MATCHFINDER_BUCKET_SIZE - 1; i > 0; i--) 4626 4626 9252
20703 4626 mf->hash_tab[hash][i] = mf->hash_tab[hash][i - 1];
20704 4626 mf->hash_tab[hash][0] = cur_pos;
20705  
20706 9252 hash = lz_hash(get_unaligned_le32(++in_next),
20707 HT_MATCHFINDER_HASH_ORDER);
20708 4626 cur_pos++;
20709 4626 ▶ } while (--remaining);
Branch: 1 branch
BranchTrueFalseTotal
} while (--remaining); 4608 18 4626
20710  
20711 18 prefetchw(&mf->hash_tab[hash]);
20712 18 *next_hash = hash;
20713 }
20714  
20715 #endif
20716  
20717 #if SUPPORT_NEAR_OPTIMAL_PARSING
20718 /* # include "bt_matchfinder.h" */
20719  
20720  
20721 #ifndef LIB_BT_MATCHFINDER_H
20722 #define LIB_BT_MATCHFINDER_H
20723  
20724 /* #include "matchfinder_common.h" */
20725  
20726  
20727 #ifndef LIB_MATCHFINDER_COMMON_H
20728 #define LIB_MATCHFINDER_COMMON_H
20729  
20730 /* #include "lib_common.h" */
20731  
20732  
20733 #ifndef LIB_LIB_COMMON_H
20734 #define LIB_LIB_COMMON_H
20735  
20736 #ifdef LIBDEFLATE_H
20737
20738 # error "lib_common.h must always be included before libdeflate.h"
20739 #endif
20740  
20741 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
20742 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
20743 #elif defined(__GNUC__)
20744 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
20745 #else
20746 # define LIBDEFLATE_EXPORT_SYM
20747 #endif
20748  
20749  
20750 #if defined(__GNUC__) && defined(__i386__)
20751 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
20752 #else
20753 # define LIBDEFLATE_ALIGN_STACK
20754 #endif
20755  
20756 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
20757  
20758 /* #include "../common_defs.h" */
20759  
20760  
20761 #ifndef COMMON_DEFS_H
20762 #define COMMON_DEFS_H
20763  
20764 /* #include "libdeflate.h" */
20765  
20766  
20767 #ifndef LIBDEFLATE_H
20768 #define LIBDEFLATE_H
20769  
20770 #include <stddef.h>
20771 #include <stdint.h>
20772  
20773 #ifdef __cplusplus
20774 extern "C" {
20775 #endif
20776  
20777 #define LIBDEFLATE_VERSION_MAJOR 1
20778 #define LIBDEFLATE_VERSION_MINOR 26
20779 #define LIBDEFLATE_VERSION_STRING "1.26"
20780  
20781  
20782 #ifndef LIBDEFLATEAPI
20783 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
20784 # define LIBDEFLATEAPI __declspec(dllimport)
20785 # else
20786 # define LIBDEFLATEAPI
20787 # endif
20788 #endif
20789  
20790  
20791  
20792  
20793  
20794 struct libdeflate_compressor;
20795 struct libdeflate_options;
20796  
20797  
20798 LIBDEFLATEAPI struct libdeflate_compressor *
20799 libdeflate_alloc_compressor(int compression_level);
20800  
20801  
20802 LIBDEFLATEAPI struct libdeflate_compressor *
20803 libdeflate_alloc_compressor_ex(int compression_level,
20804 const struct libdeflate_options *options);
20805  
20806  
20807 LIBDEFLATEAPI size_t
20808 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
20809 const void *in, size_t in_nbytes,
20810 void *out, size_t out_nbytes_avail);
20811  
20812  
20813 LIBDEFLATEAPI size_t
20814 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
20815 size_t in_nbytes);
20816  
20817  
20818 LIBDEFLATEAPI size_t
20819 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
20820 const void *in, size_t in_nbytes,
20821 void *out, size_t out_nbytes_avail);
20822  
20823  
20824 LIBDEFLATEAPI size_t
20825 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
20826 size_t in_nbytes);
20827  
20828  
20829 LIBDEFLATEAPI size_t
20830 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
20831 const void *in, size_t in_nbytes,
20832 void *out, size_t out_nbytes_avail);
20833  
20834  
20835 LIBDEFLATEAPI size_t
20836 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
20837 size_t in_nbytes);
20838  
20839  
20840 LIBDEFLATEAPI void
20841 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
20842  
20843  
20844  
20845  
20846  
20847 struct libdeflate_decompressor;
20848 struct libdeflate_options;
20849  
20850  
20851 LIBDEFLATEAPI struct libdeflate_decompressor *
20852 libdeflate_alloc_decompressor(void);
20853  
20854  
20855 LIBDEFLATEAPI struct libdeflate_decompressor *
20856 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
20857  
20858  
20859 enum libdeflate_result {
20860
20861 LIBDEFLATE_SUCCESS = 0,
20862  
20863
20864 LIBDEFLATE_BAD_DATA = 1,
20865  
20866
20867 LIBDEFLATE_SHORT_OUTPUT = 2,
20868  
20869
20870 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
20871 };
20872  
20873  
20874 LIBDEFLATEAPI enum libdeflate_result
20875 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
20876 const void *in, size_t in_nbytes,
20877 void *out, size_t out_nbytes_avail,
20878 size_t *actual_out_nbytes_ret);
20879  
20880  
20881 LIBDEFLATEAPI enum libdeflate_result
20882 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
20883 const void *in, size_t in_nbytes,
20884 void *out, size_t out_nbytes_avail,
20885 size_t *actual_in_nbytes_ret,
20886 size_t *actual_out_nbytes_ret);
20887  
20888  
20889 LIBDEFLATEAPI enum libdeflate_result
20890 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
20891 const void *in, size_t in_nbytes,
20892 void *out, size_t out_nbytes_avail,
20893 size_t *actual_out_nbytes_ret);
20894  
20895  
20896 LIBDEFLATEAPI enum libdeflate_result
20897 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
20898 const void *in, size_t in_nbytes,
20899 void *out, size_t out_nbytes_avail,
20900 size_t *actual_in_nbytes_ret,
20901 size_t *actual_out_nbytes_ret);
20902  
20903  
20904 LIBDEFLATEAPI enum libdeflate_result
20905 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
20906 const void *in, size_t in_nbytes,
20907 void *out, size_t out_nbytes_avail,
20908 size_t *actual_out_nbytes_ret);
20909  
20910  
20911 LIBDEFLATEAPI enum libdeflate_result
20912 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
20913 const void *in, size_t in_nbytes,
20914 void *out, size_t out_nbytes_avail,
20915 size_t *actual_in_nbytes_ret,
20916 size_t *actual_out_nbytes_ret);
20917  
20918  
20919 LIBDEFLATEAPI void
20920 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
20921  
20922  
20923  
20924  
20925  
20926  
20927 LIBDEFLATEAPI uint32_t
20928 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
20929  
20930  
20931  
20932 LIBDEFLATEAPI uint32_t
20933 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
20934  
20935  
20936  
20937  
20938  
20939  
20940 LIBDEFLATEAPI void
20941 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
20942 void (*free_func)(void *));
20943  
20944  
20945 struct libdeflate_options {
20946  
20947
20948 size_t sizeof_options;
20949  
20950
20951 void *(*malloc_func)(size_t);
20952 void (*free_func)(void *);
20953 };
20954  
20955 #ifdef __cplusplus
20956 }
20957 #endif
20958  
20959 #endif
20960  
20961  
20962 #include <stdbool.h>
20963 #include <stddef.h>
20964 #include <stdint.h>
20965 #ifdef _MSC_VER
20966 # include <intrin.h>
20967 # include <stdlib.h>
20968
20969
20970 # pragma warning(disable : 4146)
20971
20972 # pragma warning(disable : 4018)
20973 # pragma warning(disable : 4244)
20974 # pragma warning(disable : 4267)
20975 # pragma warning(disable : 4310)
20976
20977 # pragma warning(disable : 4100)
20978 # pragma warning(disable : 4127)
20979 # pragma warning(disable : 4189)
20980 # pragma warning(disable : 4232)
20981 # pragma warning(disable : 4245)
20982 # pragma warning(disable : 4295)
20983 #endif
20984 #ifndef FREESTANDING
20985 # include <string.h>
20986 #endif
20987  
20988  
20989  
20990  
20991  
20992  
20993 #undef ARCH_X86_64
20994 #undef ARCH_X86_32
20995 #undef ARCH_ARM64
20996 #undef ARCH_ARM32
20997 #undef ARCH_RISCV
20998 #ifdef _MSC_VER
20999
21000 # if defined(_M_X64) && !defined(_M_ARM64EC)
21001 # define ARCH_X86_64
21002 # elif defined(_M_IX86)
21003 # define ARCH_X86_32
21004 # elif defined(_M_ARM64)
21005 # define ARCH_ARM64
21006 # elif defined(_M_ARM)
21007 # define ARCH_ARM32
21008 # endif
21009 #else
21010 # if defined(__x86_64__)
21011 # define ARCH_X86_64
21012 # elif defined(__i386__)
21013 # define ARCH_X86_32
21014 # elif defined(__aarch64__)
21015 # define ARCH_ARM64
21016 # elif defined(__arm__)
21017 # define ARCH_ARM32
21018 # elif defined(__riscv)
21019 # define ARCH_RISCV
21020 # endif
21021 #endif
21022  
21023  
21024  
21025  
21026  
21027  
21028 typedef uint8_t u8;
21029 typedef uint16_t u16;
21030 typedef uint32_t u32;
21031 typedef uint64_t u64;
21032 typedef int8_t s8;
21033 typedef int16_t s16;
21034 typedef int32_t s32;
21035 typedef int64_t s64;
21036  
21037  
21038 #ifdef _MSC_VER
21039 # ifdef _WIN64
21040 typedef long long ssize_t;
21041 # else
21042 typedef long ssize_t;
21043 # endif
21044 #endif
21045  
21046  
21047 typedef size_t machine_word_t;
21048  
21049  
21050 #define WORDBYTES ((int)sizeof(machine_word_t))
21051  
21052  
21053 #define WORDBITS (8 * WORDBYTES)
21054  
21055  
21056  
21057  
21058  
21059  
21060 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
21061 # define GCC_PREREQ(major, minor) \
21062 (__GNUC__ > (major) || \
21063 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
21064 # if !GCC_PREREQ(4, 9)
21065 # error "gcc versions older than 4.9 are no longer supported"
21066 # endif
21067 #else
21068 # define GCC_PREREQ(major, minor) 0
21069 #endif
21070 #ifdef __clang__
21071 # ifdef __apple_build_version__
21072 # define CLANG_PREREQ(major, minor, apple_version) \
21073 (__apple_build_version__ >= (apple_version))
21074 # else
21075 # define CLANG_PREREQ(major, minor, apple_version) \
21076 (__clang_major__ > (major) || \
21077 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
21078 # endif
21079 # if !CLANG_PREREQ(3, 9, 8000000)
21080 # error "clang versions older than 3.9 are no longer supported"
21081 # endif
21082 #else
21083 # define CLANG_PREREQ(major, minor, apple_version) 0
21084 #endif
21085 #ifdef _MSC_VER
21086 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
21087 # if !MSVC_PREREQ(1928)
21088 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
21089 # endif
21090 #else
21091 # define MSVC_PREREQ(version) 0
21092 #endif
21093  
21094  
21095 #ifndef __has_attribute
21096 # define __has_attribute(attribute) 0
21097 #endif
21098  
21099  
21100 #ifndef __has_builtin
21101 # define __has_builtin(builtin) 0
21102 #endif
21103  
21104  
21105 #ifdef _MSC_VER
21106 # define inline __inline
21107 #endif
21108  
21109  
21110 #if defined(__GNUC__) || __has_attribute(always_inline)
21111 # define forceinline inline __attribute__((always_inline))
21112 #elif defined(_MSC_VER)
21113 # define forceinline __forceinline
21114 #else
21115 # define forceinline inline
21116 #endif
21117  
21118  
21119 #if defined(__GNUC__) || __has_attribute(unused)
21120 # define MAYBE_UNUSED __attribute__((unused))
21121 #else
21122 # define MAYBE_UNUSED
21123 #endif
21124  
21125  
21126 #if defined(__GNUC__) || __has_attribute(noreturn)
21127 # define NORETURN __attribute__((noreturn))
21128 #else
21129 # define NORETURN
21130 #endif
21131  
21132  
21133 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
21134 # define likely(expr) __builtin_expect(!!(expr), 1)
21135 #else
21136 # define likely(expr) (expr)
21137 #endif
21138  
21139  
21140 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
21141 # define unlikely(expr) __builtin_expect(!!(expr), 0)
21142 #else
21143 # define unlikely(expr) (expr)
21144 #endif
21145  
21146  
21147 #undef prefetchr
21148 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
21149 # define prefetchr(addr) __builtin_prefetch((addr), 0)
21150 #elif defined(_MSC_VER)
21151 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
21152 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
21153 # elif defined(ARCH_ARM64)
21154 # define prefetchr(addr) __prefetch2((addr), 0x00 )
21155 # elif defined(ARCH_ARM32)
21156 # define prefetchr(addr) __prefetch(addr)
21157 # endif
21158 #endif
21159 #ifndef prefetchr
21160 # define prefetchr(addr)
21161 #endif
21162  
21163  
21164 #undef prefetchw
21165 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
21166 # define prefetchw(addr) __builtin_prefetch((addr), 1)
21167 #elif defined(_MSC_VER)
21168 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
21169 # define prefetchw(addr) _m_prefetchw(addr)
21170 # elif defined(ARCH_ARM64)
21171 # define prefetchw(addr) __prefetch2((addr), 0x10 )
21172 # elif defined(ARCH_ARM32)
21173 # define prefetchw(addr) __prefetchw(addr)
21174 # endif
21175 #endif
21176 #ifndef prefetchw
21177 # define prefetchw(addr)
21178 #endif
21179  
21180  
21181 #undef _aligned_attribute
21182 #if defined(__GNUC__) || __has_attribute(aligned)
21183 # define _aligned_attribute(n) __attribute__((aligned(n)))
21184 #elif defined(_MSC_VER)
21185 # define _aligned_attribute(n) __declspec(align(n))
21186 #endif
21187  
21188  
21189 #if defined(__GNUC__) || __has_attribute(target)
21190 # define _target_attribute(attrs) __attribute__((target(attrs)))
21191 #else
21192 # define _target_attribute(attrs)
21193 #endif
21194  
21195  
21196  
21197  
21198  
21199 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
21200 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
21201 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
21202 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
21203 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
21204 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
21205 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
21206  
21207  
21208  
21209  
21210  
21211  
21212 #if defined(__BYTE_ORDER__)
21213 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
21214 #elif defined(_MSC_VER)
21215 # define CPU_IS_LITTLE_ENDIAN() true
21216 #else
21217 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
21218 {
21219 union {
21220 u32 w;
21221 u8 b;
21222 } u;
21223  
21224 u.w = 1;
21225 return u.b;
21226 }
21227 #endif
21228  
21229  
21230 static forceinline u16 bswap16(u16 v)
21231 {
21232 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
21233 return __builtin_bswap16(v);
21234 #elif defined(_MSC_VER)
21235 return _byteswap_ushort(v);
21236 #else
21237 return (v << 8) | (v >> 8);
21238 #endif
21239 }
21240  
21241  
21242 static forceinline u32 bswap32(u32 v)
21243 {
21244 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
21245 return __builtin_bswap32(v);
21246 #elif defined(_MSC_VER)
21247 return _byteswap_ulong(v);
21248 #else
21249 return ((v & 0x000000FF) << 24) |
21250 ((v & 0x0000FF00) << 8) |
21251 ((v & 0x00FF0000) >> 8) |
21252 ((v & 0xFF000000) >> 24);
21253 #endif
21254 }
21255  
21256  
21257 static forceinline u64 bswap64(u64 v)
21258 {
21259 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
21260 return __builtin_bswap64(v);
21261 #elif defined(_MSC_VER)
21262 return _byteswap_uint64(v);
21263 #else
21264 return ((v & 0x00000000000000FF) << 56) |
21265 ((v & 0x000000000000FF00) << 40) |
21266 ((v & 0x0000000000FF0000) << 24) |
21267 ((v & 0x00000000FF000000) << 8) |
21268 ((v & 0x000000FF00000000) >> 8) |
21269 ((v & 0x0000FF0000000000) >> 24) |
21270 ((v & 0x00FF000000000000) >> 40) |
21271 ((v & 0xFF00000000000000) >> 56);
21272 #endif
21273 }
21274  
21275 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
21276 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
21277 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
21278 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
21279 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
21280 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
21281  
21282  
21283  
21284  
21285  
21286  
21287 #if (defined(__GNUC__) || defined(__clang__)) && \
21288 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
21289 defined(__ARM_FEATURE_UNALIGNED) || \
21290 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
21291 defined(__riscv_misaligned_fast) || \
21292 defined(__wasm__))
21293 # define UNALIGNED_ACCESS_IS_FAST 1
21294 #elif defined(_MSC_VER)
21295 # define UNALIGNED_ACCESS_IS_FAST 1
21296 #else
21297 # define UNALIGNED_ACCESS_IS_FAST 0
21298 #endif
21299  
21300  
21301  
21302 #ifdef FREESTANDING
21303 # define MEMCOPY __builtin_memcpy
21304 #else
21305 # define MEMCOPY memcpy
21306 #endif
21307  
21308  
21309  
21310 #define DEFINE_UNALIGNED_TYPE(type) \
21311 static forceinline type \
21312 load_##type##_unaligned(const void *p) \
21313 { \
21314 type v; \
21315 \
21316 MEMCOPY(&v, p, sizeof(v)); \
21317 return v; \
21318 } \
21319 \
21320 static forceinline void \
21321 store_##type##_unaligned(type v, void *p) \
21322 { \
21323 MEMCOPY(p, &v, sizeof(v)); \
21324 }
21325  
21326 DEFINE_UNALIGNED_TYPE(u16)
21327 DEFINE_UNALIGNED_TYPE(u32)
21328 DEFINE_UNALIGNED_TYPE(u64)
21329 DEFINE_UNALIGNED_TYPE(machine_word_t)
21330  
21331 #undef MEMCOPY
21332  
21333 #define load_word_unaligned load_machine_word_t_unaligned
21334 #define store_word_unaligned store_machine_word_t_unaligned
21335  
21336  
21337  
21338 static forceinline u16
21339 get_unaligned_le16(const u8 *p)
21340 {
21341 if (UNALIGNED_ACCESS_IS_FAST)
21342 return le16_bswap(load_u16_unaligned(p));
21343 else
21344 return ((u16)p[1] << 8) | p[0];
21345 }
21346  
21347 static forceinline u16
21348 get_unaligned_be16(const u8 *p)
21349 {
21350 if (UNALIGNED_ACCESS_IS_FAST)
21351 return be16_bswap(load_u16_unaligned(p));
21352 else
21353 return ((u16)p[0] << 8) | p[1];
21354 }
21355  
21356 static forceinline u32
21357 get_unaligned_le32(const u8 *p)
21358 {
21359 if (UNALIGNED_ACCESS_IS_FAST)
21360 return le32_bswap(load_u32_unaligned(p));
21361 else
21362 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
21363 ((u32)p[1] << 8) | p[0];
21364 }
21365  
21366 static forceinline u32
21367 get_unaligned_be32(const u8 *p)
21368 {
21369 if (UNALIGNED_ACCESS_IS_FAST)
21370 return be32_bswap(load_u32_unaligned(p));
21371 else
21372 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
21373 ((u32)p[2] << 8) | p[3];
21374 }
21375  
21376 static forceinline u64
21377 get_unaligned_le64(const u8 *p)
21378 {
21379 if (UNALIGNED_ACCESS_IS_FAST)
21380 return le64_bswap(load_u64_unaligned(p));
21381 else
21382 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
21383 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
21384 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
21385 ((u64)p[1] << 8) | p[0];
21386 }
21387  
21388 static forceinline machine_word_t
21389 get_unaligned_leword(const u8 *p)
21390 {
21391 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
21392 if (WORDBITS == 32)
21393 return get_unaligned_le32(p);
21394 else
21395 return get_unaligned_le64(p);
21396 }
21397  
21398  
21399  
21400 static forceinline void
21401 put_unaligned_le16(u16 v, u8 *p)
21402 {
21403 if (UNALIGNED_ACCESS_IS_FAST) {
21404 store_u16_unaligned(le16_bswap(v), p);
21405 } else {
21406 p[0] = (u8)(v >> 0);
21407 p[1] = (u8)(v >> 8);
21408 }
21409 }
21410  
21411 static forceinline void
21412 put_unaligned_be16(u16 v, u8 *p)
21413 {
21414 if (UNALIGNED_ACCESS_IS_FAST) {
21415 store_u16_unaligned(be16_bswap(v), p);
21416 } else {
21417 p[0] = (u8)(v >> 8);
21418 p[1] = (u8)(v >> 0);
21419 }
21420 }
21421  
21422 static forceinline void
21423 put_unaligned_le32(u32 v, u8 *p)
21424 {
21425 if (UNALIGNED_ACCESS_IS_FAST) {
21426 store_u32_unaligned(le32_bswap(v), p);
21427 } else {
21428 p[0] = (u8)(v >> 0);
21429 p[1] = (u8)(v >> 8);
21430 p[2] = (u8)(v >> 16);
21431 p[3] = (u8)(v >> 24);
21432 }
21433 }
21434  
21435 static forceinline void
21436 put_unaligned_be32(u32 v, u8 *p)
21437 {
21438 if (UNALIGNED_ACCESS_IS_FAST) {
21439 store_u32_unaligned(be32_bswap(v), p);
21440 } else {
21441 p[0] = (u8)(v >> 24);
21442 p[1] = (u8)(v >> 16);
21443 p[2] = (u8)(v >> 8);
21444 p[3] = (u8)(v >> 0);
21445 }
21446 }
21447  
21448 static forceinline void
21449 put_unaligned_le64(u64 v, u8 *p)
21450 {
21451 if (UNALIGNED_ACCESS_IS_FAST) {
21452 store_u64_unaligned(le64_bswap(v), p);
21453 } else {
21454 p[0] = (u8)(v >> 0);
21455 p[1] = (u8)(v >> 8);
21456 p[2] = (u8)(v >> 16);
21457 p[3] = (u8)(v >> 24);
21458 p[4] = (u8)(v >> 32);
21459 p[5] = (u8)(v >> 40);
21460 p[6] = (u8)(v >> 48);
21461 p[7] = (u8)(v >> 56);
21462 }
21463 }
21464  
21465 static forceinline void
21466 put_unaligned_leword(machine_word_t v, u8 *p)
21467 {
21468 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
21469 if (WORDBITS == 32)
21470 put_unaligned_le32(v, p);
21471 else
21472 put_unaligned_le64(v, p);
21473 }
21474  
21475  
21476  
21477  
21478  
21479  
21480  
21481 static forceinline unsigned
21482 bsr32(u32 v)
21483 {
21484 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
21485 return 31 - __builtin_clz(v);
21486 #elif defined(_MSC_VER)
21487 unsigned long i;
21488  
21489 _BitScanReverse(&i, v);
21490 return i;
21491 #else
21492 unsigned i = 0;
21493  
21494 while ((v >>= 1) != 0)
21495 i++;
21496 return i;
21497 #endif
21498 }
21499  
21500 static forceinline unsigned
21501 bsr64(u64 v)
21502 {
21503 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
21504 return 63 - __builtin_clzll(v);
21505 #elif defined(_MSC_VER) && defined(_WIN64)
21506 unsigned long i;
21507  
21508 _BitScanReverse64(&i, v);
21509 return i;
21510 #else
21511 unsigned i = 0;
21512  
21513 while ((v >>= 1) != 0)
21514 i++;
21515 return i;
21516 #endif
21517 }
21518  
21519 static forceinline unsigned
21520 bsrw(machine_word_t v)
21521 {
21522 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
21523 if (WORDBITS == 32)
21524 return bsr32(v);
21525 else
21526 return bsr64(v);
21527 }
21528  
21529  
21530  
21531 static forceinline unsigned
21532 bsf32(u32 v)
21533 {
21534 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
21535 return __builtin_ctz(v);
21536 #elif defined(_MSC_VER)
21537 unsigned long i;
21538  
21539 _BitScanForward(&i, v);
21540 return i;
21541 #else
21542 unsigned i = 0;
21543  
21544 for (; (v & 1) == 0; v >>= 1)
21545 i++;
21546 return i;
21547 #endif
21548 }
21549  
21550 static forceinline unsigned
21551 bsf64(u64 v)
21552 {
21553 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
21554 return __builtin_ctzll(v);
21555 #elif defined(_MSC_VER) && defined(_WIN64)
21556 unsigned long i;
21557  
21558 _BitScanForward64(&i, v);
21559 return i;
21560 #else
21561 unsigned i = 0;
21562  
21563 for (; (v & 1) == 0; v >>= 1)
21564 i++;
21565 return i;
21566 #endif
21567 }
21568  
21569 static forceinline unsigned
21570 bsfw(machine_word_t v)
21571 {
21572 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
21573 if (WORDBITS == 32)
21574 return bsf32(v);
21575 else
21576 return bsf64(v);
21577 }
21578  
21579  
21580 #undef rbit32
21581 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
21582 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
21583 static forceinline u32
21584 rbit32(u32 v)
21585 {
21586 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
21587 return v;
21588 }
21589 #define rbit32 rbit32
21590 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
21591 static forceinline u32
21592 rbit32(u32 v)
21593 {
21594 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
21595 return v;
21596 }
21597 #define rbit32 rbit32
21598 #endif
21599  
21600 #endif
21601  
21602  
21603 typedef void *(*malloc_func_t)(size_t);
21604 typedef void (*free_func_t)(void *);
21605  
21606 extern malloc_func_t libdeflate_default_malloc_func;
21607 extern free_func_t libdeflate_default_free_func;
21608  
21609 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
21610 size_t alignment, size_t size);
21611 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
21612  
21613 #ifdef FREESTANDING
21614  
21615 void *memset(void *s, int c, size_t n);
21616 #define memset(s, c, n) __builtin_memset((s), (c), (n))
21617  
21618 void *memcpy(void *dest, const void *src, size_t n);
21619 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
21620  
21621 void *memmove(void *dest, const void *src, size_t n);
21622 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
21623  
21624 int memcmp(const void *s1, const void *s2, size_t n);
21625 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
21626  
21627 #undef LIBDEFLATE_ENABLE_ASSERTIONS
21628 #else
21629 # include <string.h>
21630
21631 # ifdef __clang_analyzer__
21632 # define LIBDEFLATE_ENABLE_ASSERTIONS
21633 # endif
21634 #endif
21635  
21636  
21637 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
21638 NORETURN void
21639 libdeflate_assertion_failed(const char *expr, const char *file, int line);
21640 #define ASSERT(expr) { if (unlikely(!(expr))) \
21641 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
21642 #else
21643 #define ASSERT(expr) (void)(expr)
21644 #endif
21645  
21646 #define CONCAT_IMPL(a, b) a##b
21647 #define CONCAT(a, b) CONCAT_IMPL(a, b)
21648 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
21649  
21650 #endif
21651  
21652  
21653 #ifndef MATCHFINDER_WINDOW_ORDER
21654 # error "MATCHFINDER_WINDOW_ORDER must be defined!"
21655 #endif
21656  
21657  
21658 static forceinline u32
21659 loaded_u32_to_u24(u32 v)
21660 {
21661 if (CPU_IS_LITTLE_ENDIAN())
21662 return v & 0xFFFFFF;
21663 else
21664 return v >> 8;
21665 }
21666  
21667  
21668 static forceinline u32
21669 load_u24_unaligned(const u8 *p)
21670 {
21671 #if UNALIGNED_ACCESS_IS_FAST
21672 return loaded_u32_to_u24(load_u32_unaligned(p));
21673 #else
21674 if (CPU_IS_LITTLE_ENDIAN())
21675 return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
21676 else
21677 return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
21678 #endif
21679 }
21680  
21681 #define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER)
21682  
21683 typedef s16 mf_pos_t;
21684  
21685 #define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
21686  
21687  
21688 #define MATCHFINDER_MEM_ALIGNMENT 32
21689  
21690  
21691 #define MATCHFINDER_SIZE_ALIGNMENT 1024
21692  
21693 #undef matchfinder_init
21694 #undef matchfinder_rebase
21695 #ifdef _aligned_attribute
21696 # define MATCHFINDER_ALIGNED _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
21697 # if defined(ARCH_ARM32) || defined(ARCH_ARM64)
21698 /* # include "arm/matchfinder_impl.h" */
21699  
21700  
21701 #ifndef LIB_ARM_MATCHFINDER_IMPL_H
21702 #define LIB_ARM_MATCHFINDER_IMPL_H
21703  
21704 /* #include "arm-cpu_features.h" */
21705  
21706  
21707 #ifndef LIB_ARM_CPU_FEATURES_H
21708 #define LIB_ARM_CPU_FEATURES_H
21709  
21710 /* #include "lib_common.h" */
21711  
21712  
21713 #ifndef LIB_LIB_COMMON_H
21714 #define LIB_LIB_COMMON_H
21715  
21716 #ifdef LIBDEFLATE_H
21717
21718 # error "lib_common.h must always be included before libdeflate.h"
21719 #endif
21720  
21721 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
21722 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
21723 #elif defined(__GNUC__)
21724 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
21725 #else
21726 # define LIBDEFLATE_EXPORT_SYM
21727 #endif
21728  
21729  
21730 #if defined(__GNUC__) && defined(__i386__)
21731 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
21732 #else
21733 # define LIBDEFLATE_ALIGN_STACK
21734 #endif
21735  
21736 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
21737  
21738 /* #include "../common_defs.h" */
21739  
21740  
21741 #ifndef COMMON_DEFS_H
21742 #define COMMON_DEFS_H
21743  
21744 /* #include "libdeflate.h" */
21745  
21746  
21747 #ifndef LIBDEFLATE_H
21748 #define LIBDEFLATE_H
21749  
21750 #include <stddef.h>
21751 #include <stdint.h>
21752  
21753 #ifdef __cplusplus
21754 extern "C" {
21755 #endif
21756  
21757 #define LIBDEFLATE_VERSION_MAJOR 1
21758 #define LIBDEFLATE_VERSION_MINOR 26
21759 #define LIBDEFLATE_VERSION_STRING "1.26"
21760  
21761  
21762 #ifndef LIBDEFLATEAPI
21763 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
21764 # define LIBDEFLATEAPI __declspec(dllimport)
21765 # else
21766 # define LIBDEFLATEAPI
21767 # endif
21768 #endif
21769  
21770  
21771  
21772  
21773  
21774 struct libdeflate_compressor;
21775 struct libdeflate_options;
21776  
21777  
21778 LIBDEFLATEAPI struct libdeflate_compressor *
21779 libdeflate_alloc_compressor(int compression_level);
21780  
21781  
21782 LIBDEFLATEAPI struct libdeflate_compressor *
21783 libdeflate_alloc_compressor_ex(int compression_level,
21784 const struct libdeflate_options *options);
21785  
21786  
21787 LIBDEFLATEAPI size_t
21788 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
21789 const void *in, size_t in_nbytes,
21790 void *out, size_t out_nbytes_avail);
21791  
21792  
21793 LIBDEFLATEAPI size_t
21794 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
21795 size_t in_nbytes);
21796  
21797  
21798 LIBDEFLATEAPI size_t
21799 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
21800 const void *in, size_t in_nbytes,
21801 void *out, size_t out_nbytes_avail);
21802  
21803  
21804 LIBDEFLATEAPI size_t
21805 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
21806 size_t in_nbytes);
21807  
21808  
21809 LIBDEFLATEAPI size_t
21810 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
21811 const void *in, size_t in_nbytes,
21812 void *out, size_t out_nbytes_avail);
21813  
21814  
21815 LIBDEFLATEAPI size_t
21816 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
21817 size_t in_nbytes);
21818  
21819  
21820 LIBDEFLATEAPI void
21821 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
21822  
21823  
21824  
21825  
21826  
21827 struct libdeflate_decompressor;
21828 struct libdeflate_options;
21829  
21830  
21831 LIBDEFLATEAPI struct libdeflate_decompressor *
21832 libdeflate_alloc_decompressor(void);
21833  
21834  
21835 LIBDEFLATEAPI struct libdeflate_decompressor *
21836 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
21837  
21838  
21839 enum libdeflate_result {
21840
21841 LIBDEFLATE_SUCCESS = 0,
21842  
21843
21844 LIBDEFLATE_BAD_DATA = 1,
21845  
21846
21847 LIBDEFLATE_SHORT_OUTPUT = 2,
21848  
21849
21850 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
21851 };
21852  
21853  
21854 LIBDEFLATEAPI enum libdeflate_result
21855 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
21856 const void *in, size_t in_nbytes,
21857 void *out, size_t out_nbytes_avail,
21858 size_t *actual_out_nbytes_ret);
21859  
21860  
21861 LIBDEFLATEAPI enum libdeflate_result
21862 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
21863 const void *in, size_t in_nbytes,
21864 void *out, size_t out_nbytes_avail,
21865 size_t *actual_in_nbytes_ret,
21866 size_t *actual_out_nbytes_ret);
21867  
21868  
21869 LIBDEFLATEAPI enum libdeflate_result
21870 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
21871 const void *in, size_t in_nbytes,
21872 void *out, size_t out_nbytes_avail,
21873 size_t *actual_out_nbytes_ret);
21874  
21875  
21876 LIBDEFLATEAPI enum libdeflate_result
21877 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
21878 const void *in, size_t in_nbytes,
21879 void *out, size_t out_nbytes_avail,
21880 size_t *actual_in_nbytes_ret,
21881 size_t *actual_out_nbytes_ret);
21882  
21883  
21884 LIBDEFLATEAPI enum libdeflate_result
21885 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
21886 const void *in, size_t in_nbytes,
21887 void *out, size_t out_nbytes_avail,
21888 size_t *actual_out_nbytes_ret);
21889  
21890  
21891 LIBDEFLATEAPI enum libdeflate_result
21892 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
21893 const void *in, size_t in_nbytes,
21894 void *out, size_t out_nbytes_avail,
21895 size_t *actual_in_nbytes_ret,
21896 size_t *actual_out_nbytes_ret);
21897  
21898  
21899 LIBDEFLATEAPI void
21900 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
21901  
21902  
21903  
21904  
21905  
21906  
21907 LIBDEFLATEAPI uint32_t
21908 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
21909  
21910  
21911  
21912 LIBDEFLATEAPI uint32_t
21913 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
21914  
21915  
21916  
21917  
21918  
21919  
21920 LIBDEFLATEAPI void
21921 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
21922 void (*free_func)(void *));
21923  
21924  
21925 struct libdeflate_options {
21926  
21927
21928 size_t sizeof_options;
21929  
21930
21931 void *(*malloc_func)(size_t);
21932 void (*free_func)(void *);
21933 };
21934  
21935 #ifdef __cplusplus
21936 }
21937 #endif
21938  
21939 #endif
21940  
21941  
21942 #include <stdbool.h>
21943 #include <stddef.h>
21944 #include <stdint.h>
21945 #ifdef _MSC_VER
21946 # include <intrin.h>
21947 # include <stdlib.h>
21948
21949
21950 # pragma warning(disable : 4146)
21951
21952 # pragma warning(disable : 4018)
21953 # pragma warning(disable : 4244)
21954 # pragma warning(disable : 4267)
21955 # pragma warning(disable : 4310)
21956
21957 # pragma warning(disable : 4100)
21958 # pragma warning(disable : 4127)
21959 # pragma warning(disable : 4189)
21960 # pragma warning(disable : 4232)
21961 # pragma warning(disable : 4245)
21962 # pragma warning(disable : 4295)
21963 #endif
21964 #ifndef FREESTANDING
21965 # include <string.h>
21966 #endif
21967  
21968  
21969  
21970  
21971  
21972  
21973 #undef ARCH_X86_64
21974 #undef ARCH_X86_32
21975 #undef ARCH_ARM64
21976 #undef ARCH_ARM32
21977 #undef ARCH_RISCV
21978 #ifdef _MSC_VER
21979
21980 # if defined(_M_X64) && !defined(_M_ARM64EC)
21981 # define ARCH_X86_64
21982 # elif defined(_M_IX86)
21983 # define ARCH_X86_32
21984 # elif defined(_M_ARM64)
21985 # define ARCH_ARM64
21986 # elif defined(_M_ARM)
21987 # define ARCH_ARM32
21988 # endif
21989 #else
21990 # if defined(__x86_64__)
21991 # define ARCH_X86_64
21992 # elif defined(__i386__)
21993 # define ARCH_X86_32
21994 # elif defined(__aarch64__)
21995 # define ARCH_ARM64
21996 # elif defined(__arm__)
21997 # define ARCH_ARM32
21998 # elif defined(__riscv)
21999 # define ARCH_RISCV
22000 # endif
22001 #endif
22002  
22003  
22004  
22005  
22006  
22007  
22008 typedef uint8_t u8;
22009 typedef uint16_t u16;
22010 typedef uint32_t u32;
22011 typedef uint64_t u64;
22012 typedef int8_t s8;
22013 typedef int16_t s16;
22014 typedef int32_t s32;
22015 typedef int64_t s64;
22016  
22017  
22018 #ifdef _MSC_VER
22019 # ifdef _WIN64
22020 typedef long long ssize_t;
22021 # else
22022 typedef long ssize_t;
22023 # endif
22024 #endif
22025  
22026  
22027 typedef size_t machine_word_t;
22028  
22029  
22030 #define WORDBYTES ((int)sizeof(machine_word_t))
22031  
22032  
22033 #define WORDBITS (8 * WORDBYTES)
22034  
22035  
22036  
22037  
22038  
22039  
22040 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
22041 # define GCC_PREREQ(major, minor) \
22042 (__GNUC__ > (major) || \
22043 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
22044 # if !GCC_PREREQ(4, 9)
22045 # error "gcc versions older than 4.9 are no longer supported"
22046 # endif
22047 #else
22048 # define GCC_PREREQ(major, minor) 0
22049 #endif
22050 #ifdef __clang__
22051 # ifdef __apple_build_version__
22052 # define CLANG_PREREQ(major, minor, apple_version) \
22053 (__apple_build_version__ >= (apple_version))
22054 # else
22055 # define CLANG_PREREQ(major, minor, apple_version) \
22056 (__clang_major__ > (major) || \
22057 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
22058 # endif
22059 # if !CLANG_PREREQ(3, 9, 8000000)
22060 # error "clang versions older than 3.9 are no longer supported"
22061 # endif
22062 #else
22063 # define CLANG_PREREQ(major, minor, apple_version) 0
22064 #endif
22065 #ifdef _MSC_VER
22066 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
22067 # if !MSVC_PREREQ(1928)
22068 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
22069 # endif
22070 #else
22071 # define MSVC_PREREQ(version) 0
22072 #endif
22073  
22074  
22075 #ifndef __has_attribute
22076 # define __has_attribute(attribute) 0
22077 #endif
22078  
22079  
22080 #ifndef __has_builtin
22081 # define __has_builtin(builtin) 0
22082 #endif
22083  
22084  
22085 #ifdef _MSC_VER
22086 # define inline __inline
22087 #endif
22088  
22089  
22090 #if defined(__GNUC__) || __has_attribute(always_inline)
22091 # define forceinline inline __attribute__((always_inline))
22092 #elif defined(_MSC_VER)
22093 # define forceinline __forceinline
22094 #else
22095 # define forceinline inline
22096 #endif
22097  
22098  
22099 #if defined(__GNUC__) || __has_attribute(unused)
22100 # define MAYBE_UNUSED __attribute__((unused))
22101 #else
22102 # define MAYBE_UNUSED
22103 #endif
22104  
22105  
22106 #if defined(__GNUC__) || __has_attribute(noreturn)
22107 # define NORETURN __attribute__((noreturn))
22108 #else
22109 # define NORETURN
22110 #endif
22111  
22112  
22113 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
22114 # define likely(expr) __builtin_expect(!!(expr), 1)
22115 #else
22116 # define likely(expr) (expr)
22117 #endif
22118  
22119  
22120 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
22121 # define unlikely(expr) __builtin_expect(!!(expr), 0)
22122 #else
22123 # define unlikely(expr) (expr)
22124 #endif
22125  
22126  
22127 #undef prefetchr
22128 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
22129 # define prefetchr(addr) __builtin_prefetch((addr), 0)
22130 #elif defined(_MSC_VER)
22131 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
22132 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
22133 # elif defined(ARCH_ARM64)
22134 # define prefetchr(addr) __prefetch2((addr), 0x00 )
22135 # elif defined(ARCH_ARM32)
22136 # define prefetchr(addr) __prefetch(addr)
22137 # endif
22138 #endif
22139 #ifndef prefetchr
22140 # define prefetchr(addr)
22141 #endif
22142  
22143  
22144 #undef prefetchw
22145 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
22146 # define prefetchw(addr) __builtin_prefetch((addr), 1)
22147 #elif defined(_MSC_VER)
22148 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
22149 # define prefetchw(addr) _m_prefetchw(addr)
22150 # elif defined(ARCH_ARM64)
22151 # define prefetchw(addr) __prefetch2((addr), 0x10 )
22152 # elif defined(ARCH_ARM32)
22153 # define prefetchw(addr) __prefetchw(addr)
22154 # endif
22155 #endif
22156 #ifndef prefetchw
22157 # define prefetchw(addr)
22158 #endif
22159  
22160  
22161 #undef _aligned_attribute
22162 #if defined(__GNUC__) || __has_attribute(aligned)
22163 # define _aligned_attribute(n) __attribute__((aligned(n)))
22164 #elif defined(_MSC_VER)
22165 # define _aligned_attribute(n) __declspec(align(n))
22166 #endif
22167  
22168  
22169 #if defined(__GNUC__) || __has_attribute(target)
22170 # define _target_attribute(attrs) __attribute__((target(attrs)))
22171 #else
22172 # define _target_attribute(attrs)
22173 #endif
22174  
22175  
22176  
22177  
22178  
22179 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
22180 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
22181 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
22182 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
22183 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
22184 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
22185 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
22186  
22187  
22188  
22189  
22190  
22191  
22192 #if defined(__BYTE_ORDER__)
22193 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
22194 #elif defined(_MSC_VER)
22195 # define CPU_IS_LITTLE_ENDIAN() true
22196 #else
22197 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
22198 {
22199 union {
22200 u32 w;
22201 u8 b;
22202 } u;
22203  
22204 u.w = 1;
22205 return u.b;
22206 }
22207 #endif
22208  
22209  
22210 static forceinline u16 bswap16(u16 v)
22211 {
22212 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
22213 return __builtin_bswap16(v);
22214 #elif defined(_MSC_VER)
22215 return _byteswap_ushort(v);
22216 #else
22217 return (v << 8) | (v >> 8);
22218 #endif
22219 }
22220  
22221  
22222 static forceinline u32 bswap32(u32 v)
22223 {
22224 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
22225 return __builtin_bswap32(v);
22226 #elif defined(_MSC_VER)
22227 return _byteswap_ulong(v);
22228 #else
22229 return ((v & 0x000000FF) << 24) |
22230 ((v & 0x0000FF00) << 8) |
22231 ((v & 0x00FF0000) >> 8) |
22232 ((v & 0xFF000000) >> 24);
22233 #endif
22234 }
22235  
22236  
22237 static forceinline u64 bswap64(u64 v)
22238 {
22239 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
22240 return __builtin_bswap64(v);
22241 #elif defined(_MSC_VER)
22242 return _byteswap_uint64(v);
22243 #else
22244 return ((v & 0x00000000000000FF) << 56) |
22245 ((v & 0x000000000000FF00) << 40) |
22246 ((v & 0x0000000000FF0000) << 24) |
22247 ((v & 0x00000000FF000000) << 8) |
22248 ((v & 0x000000FF00000000) >> 8) |
22249 ((v & 0x0000FF0000000000) >> 24) |
22250 ((v & 0x00FF000000000000) >> 40) |
22251 ((v & 0xFF00000000000000) >> 56);
22252 #endif
22253 }
22254  
22255 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
22256 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
22257 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
22258 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
22259 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
22260 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
22261  
22262  
22263  
22264  
22265  
22266  
22267 #if (defined(__GNUC__) || defined(__clang__)) && \
22268 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
22269 defined(__ARM_FEATURE_UNALIGNED) || \
22270 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
22271 defined(__riscv_misaligned_fast) || \
22272 defined(__wasm__))
22273 # define UNALIGNED_ACCESS_IS_FAST 1
22274 #elif defined(_MSC_VER)
22275 # define UNALIGNED_ACCESS_IS_FAST 1
22276 #else
22277 # define UNALIGNED_ACCESS_IS_FAST 0
22278 #endif
22279  
22280  
22281  
22282 #ifdef FREESTANDING
22283 # define MEMCOPY __builtin_memcpy
22284 #else
22285 # define MEMCOPY memcpy
22286 #endif
22287  
22288  
22289  
22290 #define DEFINE_UNALIGNED_TYPE(type) \
22291 static forceinline type \
22292 load_##type##_unaligned(const void *p) \
22293 { \
22294 type v; \
22295 \
22296 MEMCOPY(&v, p, sizeof(v)); \
22297 return v; \
22298 } \
22299 \
22300 static forceinline void \
22301 store_##type##_unaligned(type v, void *p) \
22302 { \
22303 MEMCOPY(p, &v, sizeof(v)); \
22304 }
22305  
22306 DEFINE_UNALIGNED_TYPE(u16)
22307 DEFINE_UNALIGNED_TYPE(u32)
22308 DEFINE_UNALIGNED_TYPE(u64)
22309 DEFINE_UNALIGNED_TYPE(machine_word_t)
22310  
22311 #undef MEMCOPY
22312  
22313 #define load_word_unaligned load_machine_word_t_unaligned
22314 #define store_word_unaligned store_machine_word_t_unaligned
22315  
22316  
22317  
22318 static forceinline u16
22319 get_unaligned_le16(const u8 *p)
22320 {
22321 if (UNALIGNED_ACCESS_IS_FAST)
22322 return le16_bswap(load_u16_unaligned(p));
22323 else
22324 return ((u16)p[1] << 8) | p[0];
22325 }
22326  
22327 static forceinline u16
22328 get_unaligned_be16(const u8 *p)
22329 {
22330 if (UNALIGNED_ACCESS_IS_FAST)
22331 return be16_bswap(load_u16_unaligned(p));
22332 else
22333 return ((u16)p[0] << 8) | p[1];
22334 }
22335  
22336 static forceinline u32
22337 get_unaligned_le32(const u8 *p)
22338 {
22339 if (UNALIGNED_ACCESS_IS_FAST)
22340 return le32_bswap(load_u32_unaligned(p));
22341 else
22342 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
22343 ((u32)p[1] << 8) | p[0];
22344 }
22345  
22346 static forceinline u32
22347 get_unaligned_be32(const u8 *p)
22348 {
22349 if (UNALIGNED_ACCESS_IS_FAST)
22350 return be32_bswap(load_u32_unaligned(p));
22351 else
22352 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
22353 ((u32)p[2] << 8) | p[3];
22354 }
22355  
22356 static forceinline u64
22357 get_unaligned_le64(const u8 *p)
22358 {
22359 if (UNALIGNED_ACCESS_IS_FAST)
22360 return le64_bswap(load_u64_unaligned(p));
22361 else
22362 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
22363 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
22364 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
22365 ((u64)p[1] << 8) | p[0];
22366 }
22367  
22368 static forceinline machine_word_t
22369 get_unaligned_leword(const u8 *p)
22370 {
22371 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
22372 if (WORDBITS == 32)
22373 return get_unaligned_le32(p);
22374 else
22375 return get_unaligned_le64(p);
22376 }
22377  
22378  
22379  
22380 static forceinline void
22381 put_unaligned_le16(u16 v, u8 *p)
22382 {
22383 if (UNALIGNED_ACCESS_IS_FAST) {
22384 store_u16_unaligned(le16_bswap(v), p);
22385 } else {
22386 p[0] = (u8)(v >> 0);
22387 p[1] = (u8)(v >> 8);
22388 }
22389 }
22390  
22391 static forceinline void
22392 put_unaligned_be16(u16 v, u8 *p)
22393 {
22394 if (UNALIGNED_ACCESS_IS_FAST) {
22395 store_u16_unaligned(be16_bswap(v), p);
22396 } else {
22397 p[0] = (u8)(v >> 8);
22398 p[1] = (u8)(v >> 0);
22399 }
22400 }
22401  
22402 static forceinline void
22403 put_unaligned_le32(u32 v, u8 *p)
22404 {
22405 if (UNALIGNED_ACCESS_IS_FAST) {
22406 store_u32_unaligned(le32_bswap(v), p);
22407 } else {
22408 p[0] = (u8)(v >> 0);
22409 p[1] = (u8)(v >> 8);
22410 p[2] = (u8)(v >> 16);
22411 p[3] = (u8)(v >> 24);
22412 }
22413 }
22414  
22415 static forceinline void
22416 put_unaligned_be32(u32 v, u8 *p)
22417 {
22418 if (UNALIGNED_ACCESS_IS_FAST) {
22419 store_u32_unaligned(be32_bswap(v), p);
22420 } else {
22421 p[0] = (u8)(v >> 24);
22422 p[1] = (u8)(v >> 16);
22423 p[2] = (u8)(v >> 8);
22424 p[3] = (u8)(v >> 0);
22425 }
22426 }
22427  
22428 static forceinline void
22429 put_unaligned_le64(u64 v, u8 *p)
22430 {
22431 if (UNALIGNED_ACCESS_IS_FAST) {
22432 store_u64_unaligned(le64_bswap(v), p);
22433 } else {
22434 p[0] = (u8)(v >> 0);
22435 p[1] = (u8)(v >> 8);
22436 p[2] = (u8)(v >> 16);
22437 p[3] = (u8)(v >> 24);
22438 p[4] = (u8)(v >> 32);
22439 p[5] = (u8)(v >> 40);
22440 p[6] = (u8)(v >> 48);
22441 p[7] = (u8)(v >> 56);
22442 }
22443 }
22444  
22445 static forceinline void
22446 put_unaligned_leword(machine_word_t v, u8 *p)
22447 {
22448 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
22449 if (WORDBITS == 32)
22450 put_unaligned_le32(v, p);
22451 else
22452 put_unaligned_le64(v, p);
22453 }
22454  
22455  
22456  
22457  
22458  
22459  
22460  
22461 static forceinline unsigned
22462 bsr32(u32 v)
22463 {
22464 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
22465 return 31 - __builtin_clz(v);
22466 #elif defined(_MSC_VER)
22467 unsigned long i;
22468  
22469 _BitScanReverse(&i, v);
22470 return i;
22471 #else
22472 unsigned i = 0;
22473  
22474 while ((v >>= 1) != 0)
22475 i++;
22476 return i;
22477 #endif
22478 }
22479  
22480 static forceinline unsigned
22481 bsr64(u64 v)
22482 {
22483 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
22484 return 63 - __builtin_clzll(v);
22485 #elif defined(_MSC_VER) && defined(_WIN64)
22486 unsigned long i;
22487  
22488 _BitScanReverse64(&i, v);
22489 return i;
22490 #else
22491 unsigned i = 0;
22492  
22493 while ((v >>= 1) != 0)
22494 i++;
22495 return i;
22496 #endif
22497 }
22498  
22499 static forceinline unsigned
22500 bsrw(machine_word_t v)
22501 {
22502 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
22503 if (WORDBITS == 32)
22504 return bsr32(v);
22505 else
22506 return bsr64(v);
22507 }
22508  
22509  
22510  
22511 static forceinline unsigned
22512 bsf32(u32 v)
22513 {
22514 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
22515 return __builtin_ctz(v);
22516 #elif defined(_MSC_VER)
22517 unsigned long i;
22518  
22519 _BitScanForward(&i, v);
22520 return i;
22521 #else
22522 unsigned i = 0;
22523  
22524 for (; (v & 1) == 0; v >>= 1)
22525 i++;
22526 return i;
22527 #endif
22528 }
22529  
22530 static forceinline unsigned
22531 bsf64(u64 v)
22532 {
22533 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
22534 return __builtin_ctzll(v);
22535 #elif defined(_MSC_VER) && defined(_WIN64)
22536 unsigned long i;
22537  
22538 _BitScanForward64(&i, v);
22539 return i;
22540 #else
22541 unsigned i = 0;
22542  
22543 for (; (v & 1) == 0; v >>= 1)
22544 i++;
22545 return i;
22546 #endif
22547 }
22548  
22549 static forceinline unsigned
22550 bsfw(machine_word_t v)
22551 {
22552 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
22553 if (WORDBITS == 32)
22554 return bsf32(v);
22555 else
22556 return bsf64(v);
22557 }
22558  
22559  
22560 #undef rbit32
22561 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
22562 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
22563 static forceinline u32
22564 rbit32(u32 v)
22565 {
22566 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
22567 return v;
22568 }
22569 #define rbit32 rbit32
22570 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
22571 static forceinline u32
22572 rbit32(u32 v)
22573 {
22574 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
22575 return v;
22576 }
22577 #define rbit32 rbit32
22578 #endif
22579  
22580 #endif
22581  
22582  
22583 typedef void *(*malloc_func_t)(size_t);
22584 typedef void (*free_func_t)(void *);
22585  
22586 extern malloc_func_t libdeflate_default_malloc_func;
22587 extern free_func_t libdeflate_default_free_func;
22588  
22589 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
22590 size_t alignment, size_t size);
22591 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
22592  
22593 #ifdef FREESTANDING
22594  
22595 void *memset(void *s, int c, size_t n);
22596 #define memset(s, c, n) __builtin_memset((s), (c), (n))
22597  
22598 void *memcpy(void *dest, const void *src, size_t n);
22599 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
22600  
22601 void *memmove(void *dest, const void *src, size_t n);
22602 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
22603  
22604 int memcmp(const void *s1, const void *s2, size_t n);
22605 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
22606  
22607 #undef LIBDEFLATE_ENABLE_ASSERTIONS
22608 #else
22609 # include <string.h>
22610
22611 # ifdef __clang_analyzer__
22612 # define LIBDEFLATE_ENABLE_ASSERTIONS
22613 # endif
22614 #endif
22615  
22616  
22617 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
22618 NORETURN void
22619 libdeflate_assertion_failed(const char *expr, const char *file, int line);
22620 #define ASSERT(expr) { if (unlikely(!(expr))) \
22621 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
22622 #else
22623 #define ASSERT(expr) (void)(expr)
22624 #endif
22625  
22626 #define CONCAT_IMPL(a, b) a##b
22627 #define CONCAT(a, b) CONCAT_IMPL(a, b)
22628 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
22629  
22630 #endif
22631  
22632  
22633 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
22634  
22635 #define ARM_CPU_FEATURE_NEON (1 << 0)
22636 #define ARM_CPU_FEATURE_PMULL (1 << 1)
22637  
22638 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
22639 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
22640 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
22641 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
22642  
22643 #if !defined(FREESTANDING) && \
22644 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
22645 (defined(__linux__) || \
22646 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
22647 (defined(_WIN32) && defined(ARCH_ARM64)))
22648  
22649 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
22650 extern volatile u32 libdeflate_arm_cpu_features;
22651  
22652 void libdeflate_init_arm_cpu_features(void);
22653  
22654 static inline u32 get_arm_cpu_features(void)
22655 {
22656 if (libdeflate_arm_cpu_features == 0)
22657 libdeflate_init_arm_cpu_features();
22658 return libdeflate_arm_cpu_features;
22659 }
22660 #else
22661 static inline u32 get_arm_cpu_features(void) { return 0; }
22662 #endif
22663  
22664  
22665 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
22666 # define HAVE_NEON(features) 1
22667 # define HAVE_NEON_NATIVE 1
22668 #else
22669 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
22670 # define HAVE_NEON_NATIVE 0
22671 #endif
22672  
22673 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
22674 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
22675 # define HAVE_NEON_INTRIN 1
22676 # include <arm_neon.h>
22677 #else
22678 # define HAVE_NEON_INTRIN 0
22679 #endif
22680  
22681  
22682 #ifdef __ARM_FEATURE_CRYPTO
22683 # define HAVE_PMULL(features) 1
22684 #else
22685 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
22686 #endif
22687 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
22688 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
22689 CPU_IS_LITTLE_ENDIAN()
22690 # define HAVE_PMULL_INTRIN 1
22691
22692 # ifdef _MSC_VER
22693 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
22694 # else
22695 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
22696 # endif
22697 #else
22698 # define HAVE_PMULL_INTRIN 0
22699 #endif
22700  
22701  
22702 #ifdef __ARM_FEATURE_CRC32
22703 # define HAVE_CRC32(features) 1
22704 #else
22705 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
22706 #endif
22707 #if defined(ARCH_ARM64) && \
22708 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
22709 # define HAVE_CRC32_INTRIN 1
22710 # if defined(__GNUC__) || defined(__clang__)
22711 # include <arm_acle.h>
22712 # endif
22713
22714 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
22715 !defined(__ARM_FEATURE_CRC32)
22716 # undef __crc32b
22717 # define __crc32b(a, b) \
22718 ({ uint32_t res; \
22719 __asm__("crc32b %w0, %w1, %w2" \
22720 : "=r" (res) : "r" (a), "r" (b)); \
22721 res; })
22722 # undef __crc32h
22723 # define __crc32h(a, b) \
22724 ({ uint32_t res; \
22725 __asm__("crc32h %w0, %w1, %w2" \
22726 : "=r" (res) : "r" (a), "r" (b)); \
22727 res; })
22728 # undef __crc32w
22729 # define __crc32w(a, b) \
22730 ({ uint32_t res; \
22731 __asm__("crc32w %w0, %w1, %w2" \
22732 : "=r" (res) : "r" (a), "r" (b)); \
22733 res; })
22734 # undef __crc32d
22735 # define __crc32d(a, b) \
22736 ({ uint32_t res; \
22737 __asm__("crc32x %w0, %w1, %2" \
22738 : "=r" (res) : "r" (a), "r" (b)); \
22739 res; })
22740 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
22741 # endif
22742 #else
22743 # define HAVE_CRC32_INTRIN 0
22744 #endif
22745  
22746  
22747 #ifdef __ARM_FEATURE_SHA3
22748 # define HAVE_SHA3(features) 1
22749 #else
22750 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
22751 #endif
22752 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
22753 (GCC_PREREQ(9, 1) || \
22754 CLANG_PREREQ(7, 0, 10010463) )
22755 # define HAVE_SHA3_INTRIN 1
22756
22757 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
22758 !defined(__ARM_FEATURE_SHA3)
22759 # undef veor3q_u8
22760 # define veor3q_u8(a, b, c) \
22761 ({ uint8x16_t res; \
22762 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
22763 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
22764 res; })
22765 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
22766 # endif
22767 #else
22768 # define HAVE_SHA3_INTRIN 0
22769 #endif
22770  
22771  
22772 #ifdef __ARM_FEATURE_DOTPROD
22773 # define HAVE_DOTPROD(features) 1
22774 #else
22775 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
22776 #endif
22777 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
22778 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
22779 # define HAVE_DOTPROD_INTRIN 1
22780
22781 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
22782 !defined(__ARM_FEATURE_DOTPROD)
22783 # undef vdotq_u32
22784 # define vdotq_u32(a, b, c) \
22785 ({ uint32x4_t res = (a); \
22786 __asm__("udot %0.4s, %1.16b, %2.16b" \
22787 : "+w" (res) : "w" (b), "w" (c)); \
22788 res; })
22789 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
22790 # endif
22791 #else
22792 # define HAVE_DOTPROD_INTRIN 0
22793 #endif
22794  
22795 #endif
22796  
22797 #endif
22798  
22799  
22800 #if HAVE_NEON_NATIVE
22801 static forceinline void
22802 matchfinder_init_neon(mf_pos_t *data, size_t size)
22803 {
22804 int16x8_t *p = (int16x8_t *)data;
22805 int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);
22806  
22807 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
22808 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
22809 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
22810  
22811 do {
22812 p[0] = v;
22813 p[1] = v;
22814 p[2] = v;
22815 p[3] = v;
22816 p += 4;
22817 size -= 4 * sizeof(*p);
22818 } while (size != 0);
22819 }
22820 #define matchfinder_init matchfinder_init_neon
22821  
22822 static forceinline void
22823 matchfinder_rebase_neon(mf_pos_t *data, size_t size)
22824 {
22825 int16x8_t *p = (int16x8_t *)data;
22826 int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);
22827  
22828 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
22829 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
22830 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
22831  
22832 do {
22833 p[0] = vqaddq_s16(p[0], v);
22834 p[1] = vqaddq_s16(p[1], v);
22835 p[2] = vqaddq_s16(p[2], v);
22836 p[3] = vqaddq_s16(p[3], v);
22837 p += 4;
22838 size -= 4 * sizeof(*p);
22839 } while (size != 0);
22840 }
22841 #define matchfinder_rebase matchfinder_rebase_neon
22842  
22843 #endif
22844  
22845 #endif
22846  
22847 # elif defined(ARCH_RISCV)
22848 # include "riscv/matchfinder_impl.h"
22849 # elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
22850 /* # include "x86/matchfinder_impl.h" */
22851  
22852  
22853 #ifndef LIB_X86_MATCHFINDER_IMPL_H
22854 #define LIB_X86_MATCHFINDER_IMPL_H
22855  
22856 /* #include "x86-cpu_features.h" */
22857  
22858  
22859 #ifndef LIB_X86_CPU_FEATURES_H
22860 #define LIB_X86_CPU_FEATURES_H
22861  
22862 /* #include "lib_common.h" */
22863  
22864  
22865 #ifndef LIB_LIB_COMMON_H
22866 #define LIB_LIB_COMMON_H
22867  
22868 #ifdef LIBDEFLATE_H
22869
22870 # error "lib_common.h must always be included before libdeflate.h"
22871 #endif
22872  
22873 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
22874 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
22875 #elif defined(__GNUC__)
22876 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
22877 #else
22878 # define LIBDEFLATE_EXPORT_SYM
22879 #endif
22880  
22881  
22882 #if defined(__GNUC__) && defined(__i386__)
22883 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
22884 #else
22885 # define LIBDEFLATE_ALIGN_STACK
22886 #endif
22887  
22888 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
22889  
22890 /* #include "../common_defs.h" */
22891  
22892  
22893 #ifndef COMMON_DEFS_H
22894 #define COMMON_DEFS_H
22895  
22896 /* #include "libdeflate.h" */
22897  
22898  
22899 #ifndef LIBDEFLATE_H
22900 #define LIBDEFLATE_H
22901  
22902 #include <stddef.h>
22903 #include <stdint.h>
22904  
22905 #ifdef __cplusplus
22906 extern "C" {
22907 #endif
22908  
22909 #define LIBDEFLATE_VERSION_MAJOR 1
22910 #define LIBDEFLATE_VERSION_MINOR 26
22911 #define LIBDEFLATE_VERSION_STRING "1.26"
22912  
22913  
22914 #ifndef LIBDEFLATEAPI
22915 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
22916 # define LIBDEFLATEAPI __declspec(dllimport)
22917 # else
22918 # define LIBDEFLATEAPI
22919 # endif
22920 #endif
22921  
22922  
22923  
22924  
22925  
22926 struct libdeflate_compressor;
22927 struct libdeflate_options;
22928  
22929  
22930 LIBDEFLATEAPI struct libdeflate_compressor *
22931 libdeflate_alloc_compressor(int compression_level);
22932  
22933  
22934 LIBDEFLATEAPI struct libdeflate_compressor *
22935 libdeflate_alloc_compressor_ex(int compression_level,
22936 const struct libdeflate_options *options);
22937  
22938  
22939 LIBDEFLATEAPI size_t
22940 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
22941 const void *in, size_t in_nbytes,
22942 void *out, size_t out_nbytes_avail);
22943  
22944  
22945 LIBDEFLATEAPI size_t
22946 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
22947 size_t in_nbytes);
22948  
22949  
22950 LIBDEFLATEAPI size_t
22951 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
22952 const void *in, size_t in_nbytes,
22953 void *out, size_t out_nbytes_avail);
22954  
22955  
22956 LIBDEFLATEAPI size_t
22957 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
22958 size_t in_nbytes);
22959  
22960  
22961 LIBDEFLATEAPI size_t
22962 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
22963 const void *in, size_t in_nbytes,
22964 void *out, size_t out_nbytes_avail);
22965  
22966  
22967 LIBDEFLATEAPI size_t
22968 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
22969 size_t in_nbytes);
22970  
22971  
22972 LIBDEFLATEAPI void
22973 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
22974  
22975  
22976  
22977  
22978  
22979 struct libdeflate_decompressor;
22980 struct libdeflate_options;
22981  
22982  
22983 LIBDEFLATEAPI struct libdeflate_decompressor *
22984 libdeflate_alloc_decompressor(void);
22985  
22986  
22987 LIBDEFLATEAPI struct libdeflate_decompressor *
22988 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
22989  
22990  
22991 enum libdeflate_result {
22992
22993 LIBDEFLATE_SUCCESS = 0,
22994  
22995
22996 LIBDEFLATE_BAD_DATA = 1,
22997  
22998
22999 LIBDEFLATE_SHORT_OUTPUT = 2,
23000  
23001
23002 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
23003 };
23004  
23005  
23006 LIBDEFLATEAPI enum libdeflate_result
23007 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
23008 const void *in, size_t in_nbytes,
23009 void *out, size_t out_nbytes_avail,
23010 size_t *actual_out_nbytes_ret);
23011  
23012  
23013 LIBDEFLATEAPI enum libdeflate_result
23014 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
23015 const void *in, size_t in_nbytes,
23016 void *out, size_t out_nbytes_avail,
23017 size_t *actual_in_nbytes_ret,
23018 size_t *actual_out_nbytes_ret);
23019  
23020  
23021 LIBDEFLATEAPI enum libdeflate_result
23022 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
23023 const void *in, size_t in_nbytes,
23024 void *out, size_t out_nbytes_avail,
23025 size_t *actual_out_nbytes_ret);
23026  
23027  
23028 LIBDEFLATEAPI enum libdeflate_result
23029 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
23030 const void *in, size_t in_nbytes,
23031 void *out, size_t out_nbytes_avail,
23032 size_t *actual_in_nbytes_ret,
23033 size_t *actual_out_nbytes_ret);
23034  
23035  
23036 LIBDEFLATEAPI enum libdeflate_result
23037 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
23038 const void *in, size_t in_nbytes,
23039 void *out, size_t out_nbytes_avail,
23040 size_t *actual_out_nbytes_ret);
23041  
23042  
23043 LIBDEFLATEAPI enum libdeflate_result
23044 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
23045 const void *in, size_t in_nbytes,
23046 void *out, size_t out_nbytes_avail,
23047 size_t *actual_in_nbytes_ret,
23048 size_t *actual_out_nbytes_ret);
23049  
23050  
23051 LIBDEFLATEAPI void
23052 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
23053  
23054  
23055  
23056  
23057  
23058  
23059 LIBDEFLATEAPI uint32_t
23060 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
23061  
23062  
23063  
23064 LIBDEFLATEAPI uint32_t
23065 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
23066  
23067  
23068  
23069  
23070  
23071  
23072 LIBDEFLATEAPI void
23073 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
23074 void (*free_func)(void *));
23075  
23076  
23077 struct libdeflate_options {
23078  
23079
23080 size_t sizeof_options;
23081  
23082
23083 void *(*malloc_func)(size_t);
23084 void (*free_func)(void *);
23085 };
23086  
23087 #ifdef __cplusplus
23088 }
23089 #endif
23090  
23091 #endif
23092  
23093  
23094 #include <stdbool.h>
23095 #include <stddef.h>
23096 #include <stdint.h>
23097 #ifdef _MSC_VER
23098 # include <intrin.h>
23099 # include <stdlib.h>
23100
23101
23102 # pragma warning(disable : 4146)
23103
23104 # pragma warning(disable : 4018)
23105 # pragma warning(disable : 4244)
23106 # pragma warning(disable : 4267)
23107 # pragma warning(disable : 4310)
23108
23109 # pragma warning(disable : 4100)
23110 # pragma warning(disable : 4127)
23111 # pragma warning(disable : 4189)
23112 # pragma warning(disable : 4232)
23113 # pragma warning(disable : 4245)
23114 # pragma warning(disable : 4295)
23115 #endif
23116 #ifndef FREESTANDING
23117 # include <string.h>
23118 #endif
23119  
23120  
23121  
23122  
23123  
23124  
23125 #undef ARCH_X86_64
23126 #undef ARCH_X86_32
23127 #undef ARCH_ARM64
23128 #undef ARCH_ARM32
23129 #undef ARCH_RISCV
23130 #ifdef _MSC_VER
23131
23132 # if defined(_M_X64) && !defined(_M_ARM64EC)
23133 # define ARCH_X86_64
23134 # elif defined(_M_IX86)
23135 # define ARCH_X86_32
23136 # elif defined(_M_ARM64)
23137 # define ARCH_ARM64
23138 # elif defined(_M_ARM)
23139 # define ARCH_ARM32
23140 # endif
23141 #else
23142 # if defined(__x86_64__)
23143 # define ARCH_X86_64
23144 # elif defined(__i386__)
23145 # define ARCH_X86_32
23146 # elif defined(__aarch64__)
23147 # define ARCH_ARM64
23148 # elif defined(__arm__)
23149 # define ARCH_ARM32
23150 # elif defined(__riscv)
23151 # define ARCH_RISCV
23152 # endif
23153 #endif
23154  
23155  
23156  
23157  
23158  
23159  
23160 typedef uint8_t u8;
23161 typedef uint16_t u16;
23162 typedef uint32_t u32;
23163 typedef uint64_t u64;
23164 typedef int8_t s8;
23165 typedef int16_t s16;
23166 typedef int32_t s32;
23167 typedef int64_t s64;
23168  
23169  
23170 #ifdef _MSC_VER
23171 # ifdef _WIN64
23172 typedef long long ssize_t;
23173 # else
23174 typedef long ssize_t;
23175 # endif
23176 #endif
23177  
23178  
23179 typedef size_t machine_word_t;
23180  
23181  
23182 #define WORDBYTES ((int)sizeof(machine_word_t))
23183  
23184  
23185 #define WORDBITS (8 * WORDBYTES)
23186  
23187  
23188  
23189  
23190  
23191  
23192 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
23193 # define GCC_PREREQ(major, minor) \
23194 (__GNUC__ > (major) || \
23195 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
23196 # if !GCC_PREREQ(4, 9)
23197 # error "gcc versions older than 4.9 are no longer supported"
23198 # endif
23199 #else
23200 # define GCC_PREREQ(major, minor) 0
23201 #endif
23202 #ifdef __clang__
23203 # ifdef __apple_build_version__
23204 # define CLANG_PREREQ(major, minor, apple_version) \
23205 (__apple_build_version__ >= (apple_version))
23206 # else
23207 # define CLANG_PREREQ(major, minor, apple_version) \
23208 (__clang_major__ > (major) || \
23209 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
23210 # endif
23211 # if !CLANG_PREREQ(3, 9, 8000000)
23212 # error "clang versions older than 3.9 are no longer supported"
23213 # endif
23214 #else
23215 # define CLANG_PREREQ(major, minor, apple_version) 0
23216 #endif
23217 #ifdef _MSC_VER
23218 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
23219 # if !MSVC_PREREQ(1928)
23220 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
23221 # endif
23222 #else
23223 # define MSVC_PREREQ(version) 0
23224 #endif
23225  
23226  
23227 #ifndef __has_attribute
23228 # define __has_attribute(attribute) 0
23229 #endif
23230  
23231  
23232 #ifndef __has_builtin
23233 # define __has_builtin(builtin) 0
23234 #endif
23235  
23236  
23237 #ifdef _MSC_VER
23238 # define inline __inline
23239 #endif
23240  
23241  
23242 #if defined(__GNUC__) || __has_attribute(always_inline)
23243 # define forceinline inline __attribute__((always_inline))
23244 #elif defined(_MSC_VER)
23245 # define forceinline __forceinline
23246 #else
23247 # define forceinline inline
23248 #endif
23249  
23250  
23251 #if defined(__GNUC__) || __has_attribute(unused)
23252 # define MAYBE_UNUSED __attribute__((unused))
23253 #else
23254 # define MAYBE_UNUSED
23255 #endif
23256  
23257  
23258 #if defined(__GNUC__) || __has_attribute(noreturn)
23259 # define NORETURN __attribute__((noreturn))
23260 #else
23261 # define NORETURN
23262 #endif
23263  
23264  
23265 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
23266 # define likely(expr) __builtin_expect(!!(expr), 1)
23267 #else
23268 # define likely(expr) (expr)
23269 #endif
23270  
23271  
23272 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
23273 # define unlikely(expr) __builtin_expect(!!(expr), 0)
23274 #else
23275 # define unlikely(expr) (expr)
23276 #endif
23277  
23278  
23279 #undef prefetchr
23280 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
23281 # define prefetchr(addr) __builtin_prefetch((addr), 0)
23282 #elif defined(_MSC_VER)
23283 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
23284 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
23285 # elif defined(ARCH_ARM64)
23286 # define prefetchr(addr) __prefetch2((addr), 0x00 )
23287 # elif defined(ARCH_ARM32)
23288 # define prefetchr(addr) __prefetch(addr)
23289 # endif
23290 #endif
23291 #ifndef prefetchr
23292 # define prefetchr(addr)
23293 #endif
23294  
23295  
23296 #undef prefetchw
23297 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
23298 # define prefetchw(addr) __builtin_prefetch((addr), 1)
23299 #elif defined(_MSC_VER)
23300 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
23301 # define prefetchw(addr) _m_prefetchw(addr)
23302 # elif defined(ARCH_ARM64)
23303 # define prefetchw(addr) __prefetch2((addr), 0x10 )
23304 # elif defined(ARCH_ARM32)
23305 # define prefetchw(addr) __prefetchw(addr)
23306 # endif
23307 #endif
23308 #ifndef prefetchw
23309 # define prefetchw(addr)
23310 #endif
23311  
23312  
23313 #undef _aligned_attribute
23314 #if defined(__GNUC__) || __has_attribute(aligned)
23315 # define _aligned_attribute(n) __attribute__((aligned(n)))
23316 #elif defined(_MSC_VER)
23317 # define _aligned_attribute(n) __declspec(align(n))
23318 #endif
23319  
23320  
23321 #if defined(__GNUC__) || __has_attribute(target)
23322 # define _target_attribute(attrs) __attribute__((target(attrs)))
23323 #else
23324 # define _target_attribute(attrs)
23325 #endif
23326  
23327  
23328  
23329  
23330  
23331 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
23332 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
23333 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
23334 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
23335 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
23336 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
23337 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
23338  
23339  
23340  
23341  
23342  
23343  
23344 #if defined(__BYTE_ORDER__)
23345 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
23346 #elif defined(_MSC_VER)
23347 # define CPU_IS_LITTLE_ENDIAN() true
23348 #else
23349 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
23350 {
23351 union {
23352 u32 w;
23353 u8 b;
23354 } u;
23355  
23356 u.w = 1;
23357 return u.b;
23358 }
23359 #endif
23360  
23361  
23362 static forceinline u16 bswap16(u16 v)
23363 {
23364 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
23365 return __builtin_bswap16(v);
23366 #elif defined(_MSC_VER)
23367 return _byteswap_ushort(v);
23368 #else
23369 return (v << 8) | (v >> 8);
23370 #endif
23371 }
23372  
23373  
23374 static forceinline u32 bswap32(u32 v)
23375 {
23376 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
23377 return __builtin_bswap32(v);
23378 #elif defined(_MSC_VER)
23379 return _byteswap_ulong(v);
23380 #else
23381 return ((v & 0x000000FF) << 24) |
23382 ((v & 0x0000FF00) << 8) |
23383 ((v & 0x00FF0000) >> 8) |
23384 ((v & 0xFF000000) >> 24);
23385 #endif
23386 }
23387  
23388  
23389 static forceinline u64 bswap64(u64 v)
23390 {
23391 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
23392 return __builtin_bswap64(v);
23393 #elif defined(_MSC_VER)
23394 return _byteswap_uint64(v);
23395 #else
23396 return ((v & 0x00000000000000FF) << 56) |
23397 ((v & 0x000000000000FF00) << 40) |
23398 ((v & 0x0000000000FF0000) << 24) |
23399 ((v & 0x00000000FF000000) << 8) |
23400 ((v & 0x000000FF00000000) >> 8) |
23401 ((v & 0x0000FF0000000000) >> 24) |
23402 ((v & 0x00FF000000000000) >> 40) |
23403 ((v & 0xFF00000000000000) >> 56);
23404 #endif
23405 }
23406  
23407 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
23408 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
23409 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
23410 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
23411 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
23412 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
23413  
23414  
23415  
23416  
23417  
23418  
23419 #if (defined(__GNUC__) || defined(__clang__)) && \
23420 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
23421 defined(__ARM_FEATURE_UNALIGNED) || \
23422 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
23423 defined(__riscv_misaligned_fast) || \
23424 defined(__wasm__))
23425 # define UNALIGNED_ACCESS_IS_FAST 1
23426 #elif defined(_MSC_VER)
23427 # define UNALIGNED_ACCESS_IS_FAST 1
23428 #else
23429 # define UNALIGNED_ACCESS_IS_FAST 0
23430 #endif
23431  
23432  
23433  
23434 #ifdef FREESTANDING
23435 # define MEMCOPY __builtin_memcpy
23436 #else
23437 # define MEMCOPY memcpy
23438 #endif
23439  
23440  
23441  
23442 #define DEFINE_UNALIGNED_TYPE(type) \
23443 static forceinline type \
23444 load_##type##_unaligned(const void *p) \
23445 { \
23446 type v; \
23447 \
23448 MEMCOPY(&v, p, sizeof(v)); \
23449 return v; \
23450 } \
23451 \
23452 static forceinline void \
23453 store_##type##_unaligned(type v, void *p) \
23454 { \
23455 MEMCOPY(p, &v, sizeof(v)); \
23456 }
23457  
23458 DEFINE_UNALIGNED_TYPE(u16)
23459 DEFINE_UNALIGNED_TYPE(u32)
23460 DEFINE_UNALIGNED_TYPE(u64)
23461 DEFINE_UNALIGNED_TYPE(machine_word_t)
23462  
23463 #undef MEMCOPY
23464  
23465 #define load_word_unaligned load_machine_word_t_unaligned
23466 #define store_word_unaligned store_machine_word_t_unaligned
23467  
23468  
23469  
23470 static forceinline u16
23471 get_unaligned_le16(const u8 *p)
23472 {
23473 if (UNALIGNED_ACCESS_IS_FAST)
23474 return le16_bswap(load_u16_unaligned(p));
23475 else
23476 return ((u16)p[1] << 8) | p[0];
23477 }
23478  
23479 static forceinline u16
23480 get_unaligned_be16(const u8 *p)
23481 {
23482 if (UNALIGNED_ACCESS_IS_FAST)
23483 return be16_bswap(load_u16_unaligned(p));
23484 else
23485 return ((u16)p[0] << 8) | p[1];
23486 }
23487  
23488 static forceinline u32
23489 get_unaligned_le32(const u8 *p)
23490 {
23491 if (UNALIGNED_ACCESS_IS_FAST)
23492 return le32_bswap(load_u32_unaligned(p));
23493 else
23494 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
23495 ((u32)p[1] << 8) | p[0];
23496 }
23497  
23498 static forceinline u32
23499 get_unaligned_be32(const u8 *p)
23500 {
23501 if (UNALIGNED_ACCESS_IS_FAST)
23502 return be32_bswap(load_u32_unaligned(p));
23503 else
23504 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
23505 ((u32)p[2] << 8) | p[3];
23506 }
23507  
23508 static forceinline u64
23509 get_unaligned_le64(const u8 *p)
23510 {
23511 if (UNALIGNED_ACCESS_IS_FAST)
23512 return le64_bswap(load_u64_unaligned(p));
23513 else
23514 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
23515 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
23516 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
23517 ((u64)p[1] << 8) | p[0];
23518 }
23519  
23520 static forceinline machine_word_t
23521 get_unaligned_leword(const u8 *p)
23522 {
23523 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
23524 if (WORDBITS == 32)
23525 return get_unaligned_le32(p);
23526 else
23527 return get_unaligned_le64(p);
23528 }
23529  
23530  
23531  
23532 static forceinline void
23533 put_unaligned_le16(u16 v, u8 *p)
23534 {
23535 if (UNALIGNED_ACCESS_IS_FAST) {
23536 store_u16_unaligned(le16_bswap(v), p);
23537 } else {
23538 p[0] = (u8)(v >> 0);
23539 p[1] = (u8)(v >> 8);
23540 }
23541 }
23542  
23543 static forceinline void
23544 put_unaligned_be16(u16 v, u8 *p)
23545 {
23546 if (UNALIGNED_ACCESS_IS_FAST) {
23547 store_u16_unaligned(be16_bswap(v), p);
23548 } else {
23549 p[0] = (u8)(v >> 8);
23550 p[1] = (u8)(v >> 0);
23551 }
23552 }
23553  
23554 static forceinline void
23555 put_unaligned_le32(u32 v, u8 *p)
23556 {
23557 if (UNALIGNED_ACCESS_IS_FAST) {
23558 store_u32_unaligned(le32_bswap(v), p);
23559 } else {
23560 p[0] = (u8)(v >> 0);
23561 p[1] = (u8)(v >> 8);
23562 p[2] = (u8)(v >> 16);
23563 p[3] = (u8)(v >> 24);
23564 }
23565 }
23566  
23567 static forceinline void
23568 put_unaligned_be32(u32 v, u8 *p)
23569 {
23570 if (UNALIGNED_ACCESS_IS_FAST) {
23571 store_u32_unaligned(be32_bswap(v), p);
23572 } else {
23573 p[0] = (u8)(v >> 24);
23574 p[1] = (u8)(v >> 16);
23575 p[2] = (u8)(v >> 8);
23576 p[3] = (u8)(v >> 0);
23577 }
23578 }
23579  
23580 static forceinline void
23581 put_unaligned_le64(u64 v, u8 *p)
23582 {
23583 if (UNALIGNED_ACCESS_IS_FAST) {
23584 store_u64_unaligned(le64_bswap(v), p);
23585 } else {
23586 p[0] = (u8)(v >> 0);
23587 p[1] = (u8)(v >> 8);
23588 p[2] = (u8)(v >> 16);
23589 p[3] = (u8)(v >> 24);
23590 p[4] = (u8)(v >> 32);
23591 p[5] = (u8)(v >> 40);
23592 p[6] = (u8)(v >> 48);
23593 p[7] = (u8)(v >> 56);
23594 }
23595 }
23596  
23597 static forceinline void
23598 put_unaligned_leword(machine_word_t v, u8 *p)
23599 {
23600 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
23601 if (WORDBITS == 32)
23602 put_unaligned_le32(v, p);
23603 else
23604 put_unaligned_le64(v, p);
23605 }
23606  
23607  
23608  
23609  
23610  
23611  
23612  
23613 static forceinline unsigned
23614 bsr32(u32 v)
23615 {
23616 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
23617 return 31 - __builtin_clz(v);
23618 #elif defined(_MSC_VER)
23619 unsigned long i;
23620  
23621 _BitScanReverse(&i, v);
23622 return i;
23623 #else
23624 unsigned i = 0;
23625  
23626 while ((v >>= 1) != 0)
23627 i++;
23628 return i;
23629 #endif
23630 }
23631  
23632 static forceinline unsigned
23633 bsr64(u64 v)
23634 {
23635 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
23636 return 63 - __builtin_clzll(v);
23637 #elif defined(_MSC_VER) && defined(_WIN64)
23638 unsigned long i;
23639  
23640 _BitScanReverse64(&i, v);
23641 return i;
23642 #else
23643 unsigned i = 0;
23644  
23645 while ((v >>= 1) != 0)
23646 i++;
23647 return i;
23648 #endif
23649 }
23650  
23651 static forceinline unsigned
23652 bsrw(machine_word_t v)
23653 {
23654 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
23655 if (WORDBITS == 32)
23656 return bsr32(v);
23657 else
23658 return bsr64(v);
23659 }
23660  
23661  
23662  
23663 static forceinline unsigned
23664 bsf32(u32 v)
23665 {
23666 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
23667 return __builtin_ctz(v);
23668 #elif defined(_MSC_VER)
23669 unsigned long i;
23670  
23671 _BitScanForward(&i, v);
23672 return i;
23673 #else
23674 unsigned i = 0;
23675  
23676 for (; (v & 1) == 0; v >>= 1)
23677 i++;
23678 return i;
23679 #endif
23680 }
23681  
23682 static forceinline unsigned
23683 bsf64(u64 v)
23684 {
23685 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
23686 return __builtin_ctzll(v);
23687 #elif defined(_MSC_VER) && defined(_WIN64)
23688 unsigned long i;
23689  
23690 _BitScanForward64(&i, v);
23691 return i;
23692 #else
23693 unsigned i = 0;
23694  
23695 for (; (v & 1) == 0; v >>= 1)
23696 i++;
23697 return i;
23698 #endif
23699 }
23700  
23701 static forceinline unsigned
23702 bsfw(machine_word_t v)
23703 {
23704 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
23705 if (WORDBITS == 32)
23706 return bsf32(v);
23707 else
23708 return bsf64(v);
23709 }
23710  
23711  
23712 #undef rbit32
23713 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
23714 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
23715 static forceinline u32
23716 rbit32(u32 v)
23717 {
23718 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
23719 return v;
23720 }
23721 #define rbit32 rbit32
23722 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
23723 static forceinline u32
23724 rbit32(u32 v)
23725 {
23726 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
23727 return v;
23728 }
23729 #define rbit32 rbit32
23730 #endif
23731  
23732 #endif
23733  
23734  
23735 typedef void *(*malloc_func_t)(size_t);
23736 typedef void (*free_func_t)(void *);
23737  
23738 extern malloc_func_t libdeflate_default_malloc_func;
23739 extern free_func_t libdeflate_default_free_func;
23740  
23741 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
23742 size_t alignment, size_t size);
23743 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
23744  
23745 #ifdef FREESTANDING
23746  
23747 void *memset(void *s, int c, size_t n);
23748 #define memset(s, c, n) __builtin_memset((s), (c), (n))
23749  
23750 void *memcpy(void *dest, const void *src, size_t n);
23751 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
23752  
23753 void *memmove(void *dest, const void *src, size_t n);
23754 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
23755  
23756 int memcmp(const void *s1, const void *s2, size_t n);
23757 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
23758  
23759 #undef LIBDEFLATE_ENABLE_ASSERTIONS
23760 #else
23761 # include <string.h>
23762
23763 # ifdef __clang_analyzer__
23764 # define LIBDEFLATE_ENABLE_ASSERTIONS
23765 # endif
23766 #endif
23767  
23768  
23769 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
23770 NORETURN void
23771 libdeflate_assertion_failed(const char *expr, const char *file, int line);
23772 #define ASSERT(expr) { if (unlikely(!(expr))) \
23773 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
23774 #else
23775 #define ASSERT(expr) (void)(expr)
23776 #endif
23777  
23778 #define CONCAT_IMPL(a, b) a##b
23779 #define CONCAT(a, b) CONCAT_IMPL(a, b)
23780 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
23781  
23782 #endif
23783  
23784  
23785 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
23786  
23787 #define X86_CPU_FEATURE_SSE2 (1 << 0)
23788 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
23789 #define X86_CPU_FEATURE_AVX (1 << 2)
23790 #define X86_CPU_FEATURE_AVX2 (1 << 3)
23791 #define X86_CPU_FEATURE_BMI2 (1 << 4)
23792  
23793 #define X86_CPU_FEATURE_ZMM (1 << 5)
23794 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
23795 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
23796 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
23797 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
23798 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
23799  
23800 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
23801  
23802 # define X86_CPU_FEATURES_KNOWN (1U << 31)
23803 extern volatile u32 libdeflate_x86_cpu_features;
23804  
23805 void libdeflate_init_x86_cpu_features(void);
23806  
23807 static inline u32 get_x86_cpu_features(void)
23808 {
23809 if (libdeflate_x86_cpu_features == 0)
23810 libdeflate_init_x86_cpu_features();
23811 return libdeflate_x86_cpu_features;
23812 }
23813  
23814 # include <immintrin.h>
23815 # if defined(_MSC_VER) && defined(__clang__)
23816 # include <tmmintrin.h>
23817 # include <smmintrin.h>
23818 # include <wmmintrin.h>
23819 # include <avxintrin.h>
23820 # include <avx2intrin.h>
23821 # include <avx512fintrin.h>
23822 # include <avx512bwintrin.h>
23823 # include <avx512vlintrin.h>
23824 # if __has_include(<avx512vlbwintrin.h>)
23825 # include <avx512vlbwintrin.h>
23826 # endif
23827 # if __has_include(<vpclmulqdqintrin.h>)
23828 # include <vpclmulqdqintrin.h>
23829 # endif
23830 # if __has_include(<avx512vnniintrin.h>)
23831 # include <avx512vnniintrin.h>
23832 # endif
23833 # if __has_include(<avx512vlvnniintrin.h>)
23834 # include <avx512vlvnniintrin.h>
23835 # endif
23836 # if __has_include(<avxvnniintrin.h>)
23837 # include <avxvnniintrin.h>
23838 # endif
23839 # endif
23840 #else
23841 static inline u32 get_x86_cpu_features(void) { return 0; }
23842 #endif
23843  
23844 #if defined(__SSE2__) || \
23845 (defined(_MSC_VER) && \
23846 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
23847 # define HAVE_SSE2(features) 1
23848 # define HAVE_SSE2_NATIVE 1
23849 #else
23850 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
23851 # define HAVE_SSE2_NATIVE 0
23852 #endif
23853  
23854 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
23855 (defined(_MSC_VER) && defined(__AVX2__))
23856 # define HAVE_PCLMULQDQ(features) 1
23857 #else
23858 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
23859 #endif
23860  
23861 #ifdef __AVX__
23862 # define HAVE_AVX(features) 1
23863 #else
23864 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
23865 #endif
23866  
23867 #ifdef __AVX2__
23868 # define HAVE_AVX2(features) 1
23869 #else
23870 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
23871 #endif
23872  
23873 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
23874 # define HAVE_BMI2(features) 1
23875 # define HAVE_BMI2_NATIVE 1
23876 #else
23877 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
23878 # define HAVE_BMI2_NATIVE 0
23879 #endif
23880  
23881 #ifdef __AVX512BW__
23882 # define HAVE_AVX512BW(features) 1
23883 #else
23884 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
23885 #endif
23886  
23887 #ifdef __AVX512VL__
23888 # define HAVE_AVX512VL(features) 1
23889 #else
23890 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
23891 #endif
23892  
23893 #ifdef __VPCLMULQDQ__
23894 # define HAVE_VPCLMULQDQ(features) 1
23895 #else
23896 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
23897 #endif
23898  
23899 #ifdef __AVX512VNNI__
23900 # define HAVE_AVX512VNNI(features) 1
23901 #else
23902 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
23903 #endif
23904  
23905 #ifdef __AVXVNNI__
23906 # define HAVE_AVXVNNI(features) 1
23907 #else
23908 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
23909 #endif
23910  
23911 #endif
23912  
23913 #endif
23914  
23915  
23916 #ifdef __AVX2__
23917 static forceinline void
23918 matchfinder_init_avx2(mf_pos_t *data, size_t size)
23919 {
23920 __m256i *p = (__m256i *)data;
23921 __m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
23922  
23923 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
23924 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
23925 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
23926  
23927 do {
23928 p[0] = v;
23929 p[1] = v;
23930 p[2] = v;
23931 p[3] = v;
23932 p += 4;
23933 size -= 4 * sizeof(*p);
23934 } while (size != 0);
23935 }
23936 #define matchfinder_init matchfinder_init_avx2
23937  
23938 static forceinline void
23939 matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
23940 {
23941 __m256i *p = (__m256i *)data;
23942 __m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
23943  
23944 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
23945 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
23946 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
23947  
23948 do {
23949
23950 p[0] = _mm256_adds_epi16(p[0], v);
23951 p[1] = _mm256_adds_epi16(p[1], v);
23952 p[2] = _mm256_adds_epi16(p[2], v);
23953 p[3] = _mm256_adds_epi16(p[3], v);
23954 p += 4;
23955 size -= 4 * sizeof(*p);
23956 } while (size != 0);
23957 }
23958 #define matchfinder_rebase matchfinder_rebase_avx2
23959  
23960 #elif HAVE_SSE2_NATIVE
23961 static forceinline void
23962 matchfinder_init_sse2(mf_pos_t *data, size_t size)
23963 {
23964 __m128i *p = (__m128i *)data;
23965 __m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
23966  
23967 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
23968 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
23969 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
23970  
23971 do {
23972 p[0] = v;
23973 p[1] = v;
23974 p[2] = v;
23975 p[3] = v;
23976 p += 4;
23977 size -= 4 * sizeof(*p);
23978 } while (size != 0);
23979 }
23980 #define matchfinder_init matchfinder_init_sse2
23981  
23982 static forceinline void
23983 matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
23984 {
23985 __m128i *p = (__m128i *)data;
23986 __m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
23987  
23988 STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
23989 STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
23990 STATIC_ASSERT(sizeof(mf_pos_t) == 2);
23991  
23992 do {
23993
23994 p[0] = _mm_adds_epi16(p[0], v);
23995 p[1] = _mm_adds_epi16(p[1], v);
23996 p[2] = _mm_adds_epi16(p[2], v);
23997 p[3] = _mm_adds_epi16(p[3], v);
23998 p += 4;
23999 size -= 4 * sizeof(*p);
24000 } while (size != 0);
24001 }
24002 #define matchfinder_rebase matchfinder_rebase_sse2
24003 #endif
24004  
24005 #endif
24006  
24007 # endif
24008 #else
24009 # define MATCHFINDER_ALIGNED
24010 #endif
24011  
24012  
24013 #ifndef matchfinder_init
24014 static forceinline void
24015 matchfinder_init(mf_pos_t *data, size_t size)
24016 {
24017 size_t num_entries = size / sizeof(*data);
24018 size_t i;
24019  
24020 for (i = 0; i < num_entries; i++)
24021 data[i] = MATCHFINDER_INITVAL;
24022 }
24023 #endif
24024  
24025  
24026 #ifndef matchfinder_rebase
24027 static forceinline void
24028 matchfinder_rebase(mf_pos_t *data, size_t size)
24029 {
24030 size_t num_entries = size / sizeof(*data);
24031 size_t i;
24032  
24033 if (MATCHFINDER_WINDOW_SIZE == 32768) {
24034
24035 for (i = 0; i < num_entries; i++)
24036 data[i] = 0x8000 | (data[i] & ~(data[i] >> 15));
24037 } else {
24038 for (i = 0; i < num_entries; i++) {
24039 if (data[i] >= 0)
24040 data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
24041 else
24042 data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
24043 }
24044 }
24045 }
24046 #endif
24047  
24048  
24049 static forceinline u32
24050 lz_hash(u32 seq, unsigned num_bits)
24051 {
24052 return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
24053 }
24054  
24055  
24056 static forceinline u32
24057 lz_extend(const u8 * const strptr, const u8 * const matchptr,
24058 const u32 start_len, const u32 max_len)
24059 {
24060 u32 len = start_len;
24061 machine_word_t v_word;
24062  
24063 if (UNALIGNED_ACCESS_IS_FAST) {
24064  
24065 if (likely(max_len - len >= 4 * WORDBYTES)) {
24066  
24067 #define COMPARE_WORD_STEP \
24068 v_word = load_word_unaligned(&matchptr[len]) ^ \
24069 load_word_unaligned(&strptr[len]); \
24070 if (v_word != 0) \
24071 goto word_differs; \
24072 len += WORDBYTES; \
24073  
24074 COMPARE_WORD_STEP
24075 COMPARE_WORD_STEP
24076 COMPARE_WORD_STEP
24077 COMPARE_WORD_STEP
24078 #undef COMPARE_WORD_STEP
24079 }
24080  
24081 while (len + WORDBYTES <= max_len) {
24082 v_word = load_word_unaligned(&matchptr[len]) ^
24083 load_word_unaligned(&strptr[len]);
24084 if (v_word != 0)
24085 goto word_differs;
24086 len += WORDBYTES;
24087 }
24088 }
24089  
24090 while (len < max_len && matchptr[len] == strptr[len])
24091 len++;
24092 return len;
24093  
24094 word_differs:
24095 if (CPU_IS_LITTLE_ENDIAN())
24096 len += (bsfw(v_word) >> 3);
24097 else
24098 len += (WORDBITS - 1 - bsrw(v_word)) >> 3;
24099 return len;
24100 }
24101  
24102 #endif
24103  
24104  
24105 #define BT_MATCHFINDER_HASH3_ORDER 16
24106 #define BT_MATCHFINDER_HASH3_WAYS 2
24107 #define BT_MATCHFINDER_HASH4_ORDER 16
24108  
24109 #define BT_MATCHFINDER_TOTAL_HASH_SIZE \
24110 (((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
24111 (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
24112  
24113  
24114 struct lz_match {
24115  
24116
24117 u16 length;
24118  
24119
24120 u16 offset;
24121 };
24122  
24123 struct MATCHFINDER_ALIGNED bt_matchfinder {
24124  
24125
24126 mf_pos_t hash3_tab[1UL << BT_MATCHFINDER_HASH3_ORDER][BT_MATCHFINDER_HASH3_WAYS];
24127  
24128
24129 mf_pos_t hash4_tab[1UL << BT_MATCHFINDER_HASH4_ORDER];
24130  
24131
24132 mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE];
24133 };
24134  
24135  
24136 static forceinline void
24137 bt_matchfinder_init(struct bt_matchfinder *mf)
24138 {
24139 STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE %
24140 MATCHFINDER_SIZE_ALIGNMENT == 0);
24141  
24142 matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE);
24143 9 }
24144  
24145 static forceinline void
24146 bt_matchfinder_slide_window(struct bt_matchfinder *mf)
24147 {
24148 STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
24149  
24150 matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
24151 0 }
24152  
24153 static forceinline mf_pos_t *
24154 bt_left_child(struct bt_matchfinder *mf, s32 node)
24155 {
24156 15102 return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 0];
24157 }
24158  
24159 static forceinline mf_pos_t *
24160 bt_right_child(struct bt_matchfinder *mf, s32 node)
24161 {
24162 30366 return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 1];
24163 }
24164  
24165  
24166 #define BT_MATCHFINDER_REQUIRED_NBYTES 5
24167  
24168  
24169 static forceinline struct lz_match *
24170 bt_matchfinder_advance_one_byte(struct bt_matchfinder * const mf,
24171 const u8 * const in_base,
24172 const ptrdiff_t cur_pos,
24173 const u32 max_len,
24174 const u32 nice_len,
24175 const u32 max_search_depth,
24176 u32 * const next_hashes,
24177 struct lz_match *lz_matchptr,
24178 const bool record_matches)
24179 {
24180 15264 const u8 *in_next = in_base + cur_pos;
24181 15264 u32 depth_remaining = max_search_depth;
24182 15264 const s32 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
24183 u32 next_hashseq;
24184 u32 hash3;
24185 u32 hash4;
24186 s32 cur_node;
24187 #if BT_MATCHFINDER_HASH3_WAYS >= 2
24188 s32 cur_node_2;
24189 #endif
24190 const u8 *matchptr;
24191 mf_pos_t *pending_lt_ptr, *pending_gt_ptr;
24192 u32 best_lt_len, best_gt_len;
24193 u32 len;
24194 15264 u32 best_len = 3;
24195  
24196 STATIC_ASSERT(BT_MATCHFINDER_HASH3_WAYS >= 1 &&
24197 BT_MATCHFINDER_HASH3_WAYS <= 2);
24198  
24199 30528 next_hashseq = get_unaligned_le32(in_next + 1);
24200  
24201 15264 hash3 = next_hashes[0];
24202 15264 hash4 = next_hashes[1];
24203  
24204 15264 next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, BT_MATCHFINDER_HASH3_ORDER);
24205 15264 next_hashes[1] = lz_hash(next_hashseq, BT_MATCHFINDER_HASH4_ORDER);
24206 15264 prefetchw(&mf->hash3_tab[next_hashes[0]]);
24207 15264 prefetchw(&mf->hash4_tab[next_hashes[1]]);
24208  
24209 15264 cur_node = mf->hash3_tab[hash3][0];
24210 15264 mf->hash3_tab[hash3][0] = cur_pos;
24211 #if BT_MATCHFINDER_HASH3_WAYS >= 2
24212 15264 cur_node_2 = mf->hash3_tab[hash3][1];
24213 15264 mf->hash3_tab[hash3][1] = cur_node;
24214 #endif
24215 225 ▶ if (record_matches && cur_node > cutoff) {
Branch: 1 branch
BranchTrueFalseTotal
if (record_matches && cur_node > cutoff) { 63 162 225
24216 63 u32 seq3 = load_u24_unaligned(in_next);
24217 126 ▶ if (seq3 == load_u24_unaligned(&in_base[cur_node])) {
Branch: 1 branch
BranchTrueFalseTotal
if (seq3 == load_u24_unaligned(&in_base[cur_node])) { 63 0 63
24218 63 lz_matchptr->length = 3;
24219 63 lz_matchptr->offset = in_next - &in_base[cur_node];
24220 63 lz_matchptr++;
24221 }
24222 #if BT_MATCHFINDER_HASH3_WAYS >= 2
24223 0 ▶ else if (cur_node_2 > cutoff &&
Branch: 1 branch
BranchTrueFalseTotal
else if (cur_node_2 > cutoff && 0 0 0
24224 0 ▶ seq3 == load_u24_unaligned(&in_base[cur_node_2]))
Branch: 1 branch
BranchTrueFalseTotal
seq3 == load_u24_unaligned(&in_base[cur_node_2])) 0 0 0
24225 {
24226 0 lz_matchptr->length = 3;
24227 0 lz_matchptr->offset = in_next - &in_base[cur_node_2];
24228 0 lz_matchptr++;
24229 }
24230 #endif
24231 }
24232  
24233 15264 cur_node = mf->hash4_tab[hash4];
24234 15264 mf->hash4_tab[hash4] = cur_pos;
24235  
24236 15264 pending_lt_ptr = bt_left_child(mf, cur_pos);
24237 15264 pending_gt_ptr = bt_right_child(mf, cur_pos);
24238  
24239 15264 ▶ if (cur_node <= cutoff) {
Branch: 2 branches
BranchTrueFalseTotal
if (cur_node <= cutoff) { 162 63 225
if (cur_node <= cutoff) { 0 15039 15039
24240 162 *pending_lt_ptr = MATCHFINDER_INITVAL;
24241 162 *pending_gt_ptr = MATCHFINDER_INITVAL;
24242 162 return lz_matchptr;
24243 }
24244  
24245 15102 best_lt_len = 0;
24246 15102 best_gt_len = 0;
24247 15102 len = 0;
24248  
24249 for (;;) {
24250 15102 matchptr = &in_base[cur_node];
24251  
24252 15102 ▶ if (matchptr[len] == in_next[len]) {
Branch: 2 branches
BranchTrueFalseTotal
if (matchptr[len] == in_next[len]) { 63 0 63
if (matchptr[len] == in_next[len]) { 15039 0 15039
24253 15102 len = lz_extend(in_next, matchptr, len + 1, max_len);
24254 15102 ▶ if (!record_matches || len > best_len) {
Branch: 4 branches
BranchTrueFalseTotal
if (!record_matches || len > best_len) { 63 0 63
if (!record_matches || len > best_len) { 63 0 63
if (!record_matches || len > best_len) { 0 15039 15039
if (!record_matches || len > best_len) { 0 0 0
24255 if (record_matches) {
24256 63 best_len = len;
24257 63 lz_matchptr->length = len;
24258 63 lz_matchptr->offset = in_next - matchptr;
24259 63 lz_matchptr++;
24260 }
24261 15102 ▶ if (len >= nice_len) {
Branch: 2 branches
BranchTrueFalseTotal
if (len >= nice_len) { 63 0 63
if (len >= nice_len) { 15039 0 15039
24262 15102 *pending_lt_ptr = *bt_left_child(mf, cur_node);
24263 15102 *pending_gt_ptr = *bt_right_child(mf, cur_node);
24264 15102 return lz_matchptr;
24265 }
24266 }
24267 }
24268  
24269 0 ▶ if (matchptr[len] < in_next[len]) {
Branch: 2 branches
BranchTrueFalseTotal
if (matchptr[len] < in_next[len]) { 0 0 0
if (matchptr[len] < in_next[len]) { 0 0 0
24270 0 *pending_lt_ptr = cur_node;
24271 0 pending_lt_ptr = bt_right_child(mf, cur_node);
24272 0 cur_node = *pending_lt_ptr;
24273 0 best_lt_len = len;
24274 0 ▶ if (best_gt_len < len)
Branch: 2 branches
BranchTrueFalseTotal
if (best_gt_len < len) 0 0 0
if (best_gt_len < len) 0 0 0
24275 0 len = best_gt_len;
24276 } else {
24277 0 *pending_gt_ptr = cur_node;
24278 0 pending_gt_ptr = bt_left_child(mf, cur_node);
24279 0 cur_node = *pending_gt_ptr;
24280 0 best_gt_len = len;
24281 0 ▶ if (best_lt_len < len)
Branch: 2 branches
BranchTrueFalseTotal
if (best_lt_len < len) 0 0 0
if (best_lt_len < len) 0 0 0
24282 0 len = best_lt_len;
24283 }
24284  
24285 0 ▶ if (cur_node <= cutoff || !--depth_remaining) {
Branch: 4 branches
BranchTrueFalseTotal
if (cur_node <= cutoff || !--depth_remaining) { 0 0 0
if (cur_node <= cutoff || !--depth_remaining) { 0 0 0
if (cur_node <= cutoff || !--depth_remaining) { 0 0 0
if (cur_node <= cutoff || !--depth_remaining) { 0 0 0
24286 0 *pending_lt_ptr = MATCHFINDER_INITVAL;
24287 0 *pending_gt_ptr = MATCHFINDER_INITVAL;
24288 0 return lz_matchptr;
24289 }
24290 }
24291 }
24292  
24293  
24294 static forceinline struct lz_match *
24295 bt_matchfinder_get_matches(struct bt_matchfinder *mf,
24296 const u8 *in_base,
24297 ptrdiff_t cur_pos,
24298 u32 max_len,
24299 u32 nice_len,
24300 u32 max_search_depth,
24301 u32 next_hashes[2],
24302 struct lz_match *lz_matchptr)
24303 {
24304 225 return bt_matchfinder_advance_one_byte(mf,
24305 in_base,
24306 cur_pos,
24307 max_len,
24308 nice_len,
24309 max_search_depth,
24310 next_hashes,
24311 lz_matchptr,
24312 true);
24313 }
24314  
24315  
24316 static forceinline void
24317 bt_matchfinder_skip_byte(struct bt_matchfinder *mf,
24318 const u8 *in_base,
24319 ptrdiff_t cur_pos,
24320 u32 nice_len,
24321 u32 max_search_depth,
24322 u32 next_hashes[2])
24323 {
24324 bt_matchfinder_advance_one_byte(mf,
24325 in_base,
24326 cur_pos,
24327 nice_len,
24328 nice_len,
24329 max_search_depth,
24330 next_hashes,
24331 NULL,
24332 false);
24333 15039 }
24334  
24335 #endif
24336  
24337  
24338 #define MAX_MATCHES_PER_POS \
24339 (DEFLATE_MAX_MATCH_LEN - DEFLATE_MIN_MATCH_LEN + 1)
24340 #endif
24341  
24342  
24343 #define MAX_BLOCK_LENGTH \
24344 MAX(SOFT_MAX_BLOCK_LENGTH + MIN_BLOCK_LENGTH - 1, \
24345 SOFT_MAX_BLOCK_LENGTH + 1 + DEFLATE_MAX_MATCH_LEN)
24346  
24347 static forceinline void
24348 check_buildtime_parameters(void)
24349 {
24350
24351 STATIC_ASSERT(SOFT_MAX_BLOCK_LENGTH >= MIN_BLOCK_LENGTH);
24352 STATIC_ASSERT(FAST_SOFT_MAX_BLOCK_LENGTH >= MIN_BLOCK_LENGTH);
24353 STATIC_ASSERT(SEQ_STORE_LENGTH * DEFLATE_MIN_MATCH_LEN >=
24354 MIN_BLOCK_LENGTH);
24355 STATIC_ASSERT(FAST_SEQ_STORE_LENGTH * HT_MATCHFINDER_MIN_MATCH_LEN >=
24356 MIN_BLOCK_LENGTH);
24357 #if SUPPORT_NEAR_OPTIMAL_PARSING
24358 STATIC_ASSERT(MIN_BLOCK_LENGTH * MAX_MATCHES_PER_POS <=
24359 MATCH_CACHE_LENGTH);
24360 #endif
24361  
24362
24363 STATIC_ASSERT(FAST_SOFT_MAX_BLOCK_LENGTH <= SOFT_MAX_BLOCK_LENGTH);
24364  
24365
24366 STATIC_ASSERT(SEQ_STORE_LENGTH * DEFLATE_MIN_MATCH_LEN <=
24367 SOFT_MAX_BLOCK_LENGTH + MIN_BLOCK_LENGTH);
24368 STATIC_ASSERT(FAST_SEQ_STORE_LENGTH * HT_MATCHFINDER_MIN_MATCH_LEN <=
24369 FAST_SOFT_MAX_BLOCK_LENGTH + MIN_BLOCK_LENGTH);
24370  
24371
24372 STATIC_ASSERT(
24373 MAX_LITLEN_CODEWORD_LEN <= DEFLATE_MAX_LITLEN_CODEWORD_LEN);
24374 STATIC_ASSERT(
24375 MAX_OFFSET_CODEWORD_LEN <= DEFLATE_MAX_OFFSET_CODEWORD_LEN);
24376 STATIC_ASSERT(
24377 MAX_PRE_CODEWORD_LEN <= DEFLATE_MAX_PRE_CODEWORD_LEN);
24378 STATIC_ASSERT(
24379 (1U << MAX_LITLEN_CODEWORD_LEN) >= DEFLATE_NUM_LITLEN_SYMS);
24380 STATIC_ASSERT(
24381 (1U << MAX_OFFSET_CODEWORD_LEN) >= DEFLATE_NUM_OFFSET_SYMS);
24382 STATIC_ASSERT(
24383 (1U << MAX_PRE_CODEWORD_LEN) >= DEFLATE_NUM_PRECODE_SYMS);
24384 37 }
24385  
24386  
24387  
24388  
24389 static const u32 deflate_length_slot_base[] = {
24390 3, 4, 5, 6, 7, 8, 9, 10,
24391 11, 13, 15, 17, 19, 23, 27, 31,
24392 35, 43, 51, 59, 67, 83, 99, 115,
24393 131, 163, 195, 227, 258,
24394 };
24395  
24396  
24397 static const u8 deflate_extra_length_bits[] = {
24398 0, 0, 0, 0, 0, 0, 0, 0,
24399 1, 1, 1, 1, 2, 2, 2, 2,
24400 3, 3, 3, 3, 4, 4, 4, 4,
24401 5, 5, 5, 5, 0,
24402 };
24403  
24404  
24405 static const u32 deflate_offset_slot_base[] = {
24406 1, 2, 3, 4, 5, 7, 9, 13,
24407 17, 25, 33, 49, 65, 97, 129, 193,
24408 257, 385, 513, 769, 1025, 1537, 2049, 3073,
24409 4097, 6145, 8193, 12289, 16385, 24577,
24410 };
24411  
24412  
24413 static const u8 deflate_extra_offset_bits[] = {
24414 0, 0, 0, 0, 1, 1, 2, 2,
24415 3, 3, 4, 4, 5, 5, 6, 6,
24416 7, 7, 8, 8, 9, 9, 10, 10,
24417 11, 11, 12, 12, 13, 13,
24418 };
24419  
24420  
24421 static const u8 deflate_length_slot[DEFLATE_MAX_MATCH_LEN + 1] = {
24422 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12,
24423 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16,
24424 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18,
24425 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20,
24426 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
24427 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
24428 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
24429 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24430 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25,
24431 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
24432 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26,
24433 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
24434 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
24435 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
24436 27, 27, 28,
24437 };
24438  
24439  
24440 static const u8 deflate_offset_slot[256] = {
24441 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
24442 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
24443 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
24444 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
24445 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
24446 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
24447 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
24448 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
24449 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
24450 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
24451 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
24452 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
24453 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
24454 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
24455 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
24456 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
24457 };
24458  
24459  
24460 static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
24461 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
24462 };
24463  
24464  
24465 static const u8 deflate_extra_precode_bits[DEFLATE_NUM_PRECODE_SYMS] = {
24466 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7
24467 };
24468  
24469  
24470 struct deflate_codewords {
24471 u32 litlen[DEFLATE_NUM_LITLEN_SYMS];
24472 u32 offset[DEFLATE_NUM_OFFSET_SYMS];
24473 };
24474  
24475  
24476 struct deflate_lens {
24477 u8 litlen[DEFLATE_NUM_LITLEN_SYMS];
24478 u8 offset[DEFLATE_NUM_OFFSET_SYMS];
24479 };
24480  
24481  
24482 struct deflate_codes {
24483 struct deflate_codewords codewords;
24484 struct deflate_lens lens;
24485 };
24486  
24487  
24488 struct deflate_freqs {
24489 u32 litlen[DEFLATE_NUM_LITLEN_SYMS];
24490 u32 offset[DEFLATE_NUM_OFFSET_SYMS];
24491 };
24492  
24493  
24494 struct deflate_sequence {
24495  
24496
24497 #define SEQ_LENGTH_SHIFT 23
24498 #define SEQ_LITRUNLEN_MASK (((u32)1 << SEQ_LENGTH_SHIFT) - 1)
24499 u32 litrunlen_and_length;
24500  
24501
24502 u16 offset;
24503  
24504
24505 u16 offset_slot;
24506 };
24507  
24508 #if SUPPORT_NEAR_OPTIMAL_PARSING
24509  
24510  
24511 struct deflate_costs {
24512  
24513
24514 u32 literal[DEFLATE_NUM_LITERALS];
24515  
24516
24517 u32 length[DEFLATE_MAX_MATCH_LEN + 1];
24518  
24519
24520 u32 offset_slot[DEFLATE_NUM_OFFSET_SYMS];
24521 };
24522  
24523  
24524 struct deflate_optimum_node {
24525  
24526 u32 cost_to_end;
24527  
24528
24529 #define OPTIMUM_OFFSET_SHIFT 9
24530 #define OPTIMUM_LEN_MASK (((u32)1 << OPTIMUM_OFFSET_SHIFT) - 1)
24531 u32 item;
24532  
24533 };
24534  
24535 #endif
24536  
24537  
24538 #define NUM_LITERAL_OBSERVATION_TYPES 8
24539 #define NUM_MATCH_OBSERVATION_TYPES 2
24540 #define NUM_OBSERVATION_TYPES (NUM_LITERAL_OBSERVATION_TYPES + \
24541 NUM_MATCH_OBSERVATION_TYPES)
24542 #define NUM_OBSERVATIONS_PER_BLOCK_CHECK 512
24543 struct block_split_stats {
24544 u32 new_observations[NUM_OBSERVATION_TYPES];
24545 u32 observations[NUM_OBSERVATION_TYPES];
24546 u32 num_new_observations;
24547 u32 num_observations;
24548 };
24549  
24550 struct deflate_output_bitstream;
24551  
24552  
24553 struct libdeflate_compressor {
24554  
24555
24556 void (*impl)(struct libdeflate_compressor *restrict c, const u8 *in,
24557 size_t in_nbytes, struct deflate_output_bitstream *os);
24558  
24559
24560 free_func_t free_func;
24561  
24562
24563 unsigned compression_level;
24564  
24565
24566 size_t max_passthrough_size;
24567  
24568
24569 u32 max_search_depth;
24570  
24571
24572 u32 nice_match_length;
24573  
24574
24575 struct deflate_freqs freqs;
24576  
24577
24578 struct block_split_stats split_stats;
24579  
24580
24581 struct deflate_codes codes;
24582  
24583
24584 struct deflate_codes static_codes;
24585  
24586
24587 union {
24588
24589 struct {
24590 u32 freqs[DEFLATE_NUM_PRECODE_SYMS];
24591 u32 codewords[DEFLATE_NUM_PRECODE_SYMS];
24592 u8 lens[DEFLATE_NUM_PRECODE_SYMS];
24593 unsigned items[DEFLATE_NUM_LITLEN_SYMS +
24594 DEFLATE_NUM_OFFSET_SYMS];
24595 unsigned num_litlen_syms;
24596 unsigned num_offset_syms;
24597 unsigned num_explicit_lens;
24598 unsigned num_items;
24599 } precode;
24600
24601 struct {
24602 u32 codewords[DEFLATE_MAX_MATCH_LEN + 1];
24603 u8 lens[DEFLATE_MAX_MATCH_LEN + 1];
24604 } length;
24605 } o;
24606  
24607 union {
24608
24609 struct {
24610
24611 struct hc_matchfinder hc_mf;
24612  
24613
24614 struct deflate_sequence sequences[SEQ_STORE_LENGTH + 1];
24615  
24616 } g;
24617  
24618
24619 struct {
24620
24621 struct ht_matchfinder ht_mf;
24622  
24623
24624 struct deflate_sequence sequences[
24625 FAST_SEQ_STORE_LENGTH + 1];
24626  
24627 } f;
24628  
24629 #if SUPPORT_NEAR_OPTIMAL_PARSING
24630
24631 struct {
24632  
24633
24634 struct bt_matchfinder bt_mf;
24635  
24636
24637 struct lz_match match_cache[MATCH_CACHE_LENGTH +
24638 MAX_MATCHES_PER_POS +
24639 DEFLATE_MAX_MATCH_LEN - 1];
24640  
24641
24642 struct deflate_optimum_node optimum_nodes[
24643 MAX_BLOCK_LENGTH + 1];
24644  
24645
24646 struct deflate_costs costs;
24647  
24648
24649 struct deflate_costs costs_saved;
24650  
24651
24652 u8 offset_slot_full[DEFLATE_MAX_MATCH_OFFSET + 1];
24653  
24654
24655 u32 prev_observations[NUM_OBSERVATION_TYPES];
24656 u32 prev_num_observations;
24657  
24658
24659 u32 new_match_len_freqs[DEFLATE_MAX_MATCH_LEN + 1];
24660 u32 match_len_freqs[DEFLATE_MAX_MATCH_LEN + 1];
24661  
24662
24663 unsigned max_optim_passes;
24664  
24665
24666 u32 min_improvement_to_continue;
24667  
24668
24669 u32 min_bits_to_use_nonfinal_path;
24670  
24671
24672 u32 max_len_to_optimize_static_block;
24673  
24674 } n;
24675 #endif
24676  
24677 } p;
24678 };
24679  
24680  
24681 typedef machine_word_t bitbuf_t;
24682  
24683  
24684 #define COMPRESS_BITBUF_NBITS (8 * sizeof(bitbuf_t) - 1)
24685  
24686  
24687 #define CAN_BUFFER(n) (7 + (n) <= COMPRESS_BITBUF_NBITS)
24688  
24689  
24690 struct deflate_output_bitstream {
24691  
24692
24693 bitbuf_t bitbuf;
24694  
24695
24696 unsigned bitcount;
24697  
24698
24699 u8 *next;
24700  
24701
24702 u8 *end;
24703  
24704
24705 bool overflow;
24706 };
24707  
24708  
24709 #define ADD_BITS(bits, n) \
24710 do { \
24711 bitbuf |= (bitbuf_t)(bits) << bitcount; \
24712 bitcount += (n); \
24713 ASSERT(bitcount <= COMPRESS_BITBUF_NBITS); \
24714 } while (0)
24715  
24716  
24717 #define FLUSH_BITS() \
24718 do { \
24719 if (UNALIGNED_ACCESS_IS_FAST && likely(out_next < out_fast_end)) { \
24720 \
24721 put_unaligned_leword(bitbuf, out_next); \
24722 bitbuf >>= bitcount & ~7; \
24723 out_next += bitcount >> 3; \
24724 bitcount &= 7; \
24725 } else { \
24726 \
24727 while (bitcount >= 8) { \
24728 ASSERT(out_next < os->end); \
24729 *out_next++ = bitbuf; \
24730 bitcount -= 8; \
24731 bitbuf >>= 8; \
24732 } \
24733 } \
24734 } while (0)
24735  
24736  
24737 static void
24738 35 heapify_subtree(u32 A[], unsigned length, unsigned subtree_idx)
24739 {
24740 unsigned parent_idx;
24741 unsigned child_idx;
24742 u32 v;
24743  
24744 35 v = A[subtree_idx];
24745 35 parent_idx = subtree_idx;
24746 87 ▶ while ((child_idx = parent_idx * 2) <= length) {
Branch: 1 branch
BranchTrueFalseTotal
while ((child_idx = parent_idx * 2) <= length) { 63 24 87
24747 63 ▶ if (child_idx < length && A[child_idx + 1] > A[child_idx])
Branch: 2 branches
BranchTrueFalseTotal
if (child_idx < length && A[child_idx + 1] > A[child_idx]) 55 8 63
if (child_idx < length && A[child_idx + 1] > A[child_idx]) 26 29 55
24748 26 child_idx++;
24749 63 ▶ if (v >= A[child_idx])
Branch: 1 branch
BranchTrueFalseTotal
if (v >= A[child_idx]) 11 52 63
24750 11 break;
24751 52 A[parent_idx] = A[child_idx];
24752 52 parent_idx = child_idx;
24753 }
24754 35 A[parent_idx] = v;
24755 35 }
24756  
24757  
24758 static void
24759 263 heapify_array(u32 A[], unsigned length)
24760 {
24761 unsigned subtree_idx;
24762  
24763 275 ▶ for (subtree_idx = length / 2; subtree_idx >= 1; subtree_idx--)
Branch: 1 branch
BranchTrueFalseTotal
for (subtree_idx = length / 2; subtree_idx >= 1; subtree_idx--) 12 263 275
24764 12 heapify_subtree(A, length, subtree_idx);
24765 263 }
24766  
24767  
24768 static void
24769 263 heap_sort(u32 A[], unsigned length)
24770 {
24771 263 A--;
24772  
24773 263 heapify_array(A, length);
24774  
24775 286 ▶ while (length >= 2) {
Branch: 1 branch
BranchTrueFalseTotal
while (length >= 2) { 23 263 286
24776 23 u32 tmp = A[length];
24777  
24778 23 A[length] = A[1];
24779 23 A[1] = tmp;
24780 23 length--;
24781 23 heapify_subtree(A, length, 1);
24782 }
24783 263 }
24784  
24785 #define NUM_SYMBOL_BITS 10
24786 #define NUM_FREQ_BITS (32 - NUM_SYMBOL_BITS)
24787 #define SYMBOL_MASK ((1 << NUM_SYMBOL_BITS) - 1)
24788 #define FREQ_MASK (~SYMBOL_MASK)
24789  
24790 #define GET_NUM_COUNTERS(num_syms) (num_syms)
24791  
24792  
24793 static unsigned
24794 263 sort_symbols(unsigned num_syms, const u32 freqs[], u8 lens[], u32 symout[])
24795 {
24796 unsigned sym;
24797 unsigned i;
24798 unsigned num_used_syms;
24799 unsigned num_counters;
24800 unsigned counters[GET_NUM_COUNTERS(DEFLATE_MAX_NUM_SYMS)];
24801  
24802
24803  
24804 263 num_counters = GET_NUM_COUNTERS(num_syms);
24805  
24806 263 memset(counters, 0, num_counters * sizeof(counters[0]));
24807  
24808
24809 33178 ▶ for (sym = 0; sym < num_syms; sym++)
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < num_syms; sym++) 32915 263 33178
24810 32915 counters[MIN(freqs[sym], num_counters - 1)]++;
24811  
24812
24813 263 num_used_syms = 0;
24814 32915 ▶ for (i = 1; i < num_counters; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 1; i < num_counters; i++) { 32652 263 32915
24815 32652 unsigned count = counters[i];
24816  
24817 32652 counters[i] = num_used_syms;
24818 32652 num_used_syms += count;
24819 }
24820  
24821
24822 33178 ▶ for (sym = 0; sym < num_syms; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < num_syms; sym++) { 32915 263 33178
24823 32915 u32 freq = freqs[sym];
24824  
24825 32915 ▶ if (freq != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (freq != 0) { 13499 19416 32915
24826 13499 symout[counters[MIN(freq, num_counters - 1)]++] =
24827 13499 sym | (freq << NUM_SYMBOL_BITS);
24828 } else {
24829 19416 lens[sym] = 0;
24830 }
24831 }
24832  
24833
24834 263 heap_sort(symout + counters[num_counters - 2],
24835 263 counters[num_counters - 1] - counters[num_counters - 2]);
24836  
24837 263 return num_used_syms;
24838 }
24839  
24840  
24841 static void
24842 203 build_tree(u32 A[], unsigned sym_count)
24843 {
24844 203 const unsigned last_idx = sym_count - 1;
24845  
24846
24847 203 unsigned i = 0;
24848  
24849
24850 203 unsigned b = 0;
24851  
24852
24853 203 unsigned e = 0;
24854  
24855 do {
24856 u32 new_freq;
24857  
24858
24859 13245 ▶ if (i + 1 <= last_idx &&
Branch: 2 branches
BranchTrueFalseTotal
if (i + 1 <= last_idx && 7970 5275 13245
if (i + 1 <= last_idx && 7767 203 7970
24860 7767 ▶ (b == e || (A[i + 1] & FREQ_MASK) <= (A[b] & FREQ_MASK))) {
Branch: 1 branch
BranchTrueFalseTotal
(b == e || (A[i + 1] & FREQ_MASK) <= (A[b] & FREQ_MASK))) { 6320 1447 7767
24861
24862 6523 new_freq = (A[i] & FREQ_MASK) + (A[i + 1] & FREQ_MASK);
24863 6523 i += 2;
24864 6722 ▶ } else if (b + 2 <= e &&
Branch: 2 branches
BranchTrueFalseTotal
} else if (b + 2 <= e && 6610 112 6722
} else if (b + 2 <= e && 1526 5084 6610
24865 1526 (i > last_idx ||
24866 1526 ▶ (A[b + 1] & FREQ_MASK) < (A[i] & FREQ_MASK))) {
Branch: 1 branch
BranchTrueFalseTotal
(A[b + 1] & FREQ_MASK) < (A[i] & FREQ_MASK))) { 1236 290 1526
24867
24868 6320 new_freq = (A[b] & FREQ_MASK) + (A[b + 1] & FREQ_MASK);
24869 6320 A[b] = (e << NUM_SYMBOL_BITS) | (A[b] & SYMBOL_MASK);
24870 6320 A[b + 1] = (e << NUM_SYMBOL_BITS) |
24871 6320 (A[b + 1] & SYMBOL_MASK);
24872 6320 b += 2;
24873 } else {
24874
24875 402 new_freq = (A[i] & FREQ_MASK) + (A[b] & FREQ_MASK);
24876 402 A[b] = (e << NUM_SYMBOL_BITS) | (A[b] & SYMBOL_MASK);
24877 402 i++;
24878 402 b++;
24879 }
24880 13245 A[e] = new_freq | (A[e] & SYMBOL_MASK);
24881
24882 13245 ▶ } while (++e < last_idx);
Branch: 1 branch
BranchTrueFalseTotal
} while (++e < last_idx); 13042 203 13245
24883 203 }
24884  
24885  
24886 static void
24887 203 compute_length_counts(u32 A[], unsigned root_idx, unsigned len_counts[],
24888 unsigned max_codeword_len)
24889 {
24890 unsigned len;
24891 int node;
24892  
24893
24894  
24895 2832 ▶ for (len = 0; len <= max_codeword_len; len++)
Branch: 1 branch
BranchTrueFalseTotal
for (len = 0; len <= max_codeword_len; len++) 2629 203 2832
24896 2629 len_counts[len] = 0;
24897 203 len_counts[1] = 2;
24898  
24899
24900 203 A[root_idx] &= SYMBOL_MASK;
24901  
24902 13245 ▶ for (node = root_idx - 1; node >= 0; node--) {
Branch: 1 branch
BranchTrueFalseTotal
for (node = root_idx - 1; node >= 0; node--) { 13042 203 13245
24903  
24904
24905  
24906 13042 unsigned parent = A[node] >> NUM_SYMBOL_BITS;
24907 13042 unsigned parent_depth = A[parent] >> NUM_SYMBOL_BITS;
24908 13042 unsigned depth = parent_depth + 1;
24909  
24910
24911 13042 A[node] = (A[node] & SYMBOL_MASK) | (depth << NUM_SYMBOL_BITS);
24912  
24913
24914 13042 ▶ if (depth >= max_codeword_len) {
Branch: 1 branch
BranchTrueFalseTotal
if (depth >= max_codeword_len) { 0 13042 13042
24915 0 depth = max_codeword_len;
24916 do {
24917 0 depth--;
24918 0 ▶ } while (len_counts[depth] == 0);
Branch: 1 branch
BranchTrueFalseTotal
} while (len_counts[depth] == 0); 0 0 0
24919 }
24920  
24921
24922 13042 len_counts[depth]--;
24923 13042 len_counts[depth + 1] += 2;
24924 }
24925 203 }
24926  
24927  
24928  
24929 #ifdef rbit32
24930 static forceinline u32 reverse_codeword(u32 codeword, u8 len)
24931 {
24932 return rbit32(codeword) >> ((32 - len) & 31);
24933 }
24934 #else
24935  
24936 static const u8 bitreverse_tab[256] = {
24937 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
24938 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
24939 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
24940 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
24941 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
24942 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
24943 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
24944 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
24945 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
24946 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
24947 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
24948 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
24949 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
24950 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
24951 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
24952 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
24953 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
24954 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
24955 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
24956 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
24957 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
24958 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
24959 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
24960 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
24961 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
24962 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
24963 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
24964 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
24965 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
24966 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
24967 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
24968 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
24969 };
24970  
24971 static forceinline u32 reverse_codeword(u32 codeword, u8 len)
24972 {
24973 STATIC_ASSERT(DEFLATE_MAX_CODEWORD_LEN <= 16);
24974 30995 codeword = ((u32)bitreverse_tab[codeword & 0xff] << 8) |
24975 30995 bitreverse_tab[codeword >> 8];
24976 30995 return codeword >> (16 - len);
24977 }
24978 #endif
24979  
24980  
24981 static void
24982 203 gen_codewords(u32 A[], u8 lens[], const unsigned len_counts[],
24983 unsigned max_codeword_len, unsigned num_syms)
24984 {
24985 u32 next_codewords[DEFLATE_MAX_CODEWORD_LEN + 1];
24986 unsigned i;
24987 unsigned len;
24988 unsigned sym;
24989  
24990
24991 2629 ▶ for (i = 0, len = max_codeword_len; len >= 1; len--) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0, len = max_codeword_len; len >= 1; len--) { 2426 203 2629
24992 2426 unsigned count = len_counts[len];
24993  
24994 15874 ▶ while (count--)
Branch: 1 branch
BranchTrueFalseTotal
while (count--) 13448 2426 15874
24995 13448 lens[A[i++] & SYMBOL_MASK] = len;
24996 }
24997  
24998
24999 203 next_codewords[0] = 0;
25000 203 next_codewords[1] = 0;
25001 2426 ▶ for (len = 2; len <= max_codeword_len; len++)
Branch: 1 branch
BranchTrueFalseTotal
for (len = 2; len <= max_codeword_len; len++) 2223 203 2426
25002 2223 next_codewords[len] =
25003 2223 (next_codewords[len - 1] + len_counts[len - 1]) << 1;
25004  
25005 31198 ▶ for (sym = 0; sym < num_syms; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < num_syms; sym++) { 30995 203 31198
25006
25007 30995 A[sym] = reverse_codeword(next_codewords[lens[sym]]++,
25008 30995 lens[sym]);
25009 }
25010 203 }
25011  
25012  
25013 static void
25014 263 deflate_make_huffman_code(unsigned num_syms, unsigned max_codeword_len,
25015 const u32 freqs[], u8 lens[], u32 codewords[])
25016 {
25017 263 u32 *A = codewords;
25018 unsigned num_used_syms;
25019  
25020 STATIC_ASSERT(DEFLATE_MAX_NUM_SYMS <= 1 << NUM_SYMBOL_BITS);
25021 STATIC_ASSERT(MAX_BLOCK_LENGTH <= ((u32)1 << NUM_FREQ_BITS) - 1);
25022  
25023
25024 263 num_used_syms = sort_symbols(num_syms, freqs, lens, A);
25025
25026  
25027
25028 263 ▶ if (unlikely(num_used_syms < 2)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(num_used_syms < 2)) { 60 203 263
25029 60 ▶ unsigned sym = num_used_syms ? (A[0] & SYMBOL_MASK) : 0;
Branch: 1 branch
BranchTrueFalseTotal
unsigned sym = num_used_syms ? (A[0] & SYMBOL_MASK) : 0; 51 9 60
25030 60 ▶ unsigned nonzero_idx = sym ? sym : 1;
Branch: 1 branch
BranchTrueFalseTotal
unsigned nonzero_idx = sym ? sym : 1; 51 9 60
25031  
25032 60 codewords[0] = 0;
25033 60 lens[0] = 1;
25034 60 codewords[nonzero_idx] = 1;
25035 60 lens[nonzero_idx] = 1;
25036 60 return;
25037 }
25038  
25039
25040  
25041 203 build_tree(A, num_used_syms);
25042  
25043 {
25044 unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
25045  
25046 203 compute_length_counts(A, num_used_syms - 2,
25047 len_counts, max_codeword_len);
25048  
25049 203 gen_codewords(A, lens, len_counts, max_codeword_len, num_syms);
25050 }
25051 }
25052  
25053  
25054 static void
25055 62 deflate_reset_symbol_frequencies(struct libdeflate_compressor *c)
25056 {
25057 62 memset(&c->freqs, 0, sizeof(c->freqs));
25058 62 }
25059  
25060  
25061 static void
25062 99 deflate_make_huffman_codes(const struct deflate_freqs *freqs,
25063 struct deflate_codes *codes)
25064 {
25065 99 deflate_make_huffman_code(DEFLATE_NUM_LITLEN_SYMS,
25066 MAX_LITLEN_CODEWORD_LEN,
25067 99 freqs->litlen,
25068 99 codes->lens.litlen,
25069 99 codes->codewords.litlen);
25070  
25071 99 deflate_make_huffman_code(DEFLATE_NUM_OFFSET_SYMS,
25072 MAX_OFFSET_CODEWORD_LEN,
25073 99 freqs->offset,
25074 99 codes->lens.offset,
25075 99 codes->codewords.offset);
25076 99 }
25077  
25078  
25079 static void
25080 37 deflate_init_static_codes(struct libdeflate_compressor *c)
25081 {
25082 unsigned i;
25083  
25084 5365 ▶ for (i = 0; i < 144; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 144; i++) 5328 37 5365
25085 5328 c->freqs.litlen[i] = 1 << (9 - 8);
25086 4181 ▶ for (; i < 256; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 256; i++) 4144 37 4181
25087 4144 c->freqs.litlen[i] = 1 << (9 - 9);
25088 925 ▶ for (; i < 280; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 280; i++) 888 37 925
25089 888 c->freqs.litlen[i] = 1 << (9 - 7);
25090 333 ▶ for (; i < 288; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 288; i++) 296 37 333
25091 296 c->freqs.litlen[i] = 1 << (9 - 8);
25092  
25093 1221 ▶ for (i = 0; i < 32; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 32; i++) 1184 37 1221
25094 1184 c->freqs.offset[i] = 1 << (5 - 5);
25095  
25096 37 deflate_make_huffman_codes(&c->freqs, &c->static_codes);
25097 37 }
25098  
25099  
25100 static forceinline unsigned
25101 deflate_get_offset_slot(u32 offset)
25102 {
25103
25104 2091 unsigned n = (256 - offset) >> 29;
25105  
25106 2091 ▶ ASSERT(offset >= 1 && offset <= 32768);
Branch: 8 branches
BranchTrueFalseTotal
ASSERT(offset >= 1 && offset <= 32768); 42 0 42
ASSERT(offset >= 1 && offset <= 32768); 0 0 0
ASSERT(offset >= 1 && offset <= 32768); 0 0 0
ASSERT(offset >= 1 && offset <= 32768); 64 0 64
ASSERT(offset >= 1 && offset <= 32768); 0 0 0
ASSERT(offset >= 1 && offset <= 32768); 1901 0 1901
ASSERT(offset >= 1 && offset <= 32768); 63 0 63
ASSERT(offset >= 1 && offset <= 32768); 21 0 21
25107  
25108 2091 return deflate_offset_slot[(offset - 1) >> n] + (n << 1);
25109 }
25110  
25111 static unsigned
25112 65 deflate_compute_precode_items(const u8 lens[], const unsigned num_lens,
25113 u32 precode_freqs[], unsigned precode_items[])
25114 {
25115 unsigned *itemptr;
25116 unsigned run_start;
25117 unsigned run_end;
25118 unsigned extra_bits;
25119 u8 len;
25120  
25121 65 memset(precode_freqs, 0,
25122 DEFLATE_NUM_PRECODE_SYMS * sizeof(precode_freqs[0]));
25123  
25124 65 itemptr = precode_items;
25125 65 run_start = 0;
25126 do {
25127
25128  
25129
25130 1973 len = lens[run_start];
25131  
25132
25133 1973 run_end = run_start;
25134 do {
25135 18873 run_end++;
25136 18873 ▶ } while (run_end != num_lens && len == lens[run_end]);
Branch: 2 branches
BranchTrueFalseTotal
} while (run_end != num_lens && len == lens[run_end]); 18808 65 18873
} while (run_end != num_lens && len == lens[run_end]); 16900 1908 18808
25137  
25138 1973 ▶ if (len == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (len == 0) { 834 1139 1973
25139
25140  
25141
25142 1083 ▶ while ((run_end - run_start) >= 11) {
Branch: 1 branch
BranchTrueFalseTotal
while ((run_end - run_start) >= 11) { 249 834 1083
25143 249 extra_bits = MIN((run_end - run_start) - 11,
25144 0x7F);
25145 249 precode_freqs[18]++;
25146 249 *itemptr++ = 18 | (extra_bits << 5);
25147 249 run_start += 11 + extra_bits;
25148 }
25149  
25150
25151 834 ▶ if ((run_end - run_start) >= 3) {
Branch: 1 branch
BranchTrueFalseTotal
if ((run_end - run_start) >= 3) { 309 525 834
25152 309 extra_bits = MIN((run_end - run_start) - 3,
25153 0x7);
25154 309 precode_freqs[17]++;
25155 309 *itemptr++ = 17 | (extra_bits << 5);
25156 309 run_start += 3 + extra_bits;
25157 }
25158 } else {
25159  
25160
25161  
25162
25163 1139 ▶ if ((run_end - run_start) >= 4) {
Branch: 1 branch
BranchTrueFalseTotal
if ((run_end - run_start) >= 4) { 6 1133 1139
25164 6 precode_freqs[len]++;
25165 6 *itemptr++ = len;
25166 6 run_start++;
25167 do {
25168 6 extra_bits = MIN((run_end - run_start) -
25169 3, 0x3);
25170 6 precode_freqs[16]++;
25171 6 *itemptr++ = 16 | (extra_bits << 5);
25172 6 run_start += 3 + extra_bits;
25173 6 ▶ } while ((run_end - run_start) >= 3);
Branch: 1 branch
BranchTrueFalseTotal
} while ((run_end - run_start) >= 3); 0 6 6
25174 }
25175 }
25176  
25177
25178 3544 ▶ while (run_start != run_end) {
Branch: 1 branch
BranchTrueFalseTotal
while (run_start != run_end) { 1571 1973 3544
25179 1571 precode_freqs[len]++;
25180 1571 *itemptr++ = len;
25181 1571 run_start++;
25182 }
25183 1973 ▶ } while (run_start != num_lens);
Branch: 1 branch
BranchTrueFalseTotal
} while (run_start != num_lens); 1908 65 1973
25184  
25185 65 return itemptr - precode_items;
25186 }
25187  
25188  
25189  
25190  
25191 static void
25192 65 deflate_precompute_huffman_header(struct libdeflate_compressor *c)
25193 {
25194
25195  
25196 65 for (c->o.precode.num_litlen_syms = DEFLATE_NUM_LITLEN_SYMS;
25197 475 ▶ c->o.precode.num_litlen_syms > 257;
Branch: 1 branch
BranchTrueFalseTotal
c->o.precode.num_litlen_syms > 257; 466 9 475
25198 410 c->o.precode.num_litlen_syms--)
25199 466 ▶ if (c->codes.lens.litlen[c->o.precode.num_litlen_syms - 1] != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (c->codes.lens.litlen[c->o.precode.num_litlen_syms - 1] != 0) 56 410 466
25200 56 break;
25201  
25202 65 for (c->o.precode.num_offset_syms = DEFLATE_NUM_OFFSET_SYMS;
25203 1582 ▶ c->o.precode.num_offset_syms > 1;
Branch: 1 branch
BranchTrueFalseTotal
c->o.precode.num_offset_syms > 1; 1582 0 1582
25204 1517 c->o.precode.num_offset_syms--)
25205 1582 ▶ if (c->codes.lens.offset[c->o.precode.num_offset_syms - 1] != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (c->codes.lens.offset[c->o.precode.num_offset_syms - 1] != 0) 65 1517 1582
25206 65 break;
25207  
25208
25209 STATIC_ASSERT(offsetof(struct deflate_lens, offset) ==
25210 DEFLATE_NUM_LITLEN_SYMS);
25211 65 ▶ if (c->o.precode.num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) {
Branch: 1 branch
BranchTrueFalseTotal
if (c->o.precode.num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) { 65 0 65
25212 65 memmove((u8 *)&c->codes.lens + c->o.precode.num_litlen_syms,
25213 65 (u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS,
25214 65 c->o.precode.num_offset_syms);
25215 }
25216  
25217
25218 65 c->o.precode.num_items =
25219 65 deflate_compute_precode_items((u8 *)&c->codes.lens,
25220 65 c->o.precode.num_litlen_syms +
25221 65 c->o.precode.num_offset_syms,
25222 65 c->o.precode.freqs,
25223 65 c->o.precode.items);
25224  
25225
25226 65 deflate_make_huffman_code(DEFLATE_NUM_PRECODE_SYMS,
25227 MAX_PRE_CODEWORD_LEN,
25228 65 c->o.precode.freqs, c->o.precode.lens,
25229 65 c->o.precode.codewords);
25230  
25231
25232 65 for (c->o.precode.num_explicit_lens = DEFLATE_NUM_PRECODE_SYMS;
25233 138 ▶ c->o.precode.num_explicit_lens > 4;
Branch: 1 branch
BranchTrueFalseTotal
c->o.precode.num_explicit_lens > 4; 138 0 138
25234 73 c->o.precode.num_explicit_lens--)
25235 138 if (c->o.precode.lens[deflate_precode_lens_permutation[
25236 138 ▶ c->o.precode.num_explicit_lens - 1]] != 0)
Branch: 1 branch
BranchTrueFalseTotal
c->o.precode.num_explicit_lens - 1]] != 0) 65 73 138
25237 65 break;
25238  
25239
25240 65 ▶ if (c->o.precode.num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) {
Branch: 1 branch
BranchTrueFalseTotal
if (c->o.precode.num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) { 65 0 65
25241 65 memmove((u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS,
25242 65 (u8 *)&c->codes.lens + c->o.precode.num_litlen_syms,
25243 65 c->o.precode.num_offset_syms);
25244 }
25245 65 }
25246  
25247  
25248 static void
25249 38 deflate_compute_full_len_codewords(struct libdeflate_compressor *c,
25250 const struct deflate_codes *codes)
25251 {
25252 u32 len;
25253  
25254 STATIC_ASSERT(MAX_LITLEN_CODEWORD_LEN +
25255 DEFLATE_MAX_EXTRA_LENGTH_BITS <= 32);
25256  
25257 9766 ▶ for (len = DEFLATE_MIN_MATCH_LEN; len <= DEFLATE_MAX_MATCH_LEN; len++) {
Branch: 1 branch
BranchTrueFalseTotal
for (len = DEFLATE_MIN_MATCH_LEN; len <= DEFLATE_MAX_MATCH_LEN; len++) { 9728 38 9766
25258 9728 unsigned slot = deflate_length_slot[len];
25259 9728 unsigned litlen_sym = DEFLATE_FIRST_LEN_SYM + slot;
25260 9728 u32 extra_bits = len - deflate_length_slot_base[slot];
25261  
25262 9728 c->o.length.codewords[len] =
25263 9728 codes->codewords.litlen[litlen_sym] |
25264 9728 (extra_bits << codes->lens.litlen[litlen_sym]);
25265 9728 c->o.length.lens[len] = codes->lens.litlen[litlen_sym] +
25266 9728 deflate_extra_length_bits[slot];
25267 }
25268 38 }
25269  
25270  
25271 #define WRITE_MATCH(c_, codes_, length_, offset_, offset_slot_) \
25272 do { \
25273 const struct libdeflate_compressor *c__ = (c_); \
25274 const struct deflate_codes *codes__ = (codes_); \
25275 u32 length__ = (length_); \
25276 u32 offset__ = (offset_); \
25277 unsigned offset_slot__ = (offset_slot_); \
25278 \
25279 \
25280 STATIC_ASSERT(CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN + \
25281 DEFLATE_MAX_EXTRA_LENGTH_BITS)); \
25282 ADD_BITS(c__->o.length.codewords[length__], \
25283 c__->o.length.lens[length__]); \
25284 \
25285 if (!CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN + \
25286 DEFLATE_MAX_EXTRA_LENGTH_BITS + \
25287 MAX_OFFSET_CODEWORD_LEN + \
25288 DEFLATE_MAX_EXTRA_OFFSET_BITS)) \
25289 FLUSH_BITS(); \
25290 \
25291 \
25292 ADD_BITS(codes__->codewords.offset[offset_slot__], \
25293 codes__->lens.offset[offset_slot__]); \
25294 \
25295 if (!CAN_BUFFER(MAX_OFFSET_CODEWORD_LEN + \
25296 DEFLATE_MAX_EXTRA_OFFSET_BITS)) \
25297 FLUSH_BITS(); \
25298 \
25299 \
25300 ADD_BITS(offset__ - deflate_offset_slot_base[offset_slot__], \
25301 deflate_extra_offset_bits[offset_slot__]); \
25302 \
25303 FLUSH_BITS(); \
25304 } while (0)
25305  
25306  
25307 static void
25308 38 deflate_flush_block(struct libdeflate_compressor *c,
25309 struct deflate_output_bitstream *os,
25310 const u8 *block_begin, u32 block_length,
25311 const struct deflate_sequence *sequences,
25312 bool is_final_block)
25313 {
25314
25315 38 const u8 *in_next = block_begin;
25316 38 const u8 * const in_end = block_begin + block_length;
25317 38 bitbuf_t bitbuf = os->bitbuf;
25318 38 unsigned bitcount = os->bitcount;
25319 38 u8 *out_next = os->next;
25320 38 u8 * const out_fast_end =
25321 38 ▶ os->end - MIN(WORDBYTES - 1, os->end - out_next);
Branch: 1 branch
BranchTrueFalseTotal
os->end - MIN(WORDBYTES - 1, os->end - out_next); 0 38 38
25322
25323 38 u32 dynamic_cost = 3;
25324 38 u32 static_cost = 3;
25325 38 u32 uncompressed_cost = 3;
25326 u32 best_cost;
25327 struct deflate_codes *codes;
25328 unsigned sym;
25329  
25330 38 ▶ ASSERT(block_length >= MIN_BLOCK_LENGTH ||
Branch: 3 branches
BranchTrueFalseTotal
ASSERT(block_length >= MIN_BLOCK_LENGTH || 36 2 38
ASSERT(block_length >= MIN_BLOCK_LENGTH || 36 0 36
ASSERT(block_length >= MIN_BLOCK_LENGTH || 36 0 36
25331 (is_final_block && block_length > 0));
25332 ASSERT(block_length <= MAX_BLOCK_LENGTH);
25333 ASSERT(bitcount <= 7);
25334 38 ASSERT((bitbuf & ~(((bitbuf_t)1 << bitcount) - 1)) == 0);
25335 38 ASSERT(out_next <= os->end);
25336 38 ASSERT(!os->overflow);
25337  
25338
25339 38 deflate_precompute_huffman_header(c);
25340  
25341
25342 38 dynamic_cost += 5 + 5 + 4 + (3 * c->o.precode.num_explicit_lens);
25343 760 ▶ for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) { 722 38 760
25344 722 u32 extra = deflate_extra_precode_bits[sym];
25345  
25346 722 dynamic_cost += c->o.precode.freqs[sym] *
25347 722 (extra + c->o.precode.lens[sym]);
25348 }
25349  
25350
25351 5510 ▶ for (sym = 0; sym < 144; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < 144; sym++) { 5472 38 5510
25352 5472 dynamic_cost += c->freqs.litlen[sym] *
25353 5472 c->codes.lens.litlen[sym];
25354 5472 static_cost += c->freqs.litlen[sym] * 8;
25355 }
25356 4294 ▶ for (; sym < 256; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (; sym < 256; sym++) { 4256 38 4294
25357 4256 dynamic_cost += c->freqs.litlen[sym] *
25358 4256 c->codes.lens.litlen[sym];
25359 4256 static_cost += c->freqs.litlen[sym] * 9;
25360 }
25361  
25362
25363 38 dynamic_cost += c->codes.lens.litlen[DEFLATE_END_OF_BLOCK];
25364 38 static_cost += 7;
25365  
25366
25367 38 for (sym = DEFLATE_FIRST_LEN_SYM;
25368 1140 ▶ sym < DEFLATE_FIRST_LEN_SYM + ARRAY_LEN(deflate_extra_length_bits);
Branch: 1 branch
BranchTrueFalseTotal
sym < DEFLATE_FIRST_LEN_SYM + ARRAY_LEN(deflate_extra_length_bits); 1102 38 1140
25369 1102 sym++) {
25370 1102 u32 extra = deflate_extra_length_bits[
25371 1102 sym - DEFLATE_FIRST_LEN_SYM];
25372  
25373 1102 dynamic_cost += c->freqs.litlen[sym] *
25374 1102 (extra + c->codes.lens.litlen[sym]);
25375 1102 static_cost += c->freqs.litlen[sym] *
25376 1102 (extra + c->static_codes.lens.litlen[sym]);
25377 }
25378  
25379
25380 1178 ▶ for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++) { 1140 38 1178
25381 1140 u32 extra = deflate_extra_offset_bits[sym];
25382  
25383 1140 dynamic_cost += c->freqs.offset[sym] *
25384 1140 (extra + c->codes.lens.offset[sym]);
25385 1140 static_cost += c->freqs.offset[sym] * (extra + 5);
25386 }
25387  
25388
25389 38 uncompressed_cost += (-(bitcount + 3) & 7) + 32 +
25390 38 (40 * (DIV_ROUND_UP(block_length,
25391 38 UINT16_MAX) - 1)) +
25392 38 (8 * block_length);
25393  
25394
25395  
25396 38 best_cost = MIN(dynamic_cost, MIN(static_cost, uncompressed_cost));
25397  
25398
25399 38 ▶ if (DIV_ROUND_UP(bitcount + best_cost, 8) > os->end - out_next) {
Branch: 1 branch
BranchTrueFalseTotal
if (DIV_ROUND_UP(bitcount + best_cost, 8) > os->end - out_next) { 0 38 38
25400 0 os->overflow = true;
25401 0 return;
25402 }
25403
25404  
25405 38 ▶ if (best_cost == uncompressed_cost) {
Branch: 1 branch
BranchTrueFalseTotal
if (best_cost == uncompressed_cost) { 0 38 38
25406
25407 do {
25408 0 u8 bfinal = 0;
25409 0 size_t len = UINT16_MAX;
25410  
25411 0 ▶ if (in_end - in_next <= UINT16_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next <= UINT16_MAX) { 0 0 0
25412 0 bfinal = is_final_block;
25413 0 len = in_end - in_next;
25414 }
25415
25416 0 ASSERT(os->end - out_next >=
25417 DIV_ROUND_UP(bitcount + 3, 8) + 4 + len);
25418
25419 STATIC_ASSERT(DEFLATE_BLOCKTYPE_UNCOMPRESSED == 0);
25420 0 *out_next++ = (bfinal << bitcount) | bitbuf;
25421 0 ▶ if (bitcount > 5)
Branch: 1 branch
BranchTrueFalseTotal
if (bitcount > 5) 0 0 0
25422 0 *out_next++ = 0;
25423 0 bitbuf = 0;
25424 0 bitcount = 0;
25425
25426 0 put_unaligned_le16(len, out_next);
25427 0 out_next += 2;
25428 0 put_unaligned_le16(~len, out_next);
25429 0 out_next += 2;
25430 0 memcpy(out_next, in_next, len);
25431 0 out_next += len;
25432 0 in_next += len;
25433 0 ▶ } while (in_next != in_end);
Branch: 1 branch
BranchTrueFalseTotal
} while (in_next != in_end); 0 0 0
25434
25435 0 goto out;
25436 }
25437  
25438 38 ▶ if (best_cost == static_cost) {
Branch: 1 branch
BranchTrueFalseTotal
if (best_cost == static_cost) { 36 2 38
25439
25440 36 codes = &c->static_codes;
25441 36 ADD_BITS(is_final_block, 1);
25442 36 ADD_BITS(DEFLATE_BLOCKTYPE_STATIC_HUFFMAN, 2);
25443 72 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 36 0 36
FLUSH_BITS(); 0 0 0
25444 } else {
25445 2 const unsigned num_explicit_lens = c->o.precode.num_explicit_lens;
25446 2 const unsigned num_precode_items = c->o.precode.num_items;
25447 unsigned precode_sym, precode_item;
25448 unsigned i;
25449  
25450
25451  
25452 2 codes = &c->codes;
25453 STATIC_ASSERT(CAN_BUFFER(1 + 2 + 5 + 5 + 4 + 3));
25454 2 ADD_BITS(is_final_block, 1);
25455 2 ADD_BITS(DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN, 2);
25456 2 ADD_BITS(c->o.precode.num_litlen_syms - 257, 5);
25457 2 ADD_BITS(c->o.precode.num_offset_syms - 1, 5);
25458 2 ADD_BITS(num_explicit_lens - 4, 4);
25459  
25460
25461 if (CAN_BUFFER(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
25462
25463 2 precode_sym = deflate_precode_lens_permutation[0];
25464 2 ADD_BITS(c->o.precode.lens[precode_sym], 3);
25465 4 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 2 0 2
FLUSH_BITS(); 0 0 0
25466 2 i = 1;
25467 do {
25468 26 precode_sym =
25469 26 deflate_precode_lens_permutation[i];
25470 26 ADD_BITS(c->o.precode.lens[precode_sym], 3);
25471 26 ▶ } while (++i < num_explicit_lens);
Branch: 1 branch
BranchTrueFalseTotal
} while (++i < num_explicit_lens); 24 2 26
25472 4 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 2 0 2
FLUSH_BITS(); 0 0 0
25473 } else {
25474 FLUSH_BITS();
25475 i = 0;
25476 do {
25477 precode_sym =
25478 deflate_precode_lens_permutation[i];
25479 ADD_BITS(c->o.precode.lens[precode_sym], 3);
25480 FLUSH_BITS();
25481 } while (++i < num_explicit_lens);
25482 }
25483  
25484
25485 2 i = 0;
25486 do {
25487 233 precode_item = c->o.precode.items[i];
25488 233 precode_sym = precode_item & 0x1F;
25489 STATIC_ASSERT(CAN_BUFFER(MAX_PRE_CODEWORD_LEN + 7));
25490 233 ADD_BITS(c->o.precode.codewords[precode_sym],
25491 c->o.precode.lens[precode_sym]);
25492 233 ADD_BITS(precode_item >> 5,
25493 deflate_extra_precode_bits[precode_sym]);
25494 466 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 233 0 233
FLUSH_BITS(); 0 0 0
25495 233 ▶ } while (++i < num_precode_items);
Branch: 1 branch
BranchTrueFalseTotal
} while (++i < num_precode_items); 231 2 233
25496 }
25497  
25498
25499 ASSERT(bitcount <= 7);
25500 38 deflate_compute_full_len_codewords(c, codes);
25501 #if SUPPORT_NEAR_OPTIMAL_PARSING
25502 38 ▶ if (sequences == NULL) {
Branch: 1 branch
BranchTrueFalseTotal
if (sequences == NULL) { 9 29 38
25503
25504 9 struct deflate_optimum_node *cur_node =
25505 &c->p.n.optimum_nodes[0];
25506 9 struct deflate_optimum_node * const end_node =
25507 &c->p.n.optimum_nodes[block_length];
25508 do {
25509 225 u32 length = cur_node->item & OPTIMUM_LEN_MASK;
25510 225 u32 offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
25511  
25512 225 ▶ if (length == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (length == 1) { 162 63 225
25513
25514 162 ADD_BITS(codes->codewords.litlen[offset],
25515 codes->lens.litlen[offset]);
25516 324 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 162 0 162
FLUSH_BITS(); 0 0 0
25517 } else {
25518
25519 126 ▶ WRITE_MATCH(c, codes, length, offset,
Branch: 2 branches
BranchTrueFalseTotal
WRITE_MATCH(c, codes, length, offset, 63 0 63
WRITE_MATCH(c, codes, length, offset, 0 0 0
25520 c->p.n.offset_slot_full[offset]);
25521 }
25522 225 cur_node += length;
25523 225 ▶ } while (cur_node != end_node);
Branch: 1 branch
BranchTrueFalseTotal
} while (cur_node != end_node); 216 9 225
25524 } else
25525 #endif
25526 {
25527
25528 const struct deflate_sequence *seq;
25529  
25530 29 for (seq = sequences; ; seq++) {
25531 2120 u32 litrunlen = seq->litrunlen_and_length &
25532 SEQ_LITRUNLEN_MASK;
25533 2120 u32 length = seq->litrunlen_and_length >>
25534 SEQ_LENGTH_SHIFT;
25535 unsigned lit;
25536  
25537
25538 if (CAN_BUFFER(4 * MAX_LITLEN_CODEWORD_LEN)) {
25539 2677 ▶ for (; litrunlen >= 4; litrunlen -= 4) {
Branch: 1 branch
BranchTrueFalseTotal
for (; litrunlen >= 4; litrunlen -= 4) { 557 2120 2677
25540 557 lit = *in_next++;
25541 557 ADD_BITS(codes->codewords.litlen[lit],
25542 codes->lens.litlen[lit]);
25543 557 lit = *in_next++;
25544 557 ADD_BITS(codes->codewords.litlen[lit],
25545 codes->lens.litlen[lit]);
25546 557 lit = *in_next++;
25547 557 ADD_BITS(codes->codewords.litlen[lit],
25548 codes->lens.litlen[lit]);
25549 557 lit = *in_next++;
25550 557 ADD_BITS(codes->codewords.litlen[lit],
25551 codes->lens.litlen[lit]);
25552 1114 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 557 0 557
FLUSH_BITS(); 0 0 0
25553 }
25554 2120 ▶ if (litrunlen-- != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (litrunlen-- != 0) { 509 1611 2120
25555 509 lit = *in_next++;
25556 509 ADD_BITS(codes->codewords.litlen[lit],
25557 codes->lens.litlen[lit]);
25558 509 ▶ if (litrunlen-- != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (litrunlen-- != 0) { 280 229 509
25559 280 lit = *in_next++;
25560 280 ADD_BITS(codes->codewords.litlen[lit],
25561 codes->lens.litlen[lit]);
25562 280 ▶ if (litrunlen-- != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (litrunlen-- != 0) { 112 168 280
25563 112 lit = *in_next++;
25564 112 ADD_BITS(codes->codewords.litlen[lit],
25565 codes->lens.litlen[lit]);
25566 }
25567 }
25568 1018 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 509 0 509
FLUSH_BITS(); 0 0 0
25569 }
25570 } else {
25571 while (litrunlen--) {
25572 lit = *in_next++;
25573 ADD_BITS(codes->codewords.litlen[lit],
25574 codes->lens.litlen[lit]);
25575 FLUSH_BITS();
25576 }
25577 }
25578  
25579 2120 ▶ if (length == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (length == 0) { 29 2091 2120
25580 ASSERT(in_next == in_end);
25581 29 break;
25582 }
25583  
25584
25585 4182 ▶ WRITE_MATCH(c, codes, length, seq->offset,
Branch: 2 branches
BranchTrueFalseTotal
WRITE_MATCH(c, codes, length, seq->offset, 2091 0 2091
WRITE_MATCH(c, codes, length, seq->offset, 0 0 0
25586 seq->offset_slot);
25587 2091 in_next += length;
25588 }
25589 }
25590  
25591
25592 ASSERT(bitcount <= 7);
25593 38 ADD_BITS(codes->codewords.litlen[DEFLATE_END_OF_BLOCK],
25594 codes->lens.litlen[DEFLATE_END_OF_BLOCK]);
25595 76 ▶ FLUSH_BITS();
Branch: 2 branches
BranchTrueFalseTotal
FLUSH_BITS(); 38 0 38
FLUSH_BITS(); 0 0 0
25596 0 out:
25597 ASSERT(bitcount <= 7);
25598
25599 38 ASSERT(8 * (out_next - os->next) + bitcount - os->bitcount == best_cost);
25600 38 os->bitbuf = bitbuf;
25601 38 os->bitcount = bitcount;
25602 38 os->next = out_next;
25603 }
25604  
25605 static void
25606 29 deflate_finish_block(struct libdeflate_compressor *c,
25607 struct deflate_output_bitstream *os,
25608 const u8 *block_begin, u32 block_length,
25609 const struct deflate_sequence *sequences,
25610 bool is_final_block)
25611 {
25612 29 c->freqs.litlen[DEFLATE_END_OF_BLOCK]++;
25613 29 deflate_make_huffman_codes(&c->freqs, &c->codes);
25614 29 deflate_flush_block(c, os, block_begin, block_length, sequences,
25615 is_final_block);
25616 29 }
25617  
25618  
25619  
25620  
25621  
25622  
25623 static void
25624 44 init_block_split_stats(struct block_split_stats *stats)
25625 {
25626 int i;
25627  
25628 484 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { 440 44 484
25629 440 stats->new_observations[i] = 0;
25630 440 stats->observations[i] = 0;
25631 }
25632 44 stats->num_new_observations = 0;
25633 44 stats->num_observations = 0;
25634 44 }
25635  
25636  
25637 static forceinline void
25638 observe_literal(struct block_split_stats *stats, u8 lit)
25639 {
25640 3237 stats->new_observations[((lit >> 5) & 0x6) | (lit & 1)]++;
25641 3237 stats->num_new_observations++;
25642 3237 }
25643  
25644  
25645 static forceinline void
25646 observe_match(struct block_split_stats *stats, u32 length)
25647 {
25648 2133 stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES +
25649 2133 (length >= 9)]++;
25650 2133 stats->num_new_observations++;
25651 2133 }
25652  
25653 static void
25654 15 merge_new_observations(struct block_split_stats *stats)
25655 {
25656 int i;
25657  
25658 165 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { 150 15 165
25659 150 stats->observations[i] += stats->new_observations[i];
25660 150 stats->new_observations[i] = 0;
25661 }
25662 15 stats->num_observations += stats->num_new_observations;
25663 15 stats->num_new_observations = 0;
25664 15 }
25665  
25666 static bool
25667 7 do_end_block_check(struct block_split_stats *stats, u32 block_length)
25668 {
25669 7 ▶ if (stats->num_observations > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (stats->num_observations > 0) { 6 1 7
25670
25671 6 u32 total_delta = 0;
25672 u32 num_items;
25673 u32 cutoff;
25674 int i;
25675  
25676 66 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { 60 6 66
25677 60 u32 expected = stats->observations[i] *
25678 60 stats->num_new_observations;
25679 60 u32 actual = stats->new_observations[i] *
25680 60 stats->num_observations;
25681 60 ▶ u32 delta = (actual > expected) ? actual - expected :
Branch: 1 branch
BranchTrueFalseTotal
u32 delta = (actual > expected) ? actual - expected : 16 44 60
25682 expected - actual;
25683  
25684 60 total_delta += delta;
25685 }
25686  
25687 6 num_items = stats->num_observations +
25688 6 stats->num_new_observations;
25689
25690 6 cutoff = stats->num_new_observations * 200 / 512 *
25691 6 stats->num_observations;
25692
25693 6 ▶ if (block_length < 10000 && num_items < 8192)
Branch: 2 branches
BranchTrueFalseTotal
if (block_length < 10000 && num_items < 8192) 1 5 6
if (block_length < 10000 && num_items < 8192) 1 0 1
25694 1 cutoff += (u64)cutoff * (8192 - num_items) / 8192;
25695  
25696
25697 6 if (total_delta +
25698 6 ▶ (block_length / 4096) * stats->num_observations >= cutoff)
Branch: 1 branch
BranchTrueFalseTotal
(block_length / 4096) * stats->num_observations >= cutoff) 1 5 6
25699 1 return true;
25700 }
25701 6 merge_new_observations(stats);
25702 6 return false;
25703 }
25704  
25705 static forceinline bool
25706 ready_to_check_block(const struct block_split_stats *stats,
25707 const u8 *in_block_begin, const u8 *in_next,
25708 const u8 *in_end)
25709 {
25710 5245 return stats->num_new_observations >= NUM_OBSERVATIONS_PER_BLOCK_CHECK
25711 450 ▶ && in_next - in_block_begin >= MIN_BLOCK_LENGTH
Branch: 4 branches
BranchTrueFalseTotal
&& in_next - in_block_begin >= MIN_BLOCK_LENGTH 0 0 0
&& in_next - in_block_begin >= MIN_BLOCK_LENGTH 0 0 0
&& in_next - in_block_begin >= MIN_BLOCK_LENGTH 89 361 450
&& in_next - in_block_begin >= MIN_BLOCK_LENGTH 0 0 0
25712 5245 ▶ && in_end - in_next >= MIN_BLOCK_LENGTH;
Branch: 4 branches
BranchTrueFalseTotal
&& in_end - in_next >= MIN_BLOCK_LENGTH; 0 0 0
&& in_end - in_next >= MIN_BLOCK_LENGTH; 0 0 0
&& in_end - in_next >= MIN_BLOCK_LENGTH; 7 82 89
&& in_end - in_next >= MIN_BLOCK_LENGTH; 0 0 0
25713 }
25714  
25715 static forceinline bool
25716 should_end_block(struct block_split_stats *stats,
25717 const u8 *in_block_begin, const u8 *in_next, const u8 *in_end)
25718 {
25719
25720 5029 ▶ if (!ready_to_check_block(stats, in_block_begin, in_next, in_end))
Branch: 3 branches
BranchTrueFalseTotal
if (!ready_to_check_block(stats, in_block_begin, in_next, in_end)) 144 0 144
if (!ready_to_check_block(stats, in_block_begin, in_next, in_end)) 4662 7 4669
if (!ready_to_check_block(stats, in_block_begin, in_next, in_end)) 216 0 216
25721 5022 return false;
25722  
25723 7 return do_end_block_check(stats, in_next - in_block_begin);
25724 }
25725  
25726  
25727  
25728 static void
25729 29 deflate_begin_sequences(struct libdeflate_compressor *c,
25730 struct deflate_sequence *first_seq)
25731 {
25732 29 deflate_reset_symbol_frequencies(c);
25733 29 first_seq->litrunlen_and_length = 0;
25734 29 }
25735  
25736 static forceinline void
25737 deflate_choose_literal(struct libdeflate_compressor *c, unsigned literal,
25738 bool gather_split_stats, struct deflate_sequence *seq)
25739 {
25740 3129 c->freqs.litlen[literal]++;
25741  
25742 3129 if (gather_split_stats)
25743 3075 observe_literal(&c->split_stats, literal);
25744  
25745 STATIC_ASSERT(MAX_BLOCK_LENGTH <= SEQ_LITRUNLEN_MASK);
25746 3129 seq->litrunlen_and_length++;
25747 3129 }
25748  
25749 static forceinline void
25750 deflate_choose_match(struct libdeflate_compressor *c,
25751 u32 length, u32 offset, bool gather_split_stats,
25752 struct deflate_sequence **seq_p)
25753 {
25754 2091 struct deflate_sequence *seq = *seq_p;
25755 2091 unsigned length_slot = deflate_length_slot[length];
25756 2091 unsigned offset_slot = deflate_get_offset_slot(offset);
25757  
25758 2091 c->freqs.litlen[DEFLATE_FIRST_LEN_SYM + length_slot]++;
25759 2091 c->freqs.offset[offset_slot]++;
25760 2091 ▶ if (gather_split_stats)
Branch: 8 branches
BranchTrueFalseTotal
if (gather_split_stats) 42 0 42
if (gather_split_stats) 0 0 0
if (gather_split_stats) 0 0 0
if (gather_split_stats) 64 0 64
if (gather_split_stats) 0 0 0
if (gather_split_stats) 1901 0 1901
if (gather_split_stats) 63 0 63
if (gather_split_stats) 0 21 21
25761 2070 ▶ observe_match(&c->split_stats, length);
Branch: 8 branches
BranchTrueFalseTotal
observe_match(&c->split_stats, length); 42 0 42
observe_match(&c->split_stats, length); 0 0 0
observe_match(&c->split_stats, length); 0 0 0
observe_match(&c->split_stats, length); 64 0 64
observe_match(&c->split_stats, length); 0 0 0
observe_match(&c->split_stats, length); 1055 846 1901
observe_match(&c->split_stats, length); 63 0 63
observe_match(&c->split_stats, length); 0 0 0
25762  
25763 2091 seq->litrunlen_and_length |= length << SEQ_LENGTH_SHIFT;
25764 2091 seq->offset = offset;
25765 2091 seq->offset_slot = offset_slot;
25766  
25767 2091 seq++;
25768 2091 seq->litrunlen_and_length = 0;
25769 2091 *seq_p = seq;
25770 2091 }
25771  
25772  
25773 static forceinline void
25774 adjust_max_and_nice_len(u32 *max_len, u32 *nice_len, size_t remaining)
25775 {
25776 15075 ▶ if (unlikely(remaining < DEFLATE_MAX_MATCH_LEN)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(remaining < DEFLATE_MAX_MATCH_LEN)) { 2304 12771 15075
25777 2389 *max_len = remaining;
25778 2389 *nice_len = MIN(*nice_len, *max_len);
25779 }
25780 22346 }
25781  
25782  
25783 static u32
25784 46 choose_min_match_len(u32 num_used_literals, u32 max_search_depth)
25785 {
25786
25787 static const u8 min_lens[] = {
25788 9, 9, 9, 9, 9, 9, 8, 8, 7, 7, 6, 6, 6, 6, 6, 6,
25789 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
25790 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4,
25791 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
25792 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
25793
25794 };
25795 u32 min_len;
25796  
25797 STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN <= 3);
25798 STATIC_ASSERT(ARRAY_LEN(min_lens) <= DEFLATE_NUM_LITERALS + 1);
25799  
25800 46 ▶ if (num_used_literals >= ARRAY_LEN(min_lens))
Branch: 1 branch
BranchTrueFalseTotal
if (num_used_literals >= ARRAY_LEN(min_lens)) 0 46 46
25801 0 return 3;
25802 46 min_len = min_lens[num_used_literals];
25803
25804 46 ▶ if (max_search_depth < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (max_search_depth < 16) { 6 40 46
25805 6 ▶ if (max_search_depth < 5)
Branch: 1 branch
BranchTrueFalseTotal
if (max_search_depth < 5) 0 6 6
25806 0 min_len = MIN(min_len, 4);
25807 6 ▶ else if (max_search_depth < 10)
Branch: 1 branch
BranchTrueFalseTotal
else if (max_search_depth < 10) 3 3 6
25808 3 min_len = MIN(min_len, 5);
25809 else
25810 3 min_len = MIN(min_len, 7);
25811 }
25812 46 return min_len;
25813 }
25814  
25815 static u32
25816 35 calculate_min_match_len(const u8 *data, size_t data_len, u32 max_search_depth)
25817 {
25818 35 u8 used[256] = { 0 };
25819 35 u32 num_used_literals = 0;
25820 size_t i;
25821  
25822
25823 35 ▶ if (data_len < 512)
Branch: 1 branch
BranchTrueFalseTotal
if (data_len < 512) 0 35 35
25824 0 return DEFLATE_MIN_MATCH_LEN;
25825  
25826
25827 35 data_len = MIN(data_len, 4096);
25828 64327 ▶ for (i = 0; i < data_len; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < data_len; i++) 64292 35 64327
25829 64292 used[data[i]] = 1;
25830 8995 ▶ for (i = 0; i < 256; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 256; i++) 8960 35 8995
25831 8960 num_used_literals += used[i];
25832 35 return choose_min_match_len(num_used_literals, max_search_depth);
25833 }
25834  
25835  
25836 static u32
25837 2 recalculate_min_match_len(const struct deflate_freqs *freqs,
25838 u32 max_search_depth)
25839 {
25840 2 u32 literal_freq = 0;
25841 u32 cutoff;
25842 2 u32 num_used_literals = 0;
25843 int i;
25844  
25845 514 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 512 2 514
25846 512 literal_freq += freqs->litlen[i];
25847  
25848 2 cutoff = literal_freq >> 10;
25849  
25850 514 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) { 512 2 514
25851 512 ▶ if (freqs->litlen[i] > cutoff)
Branch: 1 branch
BranchTrueFalseTotal
if (freqs->litlen[i] > cutoff) 119 393 512
25852 119 num_used_literals++;
25853 }
25854 2 return choose_min_match_len(num_used_literals, max_search_depth);
25855 }
25856  
25857 static forceinline const u8 *
25858 choose_max_block_end(const u8 *in_block_begin, const u8 *in_end,
25859 size_t soft_max_len)
25860 {
25861 9 ▶ if (in_end - in_block_begin < soft_max_len + MIN_BLOCK_LENGTH)
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_block_begin < soft_max_len + MIN_BLOCK_LENGTH) 9 0 9
25862 38 return in_end;
25863 0 return in_block_begin + soft_max_len;
25864 }
25865  
25866  
25867 static size_t
25868 0 deflate_compress_none(const u8 *in, size_t in_nbytes,
25869 u8 *out, size_t out_nbytes_avail)
25870 {
25871 0 const u8 *in_next = in;
25872 0 const u8 * const in_end = in + in_nbytes;
25873 0 u8 *out_next = out;
25874 0 u8 * const out_end = out + out_nbytes_avail;
25875  
25876
25877 0 ▶ if (unlikely(in_nbytes == 0)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(in_nbytes == 0)) { 0 0 0
25878 0 ▶ if (out_nbytes_avail < 5)
Branch: 1 branch
BranchTrueFalseTotal
if (out_nbytes_avail < 5) 0 0 0
25879 0 return 0;
25880
25881 0 *out_next++ = 1 | (DEFLATE_BLOCKTYPE_UNCOMPRESSED << 1);
25882
25883 put_unaligned_le32(0xFFFF0000, out_next);
25884 0 return 5;
25885 }
25886  
25887 do {
25888 0 u8 bfinal = 0;
25889 0 size_t len = UINT16_MAX;
25890  
25891 0 ▶ if (in_end - in_next <= UINT16_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next <= UINT16_MAX) { 0 0 0
25892 0 bfinal = 1;
25893 0 len = in_end - in_next;
25894 }
25895 0 ▶ if (out_end - out_next < 5 + len)
Branch: 1 branch
BranchTrueFalseTotal
if (out_end - out_next < 5 + len) 0 0 0
25896 0 return 0;
25897
25898 0 *out_next++ = bfinal | (DEFLATE_BLOCKTYPE_UNCOMPRESSED << 1);
25899  
25900
25901 0 put_unaligned_le16(len, out_next);
25902 0 out_next += 2;
25903 0 put_unaligned_le16(~len, out_next);
25904 0 out_next += 2;
25905 0 memcpy(out_next, in_next, len);
25906 0 out_next += len;
25907 0 in_next += len;
25908 0 ▶ } while (in_next != in_end);
Branch: 1 branch
BranchTrueFalseTotal
} while (in_next != in_end); 0 0 0
25909  
25910 0 return out_next - out;
25911 }
25912  
25913  
25914 static void
25915 3 deflate_compress_fastest(struct libdeflate_compressor * restrict c,
25916 const u8 *in, size_t in_nbytes,
25917 struct deflate_output_bitstream *os)
25918 {
25919 3 const u8 *in_next = in;
25920 3 const u8 *in_end = in_next + in_nbytes;
25921 3 const u8 *in_cur_base = in_next;
25922 3 u32 max_len = DEFLATE_MAX_MATCH_LEN;
25923 3 u32 nice_len = MIN(c->nice_match_length, max_len);
25924 3 u32 next_hash = 0;
25925  
25926 3 ht_matchfinder_init(&c->p.f.ht_mf);
25927  
25928 do {
25929
25930  
25931 3 ▶ const u8 * const in_block_begin = in_next;
Branch: 1 branch
BranchTrueFalseTotal
const u8 * const in_block_begin = in_next; 3 0 3
25932 3 const u8 * const in_max_block_end = choose_max_block_end(
25933 in_next, in_end, FAST_SOFT_MAX_BLOCK_LENGTH);
25934 3 struct deflate_sequence *seq = c->p.f.sequences;
25935  
25936 3 deflate_begin_sequences(c, seq);
25937  
25938 do {
25939 u32 length;
25940 u32 offset;
25941 75 size_t remaining = in_end - in_next;
25942  
25943 75 ▶ if (unlikely(remaining < DEFLATE_MAX_MATCH_LEN)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(remaining < DEFLATE_MAX_MATCH_LEN)) { 3 72 75
25944 3 max_len = remaining;
25945 3 ▶ if (max_len < HT_MATCHFINDER_REQUIRED_NBYTES) {
Branch: 1 branch
BranchTrueFalseTotal
if (max_len < HT_MATCHFINDER_REQUIRED_NBYTES) { 0 3 3
25946 do {
25947 0 deflate_choose_literal(c,
25948 0 ▶ *in_next++, false, seq);
Branch: 1 branch
BranchTrueFalseTotal
*in_next++, false, seq); 0 0 0
25949 0 ▶ } while (--max_len);
Branch: 1 branch
BranchTrueFalseTotal
} while (--max_len); 0 0 0
25950 0 break;
25951 }
25952 3 nice_len = MIN(nice_len, max_len);
25953 }
25954 75 ▶ length = ht_matchfinder_longest_match(&c->p.f.ht_mf,
Branch: 1 branch
BranchTrueFalseTotal
length = ht_matchfinder_longest_match(&c->p.f.ht_mf, 0 75 75
25955 &in_cur_base,
25956 in_next,
25957 max_len,
25958 nice_len,
25959 &next_hash,
25960 &offset);
25961 75 ▶ if (length) {
Branch: 1 branch
BranchTrueFalseTotal
if (length) { 21 54 75
25962
25963 21 ▶ deflate_choose_match(c, length, offset, false,
Branch: 1 branch
BranchTrueFalseTotal
deflate_choose_match(c, length, offset, false, 21 0 21
25964 &seq);
25965 21 ▶ ht_matchfinder_skip_bytes(&c->p.f.ht_mf,
Branch: 1 branch
BranchTrueFalseTotal
ht_matchfinder_skip_bytes(&c->p.f.ht_mf, 3 18 21
25966 &in_cur_base,
25967 in_next + 1,
25968 in_end,
25969 length - 1,
25970 &next_hash);
25971 21 in_next += length;
25972 } else {
25973
25974 54 ▶ deflate_choose_literal(c, *in_next++, false,
Branch: 1 branch
BranchTrueFalseTotal
deflate_choose_literal(c, *in_next++, false, 0 54 54
25975 seq);
25976 }
25977  
25978
25979 75 ▶ } while (in_next < in_max_block_end &&
Branch: 1 branch
BranchTrueFalseTotal
} while (in_next < in_max_block_end && 72 3 75
25980 72 ▶ seq < &c->p.f.sequences[FAST_SEQ_STORE_LENGTH]);
Branch: 1 branch
BranchTrueFalseTotal
seq < &c->p.f.sequences[FAST_SEQ_STORE_LENGTH]); 72 0 72
25981  
25982 3 deflate_finish_block(c, os, in_block_begin,
25983 3 in_next - in_block_begin,
25984 3 c->p.f.sequences, in_next == in_end);
25985 3 ▶ } while (in_next != in_end && !os->overflow);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next != in_end && !os->overflow); 0 3 3
} while (in_next != in_end && !os->overflow); 0 0 0
25986 3 }
25987  
25988  
25989 static void
25990 9 deflate_compress_greedy(struct libdeflate_compressor * restrict c,
25991 const u8 *in, size_t in_nbytes,
25992 struct deflate_output_bitstream *os)
25993 {
25994 9 const u8 *in_next = in;
25995 9 const u8 *in_end = in_next + in_nbytes;
25996 9 const u8 *in_cur_base = in_next;
25997 9 u32 max_len = DEFLATE_MAX_MATCH_LEN;
25998 9 u32 nice_len = MIN(c->nice_match_length, max_len);
25999 9 u32 next_hashes[2] = {0, 0};
26000  
26001 9 hc_matchfinder_init(&c->p.g.hc_mf);
26002  
26003 do {
26004
26005  
26006 9 ▶ const u8 * const in_block_begin = in_next;
Branch: 1 branch
BranchTrueFalseTotal
const u8 * const in_block_begin = in_next; 9 0 9
26007 9 const u8 * const in_max_block_end = choose_max_block_end(
26008 in_next, in_end, SOFT_MAX_BLOCK_LENGTH);
26009 9 struct deflate_sequence *seq = c->p.g.sequences;
26010 u32 min_len;
26011  
26012 9 init_block_split_stats(&c->split_stats);
26013 9 deflate_begin_sequences(c, seq);
26014 9 min_len = calculate_min_match_len(in_next,
26015 9 in_max_block_end - in_next,
26016 c->max_search_depth);
26017 do {
26018 u32 length;
26019 u32 offset;
26020  
26021 225 adjust_max_and_nice_len(&max_len, &nice_len,
26022 225 ▶ in_end - in_next);
Branch: 1 branch
BranchTrueFalseTotal
in_end - in_next); 9 216 225
26023 225 ▶ length = hc_matchfinder_longest_match(
Branch: 1 branch
BranchTrueFalseTotal
length = hc_matchfinder_longest_match( 0 225 225
26024 &c->p.g.hc_mf,
26025 &in_cur_base,
26026 in_next,
26027 min_len - 1,
26028 max_len,
26029 nice_len,
26030 c->max_search_depth,
26031 next_hashes,
26032 &offset);
26033  
26034 225 ▶ if (length >= min_len &&
Branch: 2 branches
BranchTrueFalseTotal
if (length >= min_len && 63 162 225
if (length >= min_len && 0 63 63
26035 0 (length > DEFLATE_MIN_MATCH_LEN ||
26036 0 ▶ offset <= 4096)) {
Branch: 1 branch
BranchTrueFalseTotal
offset <= 4096)) { 0 0 0
26037
26038 63 ▶ deflate_choose_match(c, length, offset, true,
Branch: 1 branch
BranchTrueFalseTotal
deflate_choose_match(c, length, offset, true, 63 0 63
26039 &seq);
26040 63 ▶ hc_matchfinder_skip_bytes(&c->p.g.hc_mf,
Branch: 1 branch
BranchTrueFalseTotal
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 9 54 63
26041 &in_cur_base,
26042 in_next + 1,
26043 in_end,
26044 length - 1,
26045 next_hashes);
26046 63 in_next += length;
26047 } else {
26048
26049 162 ▶ deflate_choose_literal(c, *in_next++, true,
Branch: 1 branch
BranchTrueFalseTotal
deflate_choose_literal(c, *in_next++, true, 162 0 162
26050 seq);
26051 }
26052  
26053
26054 216 } while (in_next < in_max_block_end &&
26055 225 ▶ seq < &c->p.g.sequences[SEQ_STORE_LENGTH] &&
Branch: 2 branches
BranchTrueFalseTotal
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 216 9 225
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 216 0 216
26056 432 ▶ !should_end_block(&c->split_stats,
Branch: 2 branches
BranchTrueFalseTotal
!should_end_block(&c->split_stats, 0 216 216
!should_end_block(&c->split_stats, 216 0 216
26057 in_block_begin, in_next, in_end));
26058  
26059 9 deflate_finish_block(c, os, in_block_begin,
26060 9 in_next - in_block_begin,
26061 9 c->p.g.sequences, in_next == in_end);
26062 9 ▶ } while (in_next != in_end && !os->overflow);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next != in_end && !os->overflow); 0 9 9
} while (in_next != in_end && !os->overflow); 0 0 0
26063 9 }
26064  
26065 static forceinline void
26066 deflate_compress_lazy_generic(struct libdeflate_compressor * restrict c,
26067 const u8 *in, size_t in_nbytes,
26068 struct deflate_output_bitstream *os, bool lazy2)
26069 {
26070 16 const u8 *in_next = in;
26071 16 const u8 *in_end = in_next + in_nbytes;
26072 16 const u8 *in_cur_base = in_next;
26073 16 u32 max_len = DEFLATE_MAX_MATCH_LEN;
26074 16 u32 nice_len = MIN(c->nice_match_length, max_len);
26075 16 u32 next_hashes[2] = {0, 0};
26076  
26077 16 hc_matchfinder_init(&c->p.g.hc_mf);
26078  
26079 do {
26080
26081  
26082 17 ▶ const u8 * const in_block_begin = in_next;
Branch: 2 branches
BranchTrueFalseTotal
const u8 * const in_block_begin = in_next; 6 0 6
const u8 * const in_block_begin = in_next; 11 0 11
26083 17 const u8 * const in_max_block_end = choose_max_block_end(
26084 in_next, in_end, SOFT_MAX_BLOCK_LENGTH);
26085 17 const u8 *next_recalc_min_len =
26086 17 in_next + MIN(in_end - in_next, 10000);
26087 17 struct deflate_sequence *seq = c->p.g.sequences;
26088 u32 min_len;
26089  
26090 17 init_block_split_stats(&c->split_stats);
26091 17 deflate_begin_sequences(c, seq);
26092 17 min_len = calculate_min_match_len(in_next,
26093 17 in_max_block_end - in_next,
26094 c->max_search_depth);
26095 do {
26096 u32 cur_len;
26097 u32 cur_offset;
26098 u32 next_len;
26099 u32 next_offset;
26100  
26101
26102 4829 ▶ if (in_next >= next_recalc_min_len) {
Branch: 2 branches
BranchTrueFalseTotal
if (in_next >= next_recalc_min_len) { 0 150 150
if (in_next >= next_recalc_min_len) { 2 4677 4679
26103 2 min_len = recalculate_min_match_len(
26104 2 &c->freqs,
26105 c->max_search_depth);
26106 2 next_recalc_min_len +=
26107 2 MIN(in_end - next_recalc_min_len,
26108 in_next - in_block_begin);
26109 }
26110  
26111
26112 4829 adjust_max_and_nice_len(&max_len, &nice_len,
26113 4829 ▶ in_end - in_next);
Branch: 2 branches
BranchTrueFalseTotal
in_end - in_next); 6 144 150
in_end - in_next); 40 4639 4679
26114 4829 ▶ cur_len = hc_matchfinder_longest_match(
Branch: 2 branches
BranchTrueFalseTotal
cur_len = hc_matchfinder_longest_match( 0 150 150
cur_len = hc_matchfinder_longest_match( 0 4679 4679
26115 &c->p.g.hc_mf,
26116 &in_cur_base,
26117 in_next,
26118 min_len - 1,
26119 max_len,
26120 nice_len,
26121 c->max_search_depth,
26122 next_hashes,
26123 &cur_offset);
26124 4829 ▶ if (cur_len < min_len ||
Branch: 4 branches
BranchTrueFalseTotal
if (cur_len < min_len || 42 108 150
if (cur_len < min_len || 0 42 42
if (cur_len < min_len || 1965 2714 4679
if (cur_len < min_len || 0 1965 1965
26125 0 (cur_len == DEFLATE_MIN_MATCH_LEN &&
26126 0 ▶ cur_offset > 8192)) {
Branch: 2 branches
BranchTrueFalseTotal
cur_offset > 8192)) { 0 0 0
cur_offset > 8192)) { 0 0 0
26127
26128 2822 ▶ deflate_choose_literal(c, *in_next++, true,
Branch: 2 branches
BranchTrueFalseTotal
deflate_choose_literal(c, *in_next++, true, 108 0 108
deflate_choose_literal(c, *in_next++, true, 2714 0 2714
26129 seq);
26130 2928 continue;
26131 }
26132 2007 in_next++;
26133  
26134 91 have_cur_match:
26135
26136 2098 ▶ if (cur_len >= nice_len) {
Branch: 2 branches
BranchTrueFalseTotal
if (cur_len >= nice_len) { 42 0 42
if (cur_len >= nice_len) { 64 1992 2056
26137 106 ▶ deflate_choose_match(c, cur_len, cur_offset,
Branch: 2 branches
BranchTrueFalseTotal
deflate_choose_match(c, cur_len, cur_offset, 42 0 42
deflate_choose_match(c, cur_len, cur_offset, 64 0 64
26138 true, &seq);
26139 106 ▶ hc_matchfinder_skip_bytes(&c->p.g.hc_mf,
Branch: 2 branches
BranchTrueFalseTotal
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 6 36 42
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 10 54 64
26140 &in_cur_base,
26141 in_next,
26142 in_end,
26143 cur_len - 1,
26144 next_hashes);
26145 106 in_next += cur_len - 1;
26146 106 continue;
26147 }
26148  
26149
26150 1992 adjust_max_and_nice_len(&max_len, &nice_len,
26151 1992 ▶ in_end - in_next);
Branch: 2 branches
BranchTrueFalseTotal
in_end - in_next); 0 0 0
in_end - in_next); 21 1971 1992
26152 3984 next_len = hc_matchfinder_longest_match(
26153 &c->p.g.hc_mf,
26154 &in_cur_base,
26155 in_next++,
26156 cur_len - 1,
26157 max_len,
26158 nice_len,
26159 1992 ▶ c->max_search_depth >> 1,
Branch: 2 branches
BranchTrueFalseTotal
c->max_search_depth >> 1, 0 0 0
c->max_search_depth >> 1, 0 1992 1992
26160 next_hashes,
26161 &next_offset);
26162 1992 ▶ if (next_len >= cur_len &&
Branch: 2 branches
BranchTrueFalseTotal
if (next_len >= cur_len && 0 0 0
if (next_len >= cur_len && 138 1854 1992
26163 276 4 * (int)(next_len - cur_len) +
26164 138 ((int)bsr32(cur_offset) -
26165 276 ▶ (int)bsr32(next_offset)) > 2) {
Branch: 2 branches
BranchTrueFalseTotal
(int)bsr32(next_offset)) > 2) { 0 0 0
(int)bsr32(next_offset)) > 2) { 91 47 138
26166
26167 91 ▶ deflate_choose_literal(c, *(in_next - 2), true,
Branch: 2 branches
BranchTrueFalseTotal
deflate_choose_literal(c, *(in_next - 2), true, 0 0 0
deflate_choose_literal(c, *(in_next - 2), true, 91 0 91
26168 seq);
26169 91 cur_len = next_len;
26170 91 cur_offset = next_offset;
26171 91 goto have_cur_match;
26172 }
26173  
26174 1901 ▶ if (lazy2) {
Branch: 2 branches
BranchTrueFalseTotal
if (lazy2) { 0 0 0
if (lazy2) { 0 1901 1901
26175
26176 0 adjust_max_and_nice_len(&max_len, &nice_len,
26177 0 ▶ in_end - in_next);
Branch: 2 branches
BranchTrueFalseTotal
in_end - in_next); 0 0 0
in_end - in_next); 0 0 0
26178 0 next_len = hc_matchfinder_longest_match(
26179 &c->p.g.hc_mf,
26180 &in_cur_base,
26181 in_next++,
26182 cur_len - 1,
26183 max_len,
26184 nice_len,
26185 0 ▶ c->max_search_depth >> 2,
Branch: 2 branches
BranchTrueFalseTotal
c->max_search_depth >> 2, 0 0 0
c->max_search_depth >> 2, 0 0 0
26186 next_hashes,
26187 &next_offset);
26188 0 ▶ if (next_len >= cur_len &&
Branch: 2 branches
BranchTrueFalseTotal
if (next_len >= cur_len && 0 0 0
if (next_len >= cur_len && 0 0 0
26189 0 4 * (int)(next_len - cur_len) +
26190 0 ((int)bsr32(cur_offset) -
26191 0 ▶ (int)bsr32(next_offset)) > 6) {
Branch: 2 branches
BranchTrueFalseTotal
(int)bsr32(next_offset)) > 6) { 0 0 0
(int)bsr32(next_offset)) > 6) { 0 0 0
26192
26193 0 deflate_choose_literal(
26194 0 ▶ c, *(in_next - 3), true, seq);
Branch: 2 branches
BranchTrueFalseTotal
c, *(in_next - 3), true, seq); 0 0 0
c, *(in_next - 3), true, seq); 0 0 0
26195 0 deflate_choose_literal(
26196 0 ▶ c, *(in_next - 2), true, seq);
Branch: 2 branches
BranchTrueFalseTotal
c, *(in_next - 2), true, seq); 0 0 0
c, *(in_next - 2), true, seq); 0 0 0
26197 0 cur_len = next_len;
26198 0 cur_offset = next_offset;
26199 0 goto have_cur_match;
26200 }
26201
26202 0 ▶ deflate_choose_match(c, cur_len, cur_offset,
Branch: 2 branches
BranchTrueFalseTotal
deflate_choose_match(c, cur_len, cur_offset, 0 0 0
deflate_choose_match(c, cur_len, cur_offset, 0 0 0
26203 true, &seq);
26204 0 ▶ if (cur_len > 3) {
Branch: 2 branches
BranchTrueFalseTotal
if (cur_len > 3) { 0 0 0
if (cur_len > 3) { 0 0 0
26205 0 ▶ hc_matchfinder_skip_bytes(&c->p.g.hc_mf,
Branch: 2 branches
BranchTrueFalseTotal
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 0 0 0
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 0 0 0
26206 &in_cur_base,
26207 in_next,
26208 in_end,
26209 cur_len - 3,
26210 next_hashes);
26211 0 in_next += cur_len - 3;
26212 }
26213 } else {
26214
26215 1901 ▶ deflate_choose_match(c, cur_len, cur_offset,
Branch: 2 branches
BranchTrueFalseTotal
deflate_choose_match(c, cur_len, cur_offset, 0 0 0
deflate_choose_match(c, cur_len, cur_offset, 1901 0 1901
26216 true, &seq);
26217 1901 ▶ hc_matchfinder_skip_bytes(&c->p.g.hc_mf,
Branch: 2 branches
BranchTrueFalseTotal
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 0 0 0
hc_matchfinder_skip_bytes(&c->p.g.hc_mf, 0 1901 1901
26218 &in_cur_base,
26219 in_next,
26220 in_end,
26221 cur_len - 2,
26222 next_hashes);
26223 1901 in_next += cur_len - 2;
26224 }
26225
26226 4813 } while (in_next < in_max_block_end &&
26227 4829 ▶ seq < &c->p.g.sequences[SEQ_STORE_LENGTH] &&
Branch: 4 branches
BranchTrueFalseTotal
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 144 6 150
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 144 0 144
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 4669 10 4679
seq < &c->p.g.sequences[SEQ_STORE_LENGTH] && 4669 0 4669
26228 9626 ▶ !should_end_block(&c->split_stats,
Branch: 4 branches
BranchTrueFalseTotal
!should_end_block(&c->split_stats, 0 144 144
!should_end_block(&c->split_stats, 144 0 144
!should_end_block(&c->split_stats, 450 4219 4669
!should_end_block(&c->split_stats, 4668 1 4669
26229 in_block_begin, in_next, in_end));
26230  
26231 17 deflate_finish_block(c, os, in_block_begin,
26232 17 in_next - in_block_begin,
26233 17 c->p.g.sequences, in_next == in_end);
26234 17 ▶ } while (in_next != in_end && !os->overflow);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next != in_end && !os->overflow); 0 6 6
} while (in_next != in_end && !os->overflow); 1 0 1
26235 16 }
26236  
26237  
26238 static void
26239 10 deflate_compress_lazy(struct libdeflate_compressor * restrict c,
26240 const u8 *in, size_t in_nbytes,
26241 struct deflate_output_bitstream *os)
26242 {
26243 deflate_compress_lazy_generic(c, in, in_nbytes, os, false);
26244 10 }
26245  
26246  
26247 static void
26248 6 deflate_compress_lazy2(struct libdeflate_compressor * restrict c,
26249 const u8 *in, size_t in_nbytes,
26250 struct deflate_output_bitstream *os)
26251 {
26252 deflate_compress_lazy_generic(c, in, in_nbytes, os, true);
26253 6 }
26254  
26255 #if SUPPORT_NEAR_OPTIMAL_PARSING
26256  
26257  
26258 static void
26259 24 deflate_tally_item_list(struct libdeflate_compressor *c, u32 block_length)
26260 {
26261 24 struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0];
26262 24 struct deflate_optimum_node *end_node =
26263 &c->p.n.optimum_nodes[block_length];
26264  
26265 do {
26266 600 u32 length = cur_node->item & OPTIMUM_LEN_MASK;
26267 600 u32 offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
26268  
26269 600 ▶ if (length == 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (length == 1) { 432 168 600
26270
26271 432 c->freqs.litlen[offset]++;
26272 } else {
26273
26274 168 c->freqs.litlen[DEFLATE_FIRST_LEN_SYM +
26275 168 deflate_length_slot[length]]++;
26276 168 c->freqs.offset[c->p.n.offset_slot_full[offset]]++;
26277 }
26278 600 cur_node += length;
26279 600 ▶ } while (cur_node != end_node);
Branch: 1 branch
BranchTrueFalseTotal
} while (cur_node != end_node); 576 24 600
26280  
26281
26282 24 c->freqs.litlen[DEFLATE_END_OF_BLOCK]++;
26283 24 }
26284  
26285 static void
26286 9 deflate_choose_all_literals(struct libdeflate_compressor *c,
26287 const u8 *block, u32 block_length)
26288 {
26289 u32 i;
26290  
26291 9 deflate_reset_symbol_frequencies(c);
26292 15309 ▶ for (i = 0; i < block_length; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < block_length; i++) 15300 9 15309
26293 15300 c->freqs.litlen[block[i]]++;
26294 9 c->freqs.litlen[DEFLATE_END_OF_BLOCK]++;
26295  
26296 9 deflate_make_huffman_codes(&c->freqs, &c->codes);
26297 9 }
26298  
26299  
26300 static u32
26301 27 deflate_compute_true_cost(struct libdeflate_compressor *c)
26302 {
26303 27 u32 cost = 0;
26304 unsigned sym;
26305  
26306 27 deflate_precompute_huffman_header(c);
26307  
26308 27 memset(&c->codes.lens.litlen[c->o.precode.num_litlen_syms], 0,
26309 27 DEFLATE_NUM_LITLEN_SYMS - c->o.precode.num_litlen_syms);
26310  
26311 27 cost += 5 + 5 + 4 + (3 * c->o.precode.num_explicit_lens);
26312 540 ▶ for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) {
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) { 513 27 540
26313 513 cost += c->o.precode.freqs[sym] *
26314 513 (c->o.precode.lens[sym] +
26315 513 deflate_extra_precode_bits[sym]);
26316 }
26317  
26318 6966 ▶ for (sym = 0; sym < DEFLATE_FIRST_LEN_SYM; sym++)
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < DEFLATE_FIRST_LEN_SYM; sym++) 6939 27 6966
26319 6939 cost += c->freqs.litlen[sym] * c->codes.lens.litlen[sym];
26320  
26321 810 ▶ for (; sym < DEFLATE_FIRST_LEN_SYM +
Branch: 1 branch
BranchTrueFalseTotal
for (; sym < DEFLATE_FIRST_LEN_SYM + 783 27 810
26322 783 ARRAY_LEN(deflate_extra_length_bits); sym++)
26323 783 cost += c->freqs.litlen[sym] *
26324 783 (c->codes.lens.litlen[sym] +
26325 783 deflate_extra_length_bits[sym - DEFLATE_FIRST_LEN_SYM]);
26326  
26327 837 ▶ for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++)
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++) 810 27 837
26328 810 cost += c->freqs.offset[sym] *
26329 810 (c->codes.lens.offset[sym] +
26330 810 deflate_extra_offset_bits[sym]);
26331 27 return cost;
26332 }
26333  
26334  
26335 static void
26336 15 deflate_set_costs_from_codes(struct libdeflate_compressor *c,
26337 const struct deflate_lens *lens)
26338 {
26339 unsigned i;
26340  
26341
26342 3855 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) { 3840 15 3855
26343 7680 u32 bits = (lens->litlen[i] ?
26344 3840 ▶ lens->litlen[i] : LITERAL_NOSTAT_BITS);
Branch: 1 branch
BranchTrueFalseTotal
lens->litlen[i] : LITERAL_NOSTAT_BITS); 1644 2196 3840
26345  
26346 3840 c->p.n.costs.literal[i] = bits * BIT_COST;
26347 }
26348  
26349
26350 3855 ▶ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) { 3840 15 3855
26351 3840 unsigned length_slot = deflate_length_slot[i];
26352 3840 unsigned litlen_sym = DEFLATE_FIRST_LEN_SYM + length_slot;
26353 7680 u32 bits = (lens->litlen[litlen_sym] ?
26354 3840 ▶ lens->litlen[litlen_sym] : LENGTH_NOSTAT_BITS);
Branch: 1 branch
BranchTrueFalseTotal
lens->litlen[litlen_sym] : LENGTH_NOSTAT_BITS); 1833 2007 3840
26355  
26356 3840 bits += deflate_extra_length_bits[length_slot];
26357 3840 c->p.n.costs.length[i] = bits * BIT_COST;
26358 }
26359  
26360
26361 465 ▶ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) { 450 15 465
26362 900 u32 bits = (lens->offset[i] ?
26363 450 ▶ lens->offset[i] : OFFSET_NOSTAT_BITS);
Branch: 1 branch
BranchTrueFalseTotal
lens->offset[i] : OFFSET_NOSTAT_BITS); 198 252 450
26364  
26365 450 bits += deflate_extra_offset_bits[i];
26366 450 c->p.n.costs.offset_slot[i] = bits * BIT_COST;
26367 }
26368 15 }
26369  
26370  
26371 static const struct {
26372 u8 used_lits_to_lit_cost[257];
26373 u8 len_sym_cost;
26374 } default_litlen_costs[] = {
26375 {
26376 .used_lits_to_lit_cost = {
26377 6, 6, 22, 32, 38, 43, 48, 51,
26378 54, 57, 59, 61, 64, 65, 67, 69,
26379 70, 72, 73, 74, 75, 76, 77, 79,
26380 80, 80, 81, 82, 83, 84, 85, 85,
26381 86, 87, 88, 88, 89, 89, 90, 91,
26382 91, 92, 92, 93, 93, 94, 95, 95,
26383 96, 96, 96, 97, 97, 98, 98, 99,
26384 99, 99, 100, 100, 101, 101, 101, 102,
26385 102, 102, 103, 103, 104, 104, 104, 105,
26386 105, 105, 105, 106, 106, 106, 107, 107,
26387 107, 108, 108, 108, 108, 109, 109, 109,
26388 109, 110, 110, 110, 111, 111, 111, 111,
26389 112, 112, 112, 112, 112, 113, 113, 113,
26390 113, 114, 114, 114, 114, 114, 115, 115,
26391 115, 115, 115, 116, 116, 116, 116, 116,
26392 117, 117, 117, 117, 117, 118, 118, 118,
26393 118, 118, 118, 119, 119, 119, 119, 119,
26394 120, 120, 120, 120, 120, 120, 121, 121,
26395 121, 121, 121, 121, 121, 122, 122, 122,
26396 122, 122, 122, 123, 123, 123, 123, 123,
26397 123, 123, 124, 124, 124, 124, 124, 124,
26398 124, 125, 125, 125, 125, 125, 125, 125,
26399 125, 126, 126, 126, 126, 126, 126, 126,
26400 127, 127, 127, 127, 127, 127, 127, 127,
26401 128, 128, 128, 128, 128, 128, 128, 128,
26402 128, 129, 129, 129, 129, 129, 129, 129,
26403 129, 129, 130, 130, 130, 130, 130, 130,
26404 130, 130, 130, 131, 131, 131, 131, 131,
26405 131, 131, 131, 131, 131, 132, 132, 132,
26406 132, 132, 132, 132, 132, 132, 132, 133,
26407 133, 133, 133, 133, 133, 133, 133, 133,
26408 133, 134, 134, 134, 134, 134, 134, 134,
26409 134,
26410 },
26411 .len_sym_cost = 109,
26412 }, {
26413 .used_lits_to_lit_cost = {
26414 16, 16, 32, 41, 48, 53, 57, 60,
26415 64, 66, 69, 71, 73, 75, 76, 78,
26416 80, 81, 82, 83, 85, 86, 87, 88,
26417 89, 90, 91, 92, 92, 93, 94, 95,
26418 96, 96, 97, 98, 98, 99, 99, 100,
26419 101, 101, 102, 102, 103, 103, 104, 104,
26420 105, 105, 106, 106, 107, 107, 108, 108,
26421 108, 109, 109, 110, 110, 110, 111, 111,
26422 112, 112, 112, 113, 113, 113, 114, 114,
26423 114, 115, 115, 115, 115, 116, 116, 116,
26424 117, 117, 117, 118, 118, 118, 118, 119,
26425 119, 119, 119, 120, 120, 120, 120, 121,
26426 121, 121, 121, 122, 122, 122, 122, 122,
26427 123, 123, 123, 123, 124, 124, 124, 124,
26428 124, 125, 125, 125, 125, 125, 126, 126,
26429 126, 126, 126, 127, 127, 127, 127, 127,
26430 128, 128, 128, 128, 128, 128, 129, 129,
26431 129, 129, 129, 129, 130, 130, 130, 130,
26432 130, 130, 131, 131, 131, 131, 131, 131,
26433 131, 132, 132, 132, 132, 132, 132, 133,
26434 133, 133, 133, 133, 133, 133, 134, 134,
26435 134, 134, 134, 134, 134, 134, 135, 135,
26436 135, 135, 135, 135, 135, 135, 136, 136,
26437 136, 136, 136, 136, 136, 136, 137, 137,
26438 137, 137, 137, 137, 137, 137, 138, 138,
26439 138, 138, 138, 138, 138, 138, 138, 139,
26440 139, 139, 139, 139, 139, 139, 139, 139,
26441 140, 140, 140, 140, 140, 140, 140, 140,
26442 140, 141, 141, 141, 141, 141, 141, 141,
26443 141, 141, 141, 142, 142, 142, 142, 142,
26444 142, 142, 142, 142, 142, 142, 143, 143,
26445 143, 143, 143, 143, 143, 143, 143, 143,
26446 144,
26447 },
26448 .len_sym_cost = 93,
26449 }, {
26450 .used_lits_to_lit_cost = {
26451 32, 32, 48, 57, 64, 69, 73, 76,
26452 80, 82, 85, 87, 89, 91, 92, 94,
26453 96, 97, 98, 99, 101, 102, 103, 104,
26454 105, 106, 107, 108, 108, 109, 110, 111,
26455 112, 112, 113, 114, 114, 115, 115, 116,
26456 117, 117, 118, 118, 119, 119, 120, 120,
26457 121, 121, 122, 122, 123, 123, 124, 124,
26458 124, 125, 125, 126, 126, 126, 127, 127,
26459 128, 128, 128, 129, 129, 129, 130, 130,
26460 130, 131, 131, 131, 131, 132, 132, 132,
26461 133, 133, 133, 134, 134, 134, 134, 135,
26462 135, 135, 135, 136, 136, 136, 136, 137,
26463 137, 137, 137, 138, 138, 138, 138, 138,
26464 139, 139, 139, 139, 140, 140, 140, 140,
26465 140, 141, 141, 141, 141, 141, 142, 142,
26466 142, 142, 142, 143, 143, 143, 143, 143,
26467 144, 144, 144, 144, 144, 144, 145, 145,
26468 145, 145, 145, 145, 146, 146, 146, 146,
26469 146, 146, 147, 147, 147, 147, 147, 147,
26470 147, 148, 148, 148, 148, 148, 148, 149,
26471 149, 149, 149, 149, 149, 149, 150, 150,
26472 150, 150, 150, 150, 150, 150, 151, 151,
26473 151, 151, 151, 151, 151, 151, 152, 152,
26474 152, 152, 152, 152, 152, 152, 153, 153,
26475 153, 153, 153, 153, 153, 153, 154, 154,
26476 154, 154, 154, 154, 154, 154, 154, 155,
26477 155, 155, 155, 155, 155, 155, 155, 155,
26478 156, 156, 156, 156, 156, 156, 156, 156,
26479 156, 157, 157, 157, 157, 157, 157, 157,
26480 157, 157, 157, 158, 158, 158, 158, 158,
26481 158, 158, 158, 158, 158, 158, 159, 159,
26482 159, 159, 159, 159, 159, 159, 159, 159,
26483 160,
26484 },
26485 .len_sym_cost = 84,
26486 },
26487 };
26488  
26489  
26490 static void
26491 9 deflate_choose_default_litlen_costs(struct libdeflate_compressor *c,
26492 const u8 *block_begin, u32 block_length,
26493 u32 *lit_cost, u32 *len_sym_cost)
26494 {
26495 9 u32 num_used_literals = 0;
26496 9 u32 literal_freq = block_length;
26497 9 u32 match_freq = 0;
26498 u32 cutoff;
26499 u32 i;
26500  
26501
26502 9 memset(c->freqs.litlen, 0,
26503 DEFLATE_NUM_LITERALS * sizeof(c->freqs.litlen[0]));
26504 9 cutoff = literal_freq >> 11;
26505 15309 ▶ for (i = 0; i < block_length; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < block_length; i++) 15300 9 15309
26506 15300 c->freqs.litlen[block_begin[i]]++;
26507 2313 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) { 2304 9 2313
26508 2304 ▶ if (c->freqs.litlen[i] > cutoff)
Branch: 1 branch
BranchTrueFalseTotal
if (c->freqs.litlen[i] > cutoff) 108 2196 2304
26509 108 num_used_literals++;
26510 }
26511 9 ▶ if (num_used_literals == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (num_used_literals == 0) 0 9 9
26512 0 num_used_literals = 1;
26513  
26514
26515 9 match_freq = 0;
26516 9 i = choose_min_match_len(num_used_literals, c->max_search_depth);
26517 2286 ▶ for (; i < ARRAY_LEN(c->p.n.match_len_freqs); i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i < ARRAY_LEN(c->p.n.match_len_freqs); i++) { 2277 9 2286
26518 2277 match_freq += c->p.n.match_len_freqs[i];
26519 2277 literal_freq -= i * c->p.n.match_len_freqs[i];
26520 }
26521 9 ▶ if ((s32)literal_freq < 0)
Branch: 1 branch
BranchTrueFalseTotal
if ((s32)literal_freq < 0) 0 9 9
26522 0 literal_freq = 0;
26523  
26524 9 ▶ if (match_freq > literal_freq)
Branch: 1 branch
BranchTrueFalseTotal
if (match_freq > literal_freq) 0 9 9
26525 0 i = 2;
26526 9 ▶ else if (match_freq * 4 > literal_freq)
Branch: 1 branch
BranchTrueFalseTotal
else if (match_freq * 4 > literal_freq) 9 0 9
26527 9 i = 1;
26528 else
26529 0 i = 0;
26530  
26531 STATIC_ASSERT(BIT_COST == 16);
26532 9 *lit_cost = default_litlen_costs[i].used_lits_to_lit_cost[
26533 num_used_literals];
26534 9 *len_sym_cost = default_litlen_costs[i].len_sym_cost;
26535 9 }
26536  
26537 static forceinline u32
26538 deflate_default_length_cost(u32 len, u32 len_sym_cost)
26539 {
26540 2304 unsigned slot = deflate_length_slot[len];
26541 2304 u32 num_extra_bits = deflate_extra_length_bits[slot];
26542  
26543 2304 return len_sym_cost + (num_extra_bits * BIT_COST);
26544 }
26545  
26546 static forceinline u32
26547 deflate_default_offset_slot_cost(unsigned slot)
26548 {
26549 270 u32 num_extra_bits = deflate_extra_offset_bits[slot];
26550
26551 270 u32 offset_sym_cost = 4*BIT_COST + (907*BIT_COST)/1000;
26552  
26553 270 return offset_sym_cost + (num_extra_bits * BIT_COST);
26554 }
26555  
26556  
26557 static void
26558 9 deflate_set_default_costs(struct libdeflate_compressor *c,
26559 u32 lit_cost, u32 len_sym_cost)
26560 {
26561 u32 i;
26562  
26563
26564 2313 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 2304 9 2313
26565 2304 c->p.n.costs.literal[i] = lit_cost;
26566  
26567
26568 2313 ▶ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) 2304 9 2313
26569 2304 c->p.n.costs.length[i] =
26570 2304 deflate_default_length_cost(i, len_sym_cost);
26571  
26572
26573 279 ▶ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) 270 9 279
26574 270 c->p.n.costs.offset_slot[i] =
26575 270 deflate_default_offset_slot_cost(i);
26576 9 }
26577  
26578 static forceinline void
26579 deflate_adjust_cost(u32 *cost_p, u32 default_cost, int change_amount)
26580 {
26581 0 if (change_amount == 0)
26582
26583 0 *cost_p = (default_cost + 3 * *cost_p) / 4;
26584 0 ▶ else if (change_amount == 1)
Branch: 12 branches
BranchTrueFalseTotal
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
else if (change_amount == 1) 0 0 0
26585 0 *cost_p = (default_cost + *cost_p) / 2;
26586 0 ▶ else if (change_amount == 2)
Branch: 12 branches
BranchTrueFalseTotal
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
else if (change_amount == 2) 0 0 0
26587 0 *cost_p = (5 * default_cost + 3 * *cost_p) / 8;
26588 else
26589
26590 0 *cost_p = (3 * default_cost + *cost_p) / 4;
26591 0 }
26592  
26593 static forceinline void
26594 deflate_adjust_costs_impl(struct libdeflate_compressor *c,
26595 u32 lit_cost, u32 len_sym_cost, int change_amount)
26596 {
26597 u32 i;
26598  
26599
26600 0 ▶ for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
Branch: 4 branches
BranchTrueFalseTotal
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 0 0 0
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 0 0 0
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 0 0 0
for (i = 0; i < DEFLATE_NUM_LITERALS; i++) 0 0 0
26601 0 ▶ deflate_adjust_cost(&c->p.n.costs.literal[i], lit_cost,
Branch: 4 branches
BranchTrueFalseTotal
deflate_adjust_cost(&c->p.n.costs.literal[i], lit_cost, 0 0 0
deflate_adjust_cost(&c->p.n.costs.literal[i], lit_cost, 0 0 0
deflate_adjust_cost(&c->p.n.costs.literal[i], lit_cost, 0 0 0
deflate_adjust_cost(&c->p.n.costs.literal[i], lit_cost, 0 0 0
26602 change_amount);
26603  
26604
26605 0 ▶ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++)
Branch: 4 branches
BranchTrueFalseTotal
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) 0 0 0
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) 0 0 0
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) 0 0 0
for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) 0 0 0
26606 0 ▶ deflate_adjust_cost(&c->p.n.costs.length[i],
Branch: 4 branches
BranchTrueFalseTotal
deflate_adjust_cost(&c->p.n.costs.length[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.length[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.length[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.length[i], 0 0 0
26607 deflate_default_length_cost(i,
26608 len_sym_cost),
26609 change_amount);
26610  
26611
26612 0 ▶ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++)
Branch: 4 branches
BranchTrueFalseTotal
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) 0 0 0
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) 0 0 0
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) 0 0 0
for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) 0 0 0
26613 0 ▶ deflate_adjust_cost(&c->p.n.costs.offset_slot[i],
Branch: 4 branches
BranchTrueFalseTotal
deflate_adjust_cost(&c->p.n.costs.offset_slot[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.offset_slot[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.offset_slot[i], 0 0 0
deflate_adjust_cost(&c->p.n.costs.offset_slot[i], 0 0 0
26614 deflate_default_offset_slot_cost(i),
26615 change_amount);
26616 0 }
26617  
26618  
26619 static void
26620 0 deflate_adjust_costs(struct libdeflate_compressor *c,
26621 u32 lit_cost, u32 len_sym_cost)
26622 {
26623 0 u64 total_delta = 0;
26624 u64 cutoff;
26625 int i;
26626  
26627
26628 0 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { 0 0 0
26629 0 u64 prev = (u64)c->p.n.prev_observations[i] *
26630 0 c->split_stats.num_observations;
26631 0 u64 cur = (u64)c->split_stats.observations[i] *
26632 0 c->p.n.prev_num_observations;
26633  
26634 0 ▶ total_delta += prev > cur ? prev - cur : cur - prev;
Branch: 1 branch
BranchTrueFalseTotal
total_delta += prev > cur ? prev - cur : cur - prev; 0 0 0
26635 }
26636 0 cutoff = ((u64)c->p.n.prev_num_observations *
26637 0 c->split_stats.num_observations * 200) / 512;
26638  
26639 0 ▶ if (total_delta > 3 * cutoff)
Branch: 1 branch
BranchTrueFalseTotal
if (total_delta > 3 * cutoff) 0 0 0
26640
26641 0 deflate_set_default_costs(c, lit_cost, len_sym_cost);
26642 0 ▶ else if (4 * total_delta > 9 * cutoff)
Branch: 1 branch
BranchTrueFalseTotal
else if (4 * total_delta > 9 * cutoff) 0 0 0
26643 deflate_adjust_costs_impl(c, lit_cost, len_sym_cost, 3);
26644 0 ▶ else if (2 * total_delta > 3 * cutoff)
Branch: 1 branch
BranchTrueFalseTotal
else if (2 * total_delta > 3 * cutoff) 0 0 0
26645 deflate_adjust_costs_impl(c, lit_cost, len_sym_cost, 2);
26646 0 ▶ else if (2 * total_delta > cutoff)
Branch: 1 branch
BranchTrueFalseTotal
else if (2 * total_delta > cutoff) 0 0 0
26647 deflate_adjust_costs_impl(c, lit_cost, len_sym_cost, 1);
26648 else
26649 deflate_adjust_costs_impl(c, lit_cost, len_sym_cost, 0);
26650 0 }
26651  
26652 static void
26653 9 deflate_set_initial_costs(struct libdeflate_compressor *c,
26654 const u8 *block_begin, u32 block_length,
26655 bool is_first_block)
26656 {
26657 u32 lit_cost, len_sym_cost;
26658  
26659 9 deflate_choose_default_litlen_costs(c, block_begin, block_length,
26660 &lit_cost, &len_sym_cost);
26661 9 ▶ if (is_first_block)
Branch: 1 branch
BranchTrueFalseTotal
if (is_first_block) 9 0 9
26662 9 deflate_set_default_costs(c, lit_cost, len_sym_cost);
26663 else
26664 0 deflate_adjust_costs(c, lit_cost, len_sym_cost);
26665 9 }
26666  
26667  
26668 static void
26669 24 deflate_find_min_cost_path(struct libdeflate_compressor *c,
26670 const u32 block_length,
26671 const struct lz_match *cache_ptr)
26672 {
26673 24 struct deflate_optimum_node *end_node =
26674 &c->p.n.optimum_nodes[block_length];
26675 24 struct deflate_optimum_node *cur_node = end_node;
26676  
26677 24 cur_node->cost_to_end = 0;
26678 do {
26679 unsigned num_matches;
26680 u32 literal;
26681 u32 best_cost_to_end;
26682  
26683 40800 cur_node--;
26684 40800 cache_ptr--;
26685  
26686 40800 num_matches = cache_ptr->length;
26687 40800 literal = cache_ptr->offset;
26688  
26689
26690 40800 best_cost_to_end = c->p.n.costs.literal[literal] +
26691 40800 (cur_node + 1)->cost_to_end;
26692 40800 cur_node->item = (literal << OPTIMUM_OFFSET_SHIFT) | 1;
26693  
26694
26695 40800 ▶ if (num_matches) {
Branch: 1 branch
BranchTrueFalseTotal
if (num_matches) { 168 40632 40800
26696 const struct lz_match *match;
26697 u32 len;
26698 u32 offset;
26699 u32 offset_slot;
26700 u32 offset_cost;
26701 u32 cost_to_end;
26702  
26703
26704 168 match = cache_ptr - num_matches;
26705 168 len = DEFLATE_MIN_MATCH_LEN;
26706 do {
26707 336 offset = match->offset;
26708 336 offset_slot = c->p.n.offset_slot_full[offset];
26709 336 offset_cost =
26710 c->p.n.costs.offset_slot[offset_slot];
26711 do {
26712 40032 cost_to_end = offset_cost +
26713 40032 c->p.n.costs.length[len] +
26714 40032 (cur_node + len)->cost_to_end;
26715 40032 ▶ if (cost_to_end < best_cost_to_end) {
Branch: 1 branch
BranchTrueFalseTotal
if (cost_to_end < best_cost_to_end) { 39753 279 40032
26716 39753 best_cost_to_end = cost_to_end;
26717 39753 cur_node->item = len |
26718 39753 (offset <<
26719 OPTIMUM_OFFSET_SHIFT);
26720 }
26721 40032 ▶ } while (++len <= match->length);
Branch: 1 branch
BranchTrueFalseTotal
} while (++len <= match->length); 39696 336 40032
26722 336 ▶ } while (++match != cache_ptr);
Branch: 1 branch
BranchTrueFalseTotal
} while (++match != cache_ptr); 168 168 336
26723 168 cache_ptr -= num_matches;
26724 }
26725 40800 cur_node->cost_to_end = best_cost_to_end;
26726 40800 ▶ } while (cur_node != &c->p.n.optimum_nodes[0]);
Branch: 1 branch
BranchTrueFalseTotal
} while (cur_node != &c->p.n.optimum_nodes[0]); 40776 24 40800
26727  
26728 24 deflate_reset_symbol_frequencies(c);
26729 24 deflate_tally_item_list(c, block_length);
26730 24 deflate_make_huffman_codes(&c->freqs, &c->codes);
26731 24 }
26732  
26733  
26734 static void
26735 9 deflate_optimize_and_flush_block(struct libdeflate_compressor *c,
26736 struct deflate_output_bitstream *os,
26737 const u8 *block_begin, u32 block_length,
26738 const struct lz_match *cache_ptr,
26739 bool is_first_block, bool is_final_block,
26740 bool *used_only_literals)
26741 {
26742 9 unsigned num_passes_remaining = c->p.n.max_optim_passes;
26743 9 u32 best_true_cost = UINT32_MAX;
26744 u32 true_cost;
26745 u32 only_lits_cost;
26746 9 u32 static_cost = UINT32_MAX;
26747 struct deflate_sequence seq_;
26748 9 struct deflate_sequence *seq = NULL;
26749 u32 i;
26750  
26751
26752 9 deflate_choose_all_literals(c, block_begin, block_length);
26753 9 only_lits_cost = deflate_compute_true_cost(c);
26754  
26755
26756 9 for (i = block_length;
26757 2331 ▶ i <= MIN(block_length - 1 + DEFLATE_MAX_MATCH_LEN,
Branch: 1 branch
BranchTrueFalseTotal
i <= MIN(block_length - 1 + DEFLATE_MAX_MATCH_LEN, 2322 9 2331
26758 2322 ARRAY_LEN(c->p.n.optimum_nodes) - 1); i++)
26759 2322 c->p.n.optimum_nodes[i].cost_to_end = 0x80000000;
26760  
26761
26762 9 ▶ if (block_length <= c->p.n.max_len_to_optimize_static_block) {
Branch: 1 branch
BranchTrueFalseTotal
if (block_length <= c->p.n.max_len_to_optimize_static_block) { 3 6 9
26763
26764 3 c->p.n.costs_saved = c->p.n.costs;
26765  
26766 3 deflate_set_costs_from_codes(c, &c->static_codes.lens);
26767 3 deflate_find_min_cost_path(c, block_length, cache_ptr);
26768 3 static_cost = c->p.n.optimum_nodes[0].cost_to_end / BIT_COST;
26769 3 static_cost += 7;
26770  
26771
26772 3 c->p.n.costs = c->p.n.costs_saved;
26773 }
26774  
26775
26776 9 deflate_set_initial_costs(c, block_begin, block_length, is_first_block);
26777  
26778 do {
26779
26780 18 deflate_find_min_cost_path(c, block_length, cache_ptr);
26781  
26782
26783 18 true_cost = deflate_compute_true_cost(c);
26784  
26785
26786 18 ▶ if (true_cost + c->p.n.min_improvement_to_continue >
Branch: 1 branch
BranchTrueFalseTotal
if (true_cost + c->p.n.min_improvement_to_continue > 9 9 18
26787 best_true_cost)
26788 9 break;
26789  
26790 9 best_true_cost = true_cost;
26791  
26792
26793 9 c->p.n.costs_saved = c->p.n.costs;
26794  
26795
26796 9 deflate_set_costs_from_codes(c, &c->codes.lens);
26797  
26798 9 ▶ } while (--num_passes_remaining);
Branch: 1 branch
BranchTrueFalseTotal
} while (--num_passes_remaining); 9 0 9
26799  
26800 9 *used_only_literals = false;
26801 9 ▶ if (MIN(only_lits_cost, static_cost) < best_true_cost) {
Branch: 1 branch
BranchTrueFalseTotal
if (MIN(only_lits_cost, static_cost) < best_true_cost) { 3 6 9
26802 3 ▶ if (only_lits_cost < static_cost) {
Branch: 1 branch
BranchTrueFalseTotal
if (only_lits_cost < static_cost) { 0 3 3
26803
26804 0 deflate_choose_all_literals(c, block_begin, block_length);
26805 0 deflate_set_costs_from_codes(c, &c->codes.lens);
26806 0 seq_.litrunlen_and_length = block_length;
26807 0 seq = &seq_;
26808 0 *used_only_literals = true;
26809 } else {
26810
26811 3 deflate_set_costs_from_codes(c, &c->static_codes.lens);
26812 3 deflate_find_min_cost_path(c, block_length, cache_ptr);
26813 }
26814 6 } else if (true_cost >=
26815 6 ▶ best_true_cost + c->p.n.min_bits_to_use_nonfinal_path) {
Branch: 1 branch
BranchTrueFalseTotal
best_true_cost + c->p.n.min_bits_to_use_nonfinal_path) { 0 6 6
26816
26817 0 c->p.n.costs = c->p.n.costs_saved;
26818 0 deflate_find_min_cost_path(c, block_length, cache_ptr);
26819 0 deflate_set_costs_from_codes(c, &c->codes.lens);
26820 }
26821 9 deflate_flush_block(c, os, block_begin, block_length, seq,
26822 is_final_block);
26823 9 }
26824  
26825 static void
26826 18 deflate_near_optimal_init_stats(struct libdeflate_compressor *c)
26827 {
26828 18 init_block_split_stats(&c->split_stats);
26829 18 memset(c->p.n.new_match_len_freqs, 0,
26830 sizeof(c->p.n.new_match_len_freqs));
26831 18 memset(c->p.n.match_len_freqs, 0, sizeof(c->p.n.match_len_freqs));
26832 18 }
26833  
26834 static void
26835 9 deflate_near_optimal_merge_stats(struct libdeflate_compressor *c)
26836 {
26837 unsigned i;
26838  
26839 9 merge_new_observations(&c->split_stats);
26840 2340 ▶ for (i = 0; i < ARRAY_LEN(c->p.n.match_len_freqs); i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < ARRAY_LEN(c->p.n.match_len_freqs); i++) { 2331 9 2340
26841 2331 c->p.n.match_len_freqs[i] += c->p.n.new_match_len_freqs[i];
26842 2331 c->p.n.new_match_len_freqs[i] = 0;
26843 }
26844 9 }
26845  
26846  
26847 static void
26848 9 deflate_near_optimal_save_stats(struct libdeflate_compressor *c)
26849 {
26850 int i;
26851  
26852 99 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) 90 9 99
26853 90 c->p.n.prev_observations[i] = c->split_stats.observations[i];
26854 9 c->p.n.prev_num_observations = c->split_stats.num_observations;
26855 9 }
26856  
26857 static void
26858 0 deflate_near_optimal_clear_old_stats(struct libdeflate_compressor *c)
26859 {
26860 int i;
26861  
26862 0 ▶ for (i = 0; i < NUM_OBSERVATION_TYPES; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < NUM_OBSERVATION_TYPES; i++) 0 0 0
26863 0 c->split_stats.observations[i] = 0;
26864 0 c->split_stats.num_observations = 0;
26865 0 memset(c->p.n.match_len_freqs, 0, sizeof(c->p.n.match_len_freqs));
26866 0 }
26867  
26868  
26869 static void
26870 9 deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
26871 const u8 *in, size_t in_nbytes,
26872 struct deflate_output_bitstream *os)
26873 {
26874 9 const u8 *in_next = in;
26875 9 const u8 *in_block_begin = in_next;
26876 9 const u8 *in_end = in_next + in_nbytes;
26877 9 const u8 *in_cur_base = in_next;
26878 9 const u8 *in_next_slide =
26879 9 in_next + MIN(in_end - in_next, MATCHFINDER_WINDOW_SIZE);
26880 9 u32 max_len = DEFLATE_MAX_MATCH_LEN;
26881 9 u32 nice_len = MIN(c->nice_match_length, max_len);
26882 9 struct lz_match *cache_ptr = c->p.n.match_cache;
26883 9 u32 next_hashes[2] = {0, 0};
26884 9 bool prev_block_used_only_literals = false;
26885  
26886 9 bt_matchfinder_init(&c->p.n.bt_mf);
26887 9 deflate_near_optimal_init_stats(c);
26888  
26889 do {
26890
26891 9 const u8 * const in_max_block_end = choose_max_block_end(
26892 in_block_begin, in_end, SOFT_MAX_BLOCK_LENGTH);
26893 9 const u8 *prev_end_block_check = NULL;
26894 9 bool change_detected = false;
26895 9 const u8 *next_observation = in_next;
26896 u32 min_len;
26897  
26898
26899 9 ▶ if (prev_block_used_only_literals)
Branch: 1 branch
BranchTrueFalseTotal
if (prev_block_used_only_literals) 0 9 9
26900 0 min_len = DEFLATE_MAX_MATCH_LEN + 1;
26901 else
26902 9 min_len = calculate_min_match_len(
26903 in_block_begin,
26904 9 in_max_block_end - in_block_begin,
26905 c->max_search_depth);
26906  
26907
26908 216 for (;;) {
26909 struct lz_match *matches;
26910 u32 best_len;
26911 225 size_t remaining = in_end - in_next;
26912  
26913
26914 225 ▶ if (in_next == in_next_slide) {
Branch: 1 branch
BranchTrueFalseTotal
if (in_next == in_next_slide) { 0 225 225
26915 0 bt_matchfinder_slide_window(&c->p.n.bt_mf);
26916 0 in_cur_base = in_next;
26917 0 in_next_slide = in_next +
26918 0 MIN(remaining, MATCHFINDER_WINDOW_SIZE);
26919 }
26920  
26921
26922 225 matches = cache_ptr;
26923 225 ▶ best_len = 0;
Branch: 1 branch
BranchTrueFalseTotal
best_len = 0; 9 216 225
26924 adjust_max_and_nice_len(&max_len, &nice_len, remaining);
26925 225 ▶ if (likely(max_len >= BT_MATCHFINDER_REQUIRED_NBYTES)) {
Branch: 1 branch
BranchTrueFalseTotal
if (likely(max_len >= BT_MATCHFINDER_REQUIRED_NBYTES)) { 225 0 225
26926 225 cache_ptr = bt_matchfinder_get_matches(
26927 &c->p.n.bt_mf,
26928 in_cur_base,
26929 in_next - in_cur_base,
26930 max_len,
26931 nice_len,
26932 c->max_search_depth,
26933 next_hashes,
26934 matches);
26935 225 ▶ if (cache_ptr > matches)
Branch: 1 branch
BranchTrueFalseTotal
if (cache_ptr > matches) 63 162 225
26936 63 best_len = cache_ptr[-1].length;
26937 }
26938 225 ▶ if (in_next >= next_observation) {
Branch: 1 branch
BranchTrueFalseTotal
if (in_next >= next_observation) { 225 0 225
26939 225 ▶ if (best_len >= min_len) {
Branch: 1 branch
BranchTrueFalseTotal
if (best_len >= min_len) { 63 162 225
26940 63 ▶ observe_match(&c->split_stats,
Branch: 1 branch
BranchTrueFalseTotal
observe_match(&c->split_stats, 63 0 63
26941 best_len);
26942 63 next_observation = in_next + best_len;
26943 63 c->p.n.new_match_len_freqs[best_len]++;
26944 } else {
26945 162 observe_literal(&c->split_stats,
26946 162 *in_next);
26947 162 next_observation = in_next + 1;
26948 }
26949 }
26950  
26951 225 cache_ptr->length = cache_ptr - matches;
26952 225 cache_ptr->offset = *in_next;
26953 225 in_next++;
26954 225 cache_ptr++;
26955  
26956
26957 225 ▶ if (best_len >= DEFLATE_MIN_MATCH_LEN &&
Branch: 1 branch
BranchTrueFalseTotal
if (best_len >= DEFLATE_MIN_MATCH_LEN && 63 162 225
26958 63 ▶ best_len >= nice_len) {
Branch: 1 branch
BranchTrueFalseTotal
best_len >= nice_len) { 63 0 63
26959 63 --best_len;
26960 do {
26961 15075 remaining = in_end - in_next;
26962 15075 ▶ if (in_next == in_next_slide) {
Branch: 1 branch
BranchTrueFalseTotal
if (in_next == in_next_slide) { 0 15075 15075
26963 0 bt_matchfinder_slide_window(
26964 &c->p.n.bt_mf);
26965 0 in_cur_base = in_next;
26966 0 in_next_slide = in_next +
26967 0 MIN(remaining,
26968 MATCHFINDER_WINDOW_SIZE);
26969 }
26970 adjust_max_and_nice_len(&max_len,
26971 &nice_len,
26972 remaining);
26973 15075 ▶ if (max_len >=
Branch: 1 branch
BranchTrueFalseTotal
if (max_len >= 15039 36 15075
26974 BT_MATCHFINDER_REQUIRED_NBYTES) {
26975 15039 bt_matchfinder_skip_byte(
26976 &c->p.n.bt_mf,
26977 in_cur_base,
26978 in_next - in_cur_base,
26979 nice_len,
26980 c->max_search_depth,
26981 next_hashes);
26982 }
26983 15075 cache_ptr->length = 0;
26984 15075 cache_ptr->offset = *in_next;
26985 15075 in_next++;
26986 15075 cache_ptr++;
26987 15075 ▶ } while (--best_len);
Branch: 1 branch
BranchTrueFalseTotal
} while (--best_len); 15012 63 15075
26988 }
26989
26990 225 ▶ if (in_next >= in_max_block_end)
Branch: 1 branch
BranchTrueFalseTotal
if (in_next >= in_max_block_end) 9 216 225
26991 9 break;
26992
26993 216 if (cache_ptr >=
26994 216 ▶ &c->p.n.match_cache[MATCH_CACHE_LENGTH])
Branch: 1 branch
BranchTrueFalseTotal
&c->p.n.match_cache[MATCH_CACHE_LENGTH]) 0 216 216
26995 0 break;
26996
26997 432 ▶ if (!ready_to_check_block(&c->split_stats,
Branch: 2 branches
BranchTrueFalseTotal
if (!ready_to_check_block(&c->split_stats, 0 216 216
if (!ready_to_check_block(&c->split_stats, 216 0 216
26998 in_block_begin, in_next,
26999 in_end))
27000 216 continue;
27001
27002 0 ▶ if (do_end_block_check(&c->split_stats,
Branch: 1 branch
BranchTrueFalseTotal
if (do_end_block_check(&c->split_stats, 0 0 0
27003 0 in_next - in_block_begin)) {
27004 0 change_detected = true;
27005 0 break;
27006 }
27007
27008 0 deflate_near_optimal_merge_stats(c);
27009 0 prev_end_block_check = in_next;
27010 }
27011
27012 9 ▶ if (change_detected && prev_end_block_check != NULL) {
Branch: 2 branches
BranchTrueFalseTotal
if (change_detected && prev_end_block_check != NULL) { 0 9 9
if (change_detected && prev_end_block_check != NULL) { 0 0 0
27013
27014 0 struct lz_match *orig_cache_ptr = cache_ptr;
27015 0 const u8 *in_block_end = prev_end_block_check;
27016 0 u32 block_length = in_block_end - in_block_begin;
27017 0 bool is_first = (in_block_begin == in);
27018 0 bool is_final = false;
27019 0 u32 num_bytes_to_rewind = in_next - in_block_end;
27020 size_t cache_len_rewound;
27021  
27022
27023 do {
27024 0 cache_ptr--;
27025 0 cache_ptr -= cache_ptr->length;
27026 0 ▶ } while (--num_bytes_to_rewind);
Branch: 1 branch
BranchTrueFalseTotal
} while (--num_bytes_to_rewind); 0 0 0
27027 0 cache_len_rewound = orig_cache_ptr - cache_ptr;
27028  
27029 0 deflate_optimize_and_flush_block(
27030 c, os, in_block_begin,
27031 block_length, cache_ptr,
27032 is_first, is_final,
27033 &prev_block_used_only_literals);
27034 0 memmove(c->p.n.match_cache, cache_ptr,
27035 cache_len_rewound * sizeof(*cache_ptr));
27036 0 cache_ptr = &c->p.n.match_cache[cache_len_rewound];
27037 0 deflate_near_optimal_save_stats(c);
27038
27039 0 deflate_near_optimal_clear_old_stats(c);
27040 0 in_block_begin = in_block_end;
27041 } else {
27042
27043 9 u32 block_length = in_next - in_block_begin;
27044 9 bool is_first = (in_block_begin == in);
27045 9 bool is_final = (in_next == in_end);
27046  
27047 9 deflate_near_optimal_merge_stats(c);
27048 9 deflate_optimize_and_flush_block(
27049 c, os, in_block_begin,
27050 block_length, cache_ptr,
27051 is_first, is_final,
27052 &prev_block_used_only_literals);
27053 9 cache_ptr = &c->p.n.match_cache[0];
27054 9 deflate_near_optimal_save_stats(c);
27055 9 deflate_near_optimal_init_stats(c);
27056 9 in_block_begin = in_next;
27057 }
27058 9 ▶ } while (in_next != in_end && !os->overflow);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next != in_end && !os->overflow); 0 9 9
} while (in_next != in_end && !os->overflow); 0 0 0
27059 9 }
27060  
27061  
27062 static void
27063 9 deflate_init_offset_slot_full(struct libdeflate_compressor *c)
27064 {
27065 u32 offset_slot;
27066 u32 offset;
27067 u32 offset_end;
27068  
27069 279 ▶ for (offset_slot = 0; offset_slot < ARRAY_LEN(deflate_offset_slot_base);
Branch: 1 branch
BranchTrueFalseTotal
for (offset_slot = 0; offset_slot < ARRAY_LEN(deflate_offset_slot_base); 270 9 279
27070 270 offset_slot++) {
27071 270 offset = deflate_offset_slot_base[offset_slot];
27072 270 offset_end = offset +
27073 270 (1 << deflate_extra_offset_bits[offset_slot]);
27074 do {
27075 294912 c->p.n.offset_slot_full[offset] = offset_slot;
27076 294912 ▶ } while (++offset != offset_end);
Branch: 1 branch
BranchTrueFalseTotal
} while (++offset != offset_end); 294642 270 294912
27077 }
27078 9 }
27079  
27080 #endif
27081  
27082 LIBDEFLATEAPI struct libdeflate_compressor *
27083 37 libdeflate_alloc_compressor_ex(int compression_level,
27084 const struct libdeflate_options *options)
27085 {
27086 struct libdeflate_compressor *c;
27087 37 size_t size = offsetof(struct libdeflate_compressor, p);
27088  
27089 check_buildtime_parameters();
27090  
27091
27092 37 ▶ if (options->sizeof_options != sizeof(*options))
Branch: 1 branch
BranchTrueFalseTotal
if (options->sizeof_options != sizeof(*options)) 0 37 37
27093 0 return NULL;
27094  
27095
27096 37 ▶ if (compression_level == -1)
Branch: 1 branch
BranchTrueFalseTotal
if (compression_level == -1) 0 37 37
27097 0 compression_level = 6;
27098  
27099 37 ▶ if (compression_level < 0 || compression_level > 12)
Branch: 2 branches
BranchTrueFalseTotal
if (compression_level < 0 || compression_level > 12) 37 0 37
if (compression_level < 0 || compression_level > 12) 0 37 37
27100 0 return NULL;
27101  
27102 #if SUPPORT_NEAR_OPTIMAL_PARSING
27103 37 ▶ if (compression_level >= 10)
Branch: 1 branch
BranchTrueFalseTotal
if (compression_level >= 10) 9 28 37
27104 9 size += sizeof(c->p.n);
27105 else
27106 #endif
27107 {
27108 28 ▶ if (compression_level >= 2)
Branch: 1 branch
BranchTrueFalseTotal
if (compression_level >= 2) 25 3 28
27109 25 size += sizeof(c->p.g);
27110 3 ▶ else if (compression_level == 1)
Branch: 1 branch
BranchTrueFalseTotal
else if (compression_level == 1) 3 0 3
27111 3 size += sizeof(c->p.f);
27112 }
27113  
27114 37 ▶ c = libdeflate_aligned_malloc(options->malloc_func ?
Branch: 1 branch
BranchTrueFalseTotal
c = libdeflate_aligned_malloc(options->malloc_func ? 0 37 37
27115 options->malloc_func :
27116 libdeflate_default_malloc_func,
27117 MATCHFINDER_MEM_ALIGNMENT, size);
27118 37 ▶ if (!c)
Branch: 1 branch
BranchTrueFalseTotal
if (!c) 0 37 37
27119 0 return NULL;
27120 74 c->free_func = options->free_func ?
27121 37 ▶ options->free_func : libdeflate_default_free_func;
Branch: 1 branch
BranchTrueFalseTotal
options->free_func : libdeflate_default_free_func; 0 37 37
27122  
27123 37 c->compression_level = compression_level;
27124  
27125
27126 37 c->max_passthrough_size = 55 - (compression_level * 4);
27127  
27128 37 switch (compression_level) {
27129 0 case 0:
27130 0 c->max_passthrough_size = SIZE_MAX;
27131 0 c->impl = NULL;
27132 0 break;
27133 3 case 1:
27134 3 c->impl = deflate_compress_fastest;
27135
27136 3 c->nice_match_length = 32;
27137 3 break;
27138 3 case 2:
27139 3 c->impl = deflate_compress_greedy;
27140 3 c->max_search_depth = 6;
27141 3 c->nice_match_length = 10;
27142 3 break;
27143 3 case 3:
27144 3 c->impl = deflate_compress_greedy;
27145 3 c->max_search_depth = 12;
27146 3 c->nice_match_length = 14;
27147 3 break;
27148 3 case 4:
27149 3 c->impl = deflate_compress_greedy;
27150 3 c->max_search_depth = 16;
27151 3 c->nice_match_length = 30;
27152 3 break;
27153 3 case 5:
27154 3 c->impl = deflate_compress_lazy;
27155 3 c->max_search_depth = 16;
27156 3 c->nice_match_length = 30;
27157 3 break;
27158 4 case 6:
27159 4 c->impl = deflate_compress_lazy;
27160 4 c->max_search_depth = 35;
27161 4 c->nice_match_length = 65;
27162 4 break;
27163 3 case 7:
27164 3 c->impl = deflate_compress_lazy;
27165 3 c->max_search_depth = 100;
27166 3 c->nice_match_length = 130;
27167 3 break;
27168 3 case 8:
27169 3 c->impl = deflate_compress_lazy2;
27170 3 c->max_search_depth = 300;
27171 3 c->nice_match_length = DEFLATE_MAX_MATCH_LEN;
27172 3 break;
27173 3 case 9:
27174 #if !SUPPORT_NEAR_OPTIMAL_PARSING
27175 default:
27176 #endif
27177 3 c->impl = deflate_compress_lazy2;
27178 3 c->max_search_depth = 600;
27179 3 c->nice_match_length = DEFLATE_MAX_MATCH_LEN;
27180 3 break;
27181 #if SUPPORT_NEAR_OPTIMAL_PARSING
27182 3 case 10:
27183 3 c->impl = deflate_compress_near_optimal;
27184 3 c->max_search_depth = 35;
27185 3 c->nice_match_length = 75;
27186 3 c->p.n.max_optim_passes = 2;
27187 3 c->p.n.min_improvement_to_continue = 32;
27188 3 c->p.n.min_bits_to_use_nonfinal_path = 32;
27189 3 c->p.n.max_len_to_optimize_static_block = 0;
27190 3 deflate_init_offset_slot_full(c);
27191 3 break;
27192 3 case 11:
27193 3 c->impl = deflate_compress_near_optimal;
27194 3 c->max_search_depth = 100;
27195 3 c->nice_match_length = 150;
27196 3 c->p.n.max_optim_passes = 4;
27197 3 c->p.n.min_improvement_to_continue = 16;
27198 3 c->p.n.min_bits_to_use_nonfinal_path = 16;
27199 3 c->p.n.max_len_to_optimize_static_block = 1000;
27200 3 deflate_init_offset_slot_full(c);
27201 3 break;
27202 3 case 12:
27203 default:
27204 3 c->impl = deflate_compress_near_optimal;
27205 3 c->max_search_depth = 300;
27206 3 c->nice_match_length = DEFLATE_MAX_MATCH_LEN;
27207 3 c->p.n.max_optim_passes = 10;
27208 3 c->p.n.min_improvement_to_continue = 1;
27209 3 c->p.n.min_bits_to_use_nonfinal_path = 1;
27210 3 c->p.n.max_len_to_optimize_static_block = 10000;
27211 3 deflate_init_offset_slot_full(c);
27212 3 break;
27213 #endif
27214 }
27215  
27216 37 deflate_init_static_codes(c);
27217  
27218 37 return c;
27219 }
27220  
27221  
27222 LIBDEFLATEAPI struct libdeflate_compressor *
27223 37 libdeflate_alloc_compressor(int compression_level)
27224 {
27225 static const struct libdeflate_options defaults = {
27226 .sizeof_options = sizeof(defaults),
27227 };
27228 37 return libdeflate_alloc_compressor_ex(compression_level, &defaults);
27229 }
27230  
27231 LIBDEFLATEAPI size_t
27232 37 libdeflate_deflate_compress(struct libdeflate_compressor *c,
27233 const void *in, size_t in_nbytes,
27234 void *out, size_t out_nbytes_avail)
27235 {
27236 struct deflate_output_bitstream os;
27237  
27238
27239 37 ▶ if (unlikely(in_nbytes <= c->max_passthrough_size))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(in_nbytes <= c->max_passthrough_size)) 0 37 37
27240 0 return deflate_compress_none(in, in_nbytes,
27241 out, out_nbytes_avail);
27242  
27243
27244 37 os.bitbuf = 0;
27245 37 os.bitcount = 0;
27246 37 os.next = out;
27247 37 os.end = os.next + out_nbytes_avail;
27248 37 os.overflow = false;
27249  
27250
27251 37 (*c->impl)(c, in, in_nbytes, &os);
27252  
27253
27254 37 ▶ if (os.overflow)
Branch: 1 branch
BranchTrueFalseTotal
if (os.overflow) 0 37 37
27255 0 return 0;
27256  
27257
27258 37 ASSERT(os.bitcount <= 7);
27259 37 ▶ if (os.bitcount) {
Branch: 1 branch
BranchTrueFalseTotal
if (os.bitcount) { 37 0 37
27260 37 ASSERT(os.next < os.end);
27261 37 *os.next++ = os.bitbuf;
27262 }
27263  
27264
27265 37 return os.next - (u8 *)out;
27266 }
27267  
27268 LIBDEFLATEAPI void
27269 37 libdeflate_free_compressor(struct libdeflate_compressor *c)
27270 {
27271 37 ▶ if (c)
Branch: 1 branch
BranchTrueFalseTotal
if (c) 37 0 37
27272 37 libdeflate_aligned_free(c->free_func, c);
27273 37 }
27274  
27275 unsigned int
27276 25 libdeflate_get_compression_level(struct libdeflate_compressor *c)
27277 {
27278 25 return c->compression_level;
27279 }
27280  
27281 LIBDEFLATEAPI size_t
27282 37 libdeflate_deflate_compress_bound(struct libdeflate_compressor *c,
27283 size_t in_nbytes)
27284 {
27285 size_t max_blocks;
27286  
27287
27288  
27289
27290 STATIC_ASSERT(2 * MIN_BLOCK_LENGTH <= UINT16_MAX);
27291 37 ▶ max_blocks = MAX(DIV_ROUND_UP(in_nbytes, MIN_BLOCK_LENGTH), 1);
Branch: 1 branch
BranchTrueFalseTotal
max_blocks = MAX(DIV_ROUND_UP(in_nbytes, MIN_BLOCK_LENGTH), 1); 37 0 37
27292  
27293
27294 37 return (5 * max_blocks) + in_nbytes;
27295 }
27296 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/deflate_decompress.c */
27297  
27298  
27299 /* #include "lib_common.h" */
27300  
27301  
27302 #ifndef LIB_LIB_COMMON_H
27303 #define LIB_LIB_COMMON_H
27304  
27305 #ifdef LIBDEFLATE_H
27306
27307 # error "lib_common.h must always be included before libdeflate.h"
27308 #endif
27309  
27310 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
27311 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
27312 #elif defined(__GNUC__)
27313 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
27314 #else
27315 # define LIBDEFLATE_EXPORT_SYM
27316 #endif
27317  
27318  
27319 #if defined(__GNUC__) && defined(__i386__)
27320 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
27321 #else
27322 # define LIBDEFLATE_ALIGN_STACK
27323 #endif
27324  
27325 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
27326  
27327 /* #include "../common_defs.h" */
27328  
27329  
27330 #ifndef COMMON_DEFS_H
27331 #define COMMON_DEFS_H
27332  
27333 /* #include "libdeflate.h" */
27334  
27335  
27336 #ifndef LIBDEFLATE_H
27337 #define LIBDEFLATE_H
27338  
27339 #include <stddef.h>
27340 #include <stdint.h>
27341  
27342 #ifdef __cplusplus
27343 extern "C" {
27344 #endif
27345  
27346 #define LIBDEFLATE_VERSION_MAJOR 1
27347 #define LIBDEFLATE_VERSION_MINOR 26
27348 #define LIBDEFLATE_VERSION_STRING "1.26"
27349  
27350  
27351 #ifndef LIBDEFLATEAPI
27352 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
27353 # define LIBDEFLATEAPI __declspec(dllimport)
27354 # else
27355 # define LIBDEFLATEAPI
27356 # endif
27357 #endif
27358  
27359  
27360  
27361  
27362  
27363 struct libdeflate_compressor;
27364 struct libdeflate_options;
27365  
27366  
27367 LIBDEFLATEAPI struct libdeflate_compressor *
27368 libdeflate_alloc_compressor(int compression_level);
27369  
27370  
27371 LIBDEFLATEAPI struct libdeflate_compressor *
27372 libdeflate_alloc_compressor_ex(int compression_level,
27373 const struct libdeflate_options *options);
27374  
27375  
27376 LIBDEFLATEAPI size_t
27377 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
27378 const void *in, size_t in_nbytes,
27379 void *out, size_t out_nbytes_avail);
27380  
27381  
27382 LIBDEFLATEAPI size_t
27383 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
27384 size_t in_nbytes);
27385  
27386  
27387 LIBDEFLATEAPI size_t
27388 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
27389 const void *in, size_t in_nbytes,
27390 void *out, size_t out_nbytes_avail);
27391  
27392  
27393 LIBDEFLATEAPI size_t
27394 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
27395 size_t in_nbytes);
27396  
27397  
27398 LIBDEFLATEAPI size_t
27399 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
27400 const void *in, size_t in_nbytes,
27401 void *out, size_t out_nbytes_avail);
27402  
27403  
27404 LIBDEFLATEAPI size_t
27405 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
27406 size_t in_nbytes);
27407  
27408  
27409 LIBDEFLATEAPI void
27410 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
27411  
27412  
27413  
27414  
27415  
27416 struct libdeflate_decompressor;
27417 struct libdeflate_options;
27418  
27419  
27420 LIBDEFLATEAPI struct libdeflate_decompressor *
27421 libdeflate_alloc_decompressor(void);
27422  
27423  
27424 LIBDEFLATEAPI struct libdeflate_decompressor *
27425 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
27426  
27427  
27428 enum libdeflate_result {
27429
27430 LIBDEFLATE_SUCCESS = 0,
27431  
27432
27433 LIBDEFLATE_BAD_DATA = 1,
27434  
27435
27436 LIBDEFLATE_SHORT_OUTPUT = 2,
27437  
27438
27439 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
27440 };
27441  
27442  
27443 LIBDEFLATEAPI enum libdeflate_result
27444 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
27445 const void *in, size_t in_nbytes,
27446 void *out, size_t out_nbytes_avail,
27447 size_t *actual_out_nbytes_ret);
27448  
27449  
27450 LIBDEFLATEAPI enum libdeflate_result
27451 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
27452 const void *in, size_t in_nbytes,
27453 void *out, size_t out_nbytes_avail,
27454 size_t *actual_in_nbytes_ret,
27455 size_t *actual_out_nbytes_ret);
27456  
27457  
27458 LIBDEFLATEAPI enum libdeflate_result
27459 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
27460 const void *in, size_t in_nbytes,
27461 void *out, size_t out_nbytes_avail,
27462 size_t *actual_out_nbytes_ret);
27463  
27464  
27465 LIBDEFLATEAPI enum libdeflate_result
27466 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
27467 const void *in, size_t in_nbytes,
27468 void *out, size_t out_nbytes_avail,
27469 size_t *actual_in_nbytes_ret,
27470 size_t *actual_out_nbytes_ret);
27471  
27472  
27473 LIBDEFLATEAPI enum libdeflate_result
27474 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
27475 const void *in, size_t in_nbytes,
27476 void *out, size_t out_nbytes_avail,
27477 size_t *actual_out_nbytes_ret);
27478  
27479  
27480 LIBDEFLATEAPI enum libdeflate_result
27481 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
27482 const void *in, size_t in_nbytes,
27483 void *out, size_t out_nbytes_avail,
27484 size_t *actual_in_nbytes_ret,
27485 size_t *actual_out_nbytes_ret);
27486  
27487  
27488 LIBDEFLATEAPI void
27489 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
27490  
27491  
27492  
27493  
27494  
27495  
27496 LIBDEFLATEAPI uint32_t
27497 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
27498  
27499  
27500  
27501 LIBDEFLATEAPI uint32_t
27502 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
27503  
27504  
27505  
27506  
27507  
27508  
27509 LIBDEFLATEAPI void
27510 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
27511 void (*free_func)(void *));
27512  
27513  
27514 struct libdeflate_options {
27515  
27516
27517 size_t sizeof_options;
27518  
27519
27520 void *(*malloc_func)(size_t);
27521 void (*free_func)(void *);
27522 };
27523  
27524 #ifdef __cplusplus
27525 }
27526 #endif
27527  
27528 #endif
27529  
27530  
27531 #include <stdbool.h>
27532 #include <stddef.h>
27533 #include <stdint.h>
27534 #ifdef _MSC_VER
27535 # include <intrin.h>
27536 # include <stdlib.h>
27537
27538
27539 # pragma warning(disable : 4146)
27540
27541 # pragma warning(disable : 4018)
27542 # pragma warning(disable : 4244)
27543 # pragma warning(disable : 4267)
27544 # pragma warning(disable : 4310)
27545
27546 # pragma warning(disable : 4100)
27547 # pragma warning(disable : 4127)
27548 # pragma warning(disable : 4189)
27549 # pragma warning(disable : 4232)
27550 # pragma warning(disable : 4245)
27551 # pragma warning(disable : 4295)
27552 #endif
27553 #ifndef FREESTANDING
27554 # include <string.h>
27555 #endif
27556  
27557  
27558  
27559  
27560  
27561  
27562 #undef ARCH_X86_64
27563 #undef ARCH_X86_32
27564 #undef ARCH_ARM64
27565 #undef ARCH_ARM32
27566 #undef ARCH_RISCV
27567 #ifdef _MSC_VER
27568
27569 # if defined(_M_X64) && !defined(_M_ARM64EC)
27570 # define ARCH_X86_64
27571 # elif defined(_M_IX86)
27572 # define ARCH_X86_32
27573 # elif defined(_M_ARM64)
27574 # define ARCH_ARM64
27575 # elif defined(_M_ARM)
27576 # define ARCH_ARM32
27577 # endif
27578 #else
27579 # if defined(__x86_64__)
27580 # define ARCH_X86_64
27581 # elif defined(__i386__)
27582 # define ARCH_X86_32
27583 # elif defined(__aarch64__)
27584 # define ARCH_ARM64
27585 # elif defined(__arm__)
27586 # define ARCH_ARM32
27587 # elif defined(__riscv)
27588 # define ARCH_RISCV
27589 # endif
27590 #endif
27591  
27592  
27593  
27594  
27595  
27596  
27597 typedef uint8_t u8;
27598 typedef uint16_t u16;
27599 typedef uint32_t u32;
27600 typedef uint64_t u64;
27601 typedef int8_t s8;
27602 typedef int16_t s16;
27603 typedef int32_t s32;
27604 typedef int64_t s64;
27605  
27606  
27607 #ifdef _MSC_VER
27608 # ifdef _WIN64
27609 typedef long long ssize_t;
27610 # else
27611 typedef long ssize_t;
27612 # endif
27613 #endif
27614  
27615  
27616 typedef size_t machine_word_t;
27617  
27618  
27619 #define WORDBYTES ((int)sizeof(machine_word_t))
27620  
27621  
27622 #define WORDBITS (8 * WORDBYTES)
27623  
27624  
27625  
27626  
27627  
27628  
27629 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
27630 # define GCC_PREREQ(major, minor) \
27631 (__GNUC__ > (major) || \
27632 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
27633 # if !GCC_PREREQ(4, 9)
27634 # error "gcc versions older than 4.9 are no longer supported"
27635 # endif
27636 #else
27637 # define GCC_PREREQ(major, minor) 0
27638 #endif
27639 #ifdef __clang__
27640 # ifdef __apple_build_version__
27641 # define CLANG_PREREQ(major, minor, apple_version) \
27642 (__apple_build_version__ >= (apple_version))
27643 # else
27644 # define CLANG_PREREQ(major, minor, apple_version) \
27645 (__clang_major__ > (major) || \
27646 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
27647 # endif
27648 # if !CLANG_PREREQ(3, 9, 8000000)
27649 # error "clang versions older than 3.9 are no longer supported"
27650 # endif
27651 #else
27652 # define CLANG_PREREQ(major, minor, apple_version) 0
27653 #endif
27654 #ifdef _MSC_VER
27655 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
27656 # if !MSVC_PREREQ(1928)
27657 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
27658 # endif
27659 #else
27660 # define MSVC_PREREQ(version) 0
27661 #endif
27662  
27663  
27664 #ifndef __has_attribute
27665 # define __has_attribute(attribute) 0
27666 #endif
27667  
27668  
27669 #ifndef __has_builtin
27670 # define __has_builtin(builtin) 0
27671 #endif
27672  
27673  
27674 #ifdef _MSC_VER
27675 # define inline __inline
27676 #endif
27677  
27678  
27679 #if defined(__GNUC__) || __has_attribute(always_inline)
27680 # define forceinline inline __attribute__((always_inline))
27681 #elif defined(_MSC_VER)
27682 # define forceinline __forceinline
27683 #else
27684 # define forceinline inline
27685 #endif
27686  
27687  
27688 #if defined(__GNUC__) || __has_attribute(unused)
27689 # define MAYBE_UNUSED __attribute__((unused))
27690 #else
27691 # define MAYBE_UNUSED
27692 #endif
27693  
27694  
27695 #if defined(__GNUC__) || __has_attribute(noreturn)
27696 # define NORETURN __attribute__((noreturn))
27697 #else
27698 # define NORETURN
27699 #endif
27700  
27701  
27702 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
27703 # define likely(expr) __builtin_expect(!!(expr), 1)
27704 #else
27705 # define likely(expr) (expr)
27706 #endif
27707  
27708  
27709 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
27710 # define unlikely(expr) __builtin_expect(!!(expr), 0)
27711 #else
27712 # define unlikely(expr) (expr)
27713 #endif
27714  
27715  
27716 #undef prefetchr
27717 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
27718 # define prefetchr(addr) __builtin_prefetch((addr), 0)
27719 #elif defined(_MSC_VER)
27720 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
27721 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
27722 # elif defined(ARCH_ARM64)
27723 # define prefetchr(addr) __prefetch2((addr), 0x00 )
27724 # elif defined(ARCH_ARM32)
27725 # define prefetchr(addr) __prefetch(addr)
27726 # endif
27727 #endif
27728 #ifndef prefetchr
27729 # define prefetchr(addr)
27730 #endif
27731  
27732  
27733 #undef prefetchw
27734 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
27735 # define prefetchw(addr) __builtin_prefetch((addr), 1)
27736 #elif defined(_MSC_VER)
27737 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
27738 # define prefetchw(addr) _m_prefetchw(addr)
27739 # elif defined(ARCH_ARM64)
27740 # define prefetchw(addr) __prefetch2((addr), 0x10 )
27741 # elif defined(ARCH_ARM32)
27742 # define prefetchw(addr) __prefetchw(addr)
27743 # endif
27744 #endif
27745 #ifndef prefetchw
27746 # define prefetchw(addr)
27747 #endif
27748  
27749  
27750 #undef _aligned_attribute
27751 #if defined(__GNUC__) || __has_attribute(aligned)
27752 # define _aligned_attribute(n) __attribute__((aligned(n)))
27753 #elif defined(_MSC_VER)
27754 # define _aligned_attribute(n) __declspec(align(n))
27755 #endif
27756  
27757  
27758 #if defined(__GNUC__) || __has_attribute(target)
27759 # define _target_attribute(attrs) __attribute__((target(attrs)))
27760 #else
27761 # define _target_attribute(attrs)
27762 #endif
27763  
27764  
27765  
27766  
27767  
27768 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
27769 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
27770 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
27771 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
27772 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
27773 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
27774 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
27775  
27776  
27777  
27778  
27779  
27780  
27781 #if defined(__BYTE_ORDER__)
27782 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
27783 #elif defined(_MSC_VER)
27784 # define CPU_IS_LITTLE_ENDIAN() true
27785 #else
27786 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
27787 {
27788 union {
27789 u32 w;
27790 u8 b;
27791 } u;
27792  
27793 u.w = 1;
27794 return u.b;
27795 }
27796 #endif
27797  
27798  
27799 static forceinline u16 bswap16(u16 v)
27800 {
27801 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
27802 return __builtin_bswap16(v);
27803 #elif defined(_MSC_VER)
27804 return _byteswap_ushort(v);
27805 #else
27806 return (v << 8) | (v >> 8);
27807 #endif
27808 }
27809  
27810  
27811 static forceinline u32 bswap32(u32 v)
27812 {
27813 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
27814 return __builtin_bswap32(v);
27815 #elif defined(_MSC_VER)
27816 return _byteswap_ulong(v);
27817 #else
27818 return ((v & 0x000000FF) << 24) |
27819 ((v & 0x0000FF00) << 8) |
27820 ((v & 0x00FF0000) >> 8) |
27821 ((v & 0xFF000000) >> 24);
27822 #endif
27823 }
27824  
27825  
27826 static forceinline u64 bswap64(u64 v)
27827 {
27828 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
27829 return __builtin_bswap64(v);
27830 #elif defined(_MSC_VER)
27831 return _byteswap_uint64(v);
27832 #else
27833 return ((v & 0x00000000000000FF) << 56) |
27834 ((v & 0x000000000000FF00) << 40) |
27835 ((v & 0x0000000000FF0000) << 24) |
27836 ((v & 0x00000000FF000000) << 8) |
27837 ((v & 0x000000FF00000000) >> 8) |
27838 ((v & 0x0000FF0000000000) >> 24) |
27839 ((v & 0x00FF000000000000) >> 40) |
27840 ((v & 0xFF00000000000000) >> 56);
27841 #endif
27842 }
27843  
27844 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
27845 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
27846 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
27847 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
27848 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
27849 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
27850  
27851  
27852  
27853  
27854  
27855  
27856 #if (defined(__GNUC__) || defined(__clang__)) && \
27857 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
27858 defined(__ARM_FEATURE_UNALIGNED) || \
27859 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
27860 defined(__riscv_misaligned_fast) || \
27861 defined(__wasm__))
27862 # define UNALIGNED_ACCESS_IS_FAST 1
27863 #elif defined(_MSC_VER)
27864 # define UNALIGNED_ACCESS_IS_FAST 1
27865 #else
27866 # define UNALIGNED_ACCESS_IS_FAST 0
27867 #endif
27868  
27869  
27870  
27871 #ifdef FREESTANDING
27872 # define MEMCOPY __builtin_memcpy
27873 #else
27874 # define MEMCOPY memcpy
27875 #endif
27876  
27877  
27878  
27879 #define DEFINE_UNALIGNED_TYPE(type) \
27880 static forceinline type \
27881 load_##type##_unaligned(const void *p) \
27882 { \
27883 type v; \
27884 \
27885 MEMCOPY(&v, p, sizeof(v)); \
27886 return v; \
27887 } \
27888 \
27889 static forceinline void \
27890 store_##type##_unaligned(type v, void *p) \
27891 { \
27892 MEMCOPY(p, &v, sizeof(v)); \
27893 }
27894  
27895 DEFINE_UNALIGNED_TYPE(u16)
27896 DEFINE_UNALIGNED_TYPE(u32)
27897 DEFINE_UNALIGNED_TYPE(u64)
27898 DEFINE_UNALIGNED_TYPE(machine_word_t)
27899  
27900 #undef MEMCOPY
27901  
27902 #define load_word_unaligned load_machine_word_t_unaligned
27903 #define store_word_unaligned store_machine_word_t_unaligned
27904  
27905  
27906  
27907 static forceinline u16
27908 get_unaligned_le16(const u8 *p)
27909 {
27910 if (UNALIGNED_ACCESS_IS_FAST)
27911 return le16_bswap(load_u16_unaligned(p));
27912 else
27913 return ((u16)p[1] << 8) | p[0];
27914 }
27915  
27916 static forceinline u16
27917 get_unaligned_be16(const u8 *p)
27918 {
27919 if (UNALIGNED_ACCESS_IS_FAST)
27920 return be16_bswap(load_u16_unaligned(p));
27921 else
27922 return ((u16)p[0] << 8) | p[1];
27923 }
27924  
27925 static forceinline u32
27926 get_unaligned_le32(const u8 *p)
27927 {
27928 if (UNALIGNED_ACCESS_IS_FAST)
27929 return le32_bswap(load_u32_unaligned(p));
27930 else
27931 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
27932 ((u32)p[1] << 8) | p[0];
27933 }
27934  
27935 static forceinline u32
27936 get_unaligned_be32(const u8 *p)
27937 {
27938 if (UNALIGNED_ACCESS_IS_FAST)
27939 return be32_bswap(load_u32_unaligned(p));
27940 else
27941 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
27942 ((u32)p[2] << 8) | p[3];
27943 }
27944  
27945 static forceinline u64
27946 get_unaligned_le64(const u8 *p)
27947 {
27948 if (UNALIGNED_ACCESS_IS_FAST)
27949 return le64_bswap(load_u64_unaligned(p));
27950 else
27951 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
27952 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
27953 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
27954 ((u64)p[1] << 8) | p[0];
27955 }
27956  
27957 static forceinline machine_word_t
27958 get_unaligned_leword(const u8 *p)
27959 {
27960 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
27961 if (WORDBITS == 32)
27962 return get_unaligned_le32(p);
27963 else
27964 return get_unaligned_le64(p);
27965 }
27966  
27967  
27968  
27969 static forceinline void
27970 put_unaligned_le16(u16 v, u8 *p)
27971 {
27972 if (UNALIGNED_ACCESS_IS_FAST) {
27973 store_u16_unaligned(le16_bswap(v), p);
27974 } else {
27975 p[0] = (u8)(v >> 0);
27976 p[1] = (u8)(v >> 8);
27977 }
27978 }
27979  
27980 static forceinline void
27981 put_unaligned_be16(u16 v, u8 *p)
27982 {
27983 if (UNALIGNED_ACCESS_IS_FAST) {
27984 store_u16_unaligned(be16_bswap(v), p);
27985 } else {
27986 p[0] = (u8)(v >> 8);
27987 p[1] = (u8)(v >> 0);
27988 }
27989 }
27990  
27991 static forceinline void
27992 put_unaligned_le32(u32 v, u8 *p)
27993 {
27994 if (UNALIGNED_ACCESS_IS_FAST) {
27995 store_u32_unaligned(le32_bswap(v), p);
27996 } else {
27997 p[0] = (u8)(v >> 0);
27998 p[1] = (u8)(v >> 8);
27999 p[2] = (u8)(v >> 16);
28000 p[3] = (u8)(v >> 24);
28001 }
28002 }
28003  
28004 static forceinline void
28005 put_unaligned_be32(u32 v, u8 *p)
28006 {
28007 if (UNALIGNED_ACCESS_IS_FAST) {
28008 store_u32_unaligned(be32_bswap(v), p);
28009 } else {
28010 p[0] = (u8)(v >> 24);
28011 p[1] = (u8)(v >> 16);
28012 p[2] = (u8)(v >> 8);
28013 p[3] = (u8)(v >> 0);
28014 }
28015 }
28016  
28017 static forceinline void
28018 put_unaligned_le64(u64 v, u8 *p)
28019 {
28020 if (UNALIGNED_ACCESS_IS_FAST) {
28021 store_u64_unaligned(le64_bswap(v), p);
28022 } else {
28023 p[0] = (u8)(v >> 0);
28024 p[1] = (u8)(v >> 8);
28025 p[2] = (u8)(v >> 16);
28026 p[3] = (u8)(v >> 24);
28027 p[4] = (u8)(v >> 32);
28028 p[5] = (u8)(v >> 40);
28029 p[6] = (u8)(v >> 48);
28030 p[7] = (u8)(v >> 56);
28031 }
28032 }
28033  
28034 static forceinline void
28035 put_unaligned_leword(machine_word_t v, u8 *p)
28036 {
28037 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
28038 if (WORDBITS == 32)
28039 put_unaligned_le32(v, p);
28040 else
28041 put_unaligned_le64(v, p);
28042 }
28043  
28044  
28045  
28046  
28047  
28048  
28049  
28050 static forceinline unsigned
28051 bsr32(u32 v)
28052 {
28053 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
28054 return 31 - __builtin_clz(v);
28055 #elif defined(_MSC_VER)
28056 unsigned long i;
28057  
28058 _BitScanReverse(&i, v);
28059 return i;
28060 #else
28061 unsigned i = 0;
28062  
28063 while ((v >>= 1) != 0)
28064 i++;
28065 return i;
28066 #endif
28067 }
28068  
28069 static forceinline unsigned
28070 bsr64(u64 v)
28071 {
28072 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
28073 return 63 - __builtin_clzll(v);
28074 #elif defined(_MSC_VER) && defined(_WIN64)
28075 unsigned long i;
28076  
28077 _BitScanReverse64(&i, v);
28078 return i;
28079 #else
28080 unsigned i = 0;
28081  
28082 while ((v >>= 1) != 0)
28083 i++;
28084 return i;
28085 #endif
28086 }
28087  
28088 static forceinline unsigned
28089 bsrw(machine_word_t v)
28090 {
28091 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
28092 if (WORDBITS == 32)
28093 return bsr32(v);
28094 else
28095 return bsr64(v);
28096 }
28097  
28098  
28099  
28100 static forceinline unsigned
28101 bsf32(u32 v)
28102 {
28103 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
28104 return __builtin_ctz(v);
28105 #elif defined(_MSC_VER)
28106 unsigned long i;
28107  
28108 _BitScanForward(&i, v);
28109 return i;
28110 #else
28111 unsigned i = 0;
28112  
28113 for (; (v & 1) == 0; v >>= 1)
28114 i++;
28115 return i;
28116 #endif
28117 }
28118  
28119 static forceinline unsigned
28120 bsf64(u64 v)
28121 {
28122 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
28123 return __builtin_ctzll(v);
28124 #elif defined(_MSC_VER) && defined(_WIN64)
28125 unsigned long i;
28126  
28127 _BitScanForward64(&i, v);
28128 return i;
28129 #else
28130 unsigned i = 0;
28131  
28132 for (; (v & 1) == 0; v >>= 1)
28133 i++;
28134 return i;
28135 #endif
28136 }
28137  
28138 static forceinline unsigned
28139 bsfw(machine_word_t v)
28140 {
28141 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
28142 if (WORDBITS == 32)
28143 return bsf32(v);
28144 else
28145 return bsf64(v);
28146 }
28147  
28148  
28149 #undef rbit32
28150 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
28151 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
28152 static forceinline u32
28153 rbit32(u32 v)
28154 {
28155 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
28156 return v;
28157 }
28158 #define rbit32 rbit32
28159 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
28160 static forceinline u32
28161 rbit32(u32 v)
28162 {
28163 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
28164 return v;
28165 }
28166 #define rbit32 rbit32
28167 #endif
28168  
28169 #endif
28170  
28171  
28172 typedef void *(*malloc_func_t)(size_t);
28173 typedef void (*free_func_t)(void *);
28174  
28175 extern malloc_func_t libdeflate_default_malloc_func;
28176 extern free_func_t libdeflate_default_free_func;
28177  
28178 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
28179 size_t alignment, size_t size);
28180 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
28181  
28182 #ifdef FREESTANDING
28183  
28184 void *memset(void *s, int c, size_t n);
28185 #define memset(s, c, n) __builtin_memset((s), (c), (n))
28186  
28187 void *memcpy(void *dest, const void *src, size_t n);
28188 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
28189  
28190 void *memmove(void *dest, const void *src, size_t n);
28191 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
28192  
28193 int memcmp(const void *s1, const void *s2, size_t n);
28194 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
28195  
28196 #undef LIBDEFLATE_ENABLE_ASSERTIONS
28197 #else
28198 # include <string.h>
28199
28200 # ifdef __clang_analyzer__
28201 # define LIBDEFLATE_ENABLE_ASSERTIONS
28202 # endif
28203 #endif
28204  
28205  
28206 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
28207 NORETURN void
28208 libdeflate_assertion_failed(const char *expr, const char *file, int line);
28209 #define ASSERT(expr) { if (unlikely(!(expr))) \
28210 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
28211 #else
28212 #define ASSERT(expr) (void)(expr)
28213 #endif
28214  
28215 #define CONCAT_IMPL(a, b) a##b
28216 #define CONCAT(a, b) CONCAT_IMPL(a, b)
28217 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
28218  
28219 #endif
28220  
28221 /* #include "deflate_constants.h" */
28222  
28223  
28224 #ifndef LIB_DEFLATE_CONSTANTS_H
28225 #define LIB_DEFLATE_CONSTANTS_H
28226  
28227  
28228 #define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
28229 #define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
28230 #define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
28231  
28232  
28233 #define DEFLATE_MIN_MATCH_LEN 3
28234 #define DEFLATE_MAX_MATCH_LEN 258
28235  
28236  
28237 #define DEFLATE_MAX_MATCH_OFFSET 32768
28238  
28239  
28240 #define DEFLATE_WINDOW_ORDER 15
28241  
28242  
28243 #define DEFLATE_NUM_PRECODE_SYMS 19
28244 #define DEFLATE_NUM_LITLEN_SYMS 288
28245 #define DEFLATE_NUM_OFFSET_SYMS 32
28246  
28247  
28248 #define DEFLATE_MAX_NUM_SYMS 288
28249  
28250  
28251 #define DEFLATE_NUM_LITERALS 256
28252 #define DEFLATE_END_OF_BLOCK 256
28253 #define DEFLATE_FIRST_LEN_SYM 257
28254  
28255  
28256 #define DEFLATE_MAX_PRE_CODEWORD_LEN 7
28257 #define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
28258 #define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
28259  
28260  
28261 #define DEFLATE_MAX_CODEWORD_LEN 15
28262  
28263  
28264 #define DEFLATE_MAX_LENS_OVERRUN 137
28265  
28266  
28267 #define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
28268 #define DEFLATE_MAX_EXTRA_OFFSET_BITS 13
28269  
28270 #endif
28271  
28272  
28273  
28274 #if 0
28275 # pragma message("UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!")
28276 # define SAFETY_CHECK(expr) (void)(expr)
28277 #else
28278 # define SAFETY_CHECK(expr) if (unlikely(!(expr))) return LIBDEFLATE_BAD_DATA
28279 #endif
28280  
28281  
28282  
28283  
28284  
28285  
28286 /* typedef machine_word_t bitbuf_t; */
28287 #define DECOMPRESS_BITBUF_NBITS (8 * (int)sizeof(bitbuf_t))
28288  
28289  
28290 #define BITMASK(n) (((bitbuf_t)1 << (n)) - 1)
28291  
28292  
28293 #define MAX_BITSLEFT \
28294 (UNALIGNED_ACCESS_IS_FAST ? DECOMPRESS_BITBUF_NBITS - 1 : DECOMPRESS_BITBUF_NBITS)
28295  
28296  
28297 #define CONSUMABLE_NBITS (MAX_BITSLEFT - 7)
28298  
28299  
28300 #define FASTLOOP_PRELOADABLE_NBITS \
28301 (UNALIGNED_ACCESS_IS_FAST ? DECOMPRESS_BITBUF_NBITS : CONSUMABLE_NBITS)
28302  
28303  
28304 #define PRELOAD_SLACK MAX(0, FASTLOOP_PRELOADABLE_NBITS - MAX_BITSLEFT)
28305  
28306  
28307 #define CAN_CONSUME(n) (CONSUMABLE_NBITS >= (n))
28308  
28309  
28310 #define CAN_CONSUME_AND_THEN_PRELOAD(consume_nbits, preload_nbits) \
28311 (CONSUMABLE_NBITS >= (consume_nbits) && \
28312 FASTLOOP_PRELOADABLE_NBITS >= (consume_nbits) + (preload_nbits))
28313  
28314  
28315 #define REFILL_BITS_BRANCHLESS() \
28316 do { \
28317 bitbuf |= get_unaligned_leword(in_next) << (u8)bitsleft; \
28318 in_next += sizeof(bitbuf_t) - 1; \
28319 in_next -= (bitsleft >> 3) & 0x7; \
28320 bitsleft |= MAX_BITSLEFT & ~7; \
28321 } while (0)
28322  
28323  
28324 #define REFILL_BITS() \
28325 do { \
28326 if (UNALIGNED_ACCESS_IS_FAST && \
28327 likely(in_end - in_next >= sizeof(bitbuf_t))) { \
28328 REFILL_BITS_BRANCHLESS(); \
28329 } else { \
28330 while ((u8)bitsleft < CONSUMABLE_NBITS) { \
28331 if (likely(in_next != in_end)) { \
28332 bitbuf |= (bitbuf_t)*in_next++ << \
28333 (u8)bitsleft; \
28334 } else { \
28335 overread_count++; \
28336 SAFETY_CHECK(overread_count <= \
28337 sizeof(bitbuf_t)); \
28338 } \
28339 bitsleft += 8; \
28340 } \
28341 } \
28342 } while (0)
28343  
28344  
28345 #define REFILL_BITS_IN_FASTLOOP() \
28346 do { \
28347 STATIC_ASSERT(UNALIGNED_ACCESS_IS_FAST || \
28348 FASTLOOP_PRELOADABLE_NBITS == CONSUMABLE_NBITS); \
28349 if (UNALIGNED_ACCESS_IS_FAST) { \
28350 REFILL_BITS_BRANCHLESS(); \
28351 } else { \
28352 while ((u8)bitsleft < CONSUMABLE_NBITS) { \
28353 bitbuf |= (bitbuf_t)*in_next++ << (u8)bitsleft; \
28354 bitsleft += 8; \
28355 } \
28356 } \
28357 } while (0)
28358  
28359  
28360 #define FASTLOOP_MAX_BYTES_WRITTEN \
28361 (2 + DEFLATE_MAX_MATCH_LEN + (5 * WORDBYTES) - 1)
28362  
28363  
28364 #define FASTLOOP_MAX_BYTES_READ \
28365 (DIV_ROUND_UP(MAX_BITSLEFT + (2 * LITLEN_TABLEBITS) + \
28366 LENGTH_MAXBITS + OFFSET_MAXBITS, 8) + \
28367 sizeof(bitbuf_t))
28368  
28369  
28370  
28371  
28372  
28373  
28374  
28375 #define PRECODE_TABLEBITS 7
28376 #define PRECODE_ENOUGH 128
28377 #define LITLEN_TABLEBITS 11
28378 #define LITLEN_ENOUGH 2342
28379 #define OFFSET_TABLEBITS 8
28380 #define OFFSET_ENOUGH 402
28381  
28382  
28383 static forceinline u32
28384 make_decode_table_entry(const u32 decode_results[], u32 sym, u32 len)
28385 {
28386 0 return decode_results[sym] + (len << 8) + len;
28387 }
28388  
28389  
28390 static const u32 precode_decode_results[] = {
28391 #define ENTRY(presym) ((u32)presym << 16)
28392 ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,
28393 ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,
28394 ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,
28395 ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,
28396 ENTRY(16) , ENTRY(17) , ENTRY(18) ,
28397 #undef ENTRY
28398 };
28399  
28400  
28401  
28402  
28403 #define HUFFDEC_LITERAL 0x80000000
28404  
28405  
28406 #define HUFFDEC_EXCEPTIONAL 0x00008000
28407  
28408  
28409 #define HUFFDEC_SUBTABLE_POINTER 0x00004000
28410  
28411  
28412 #define HUFFDEC_END_OF_BLOCK 0x00002000
28413  
28414  
28415 #define LENGTH_MAXBITS (DEFLATE_MAX_LITLEN_CODEWORD_LEN + \
28416 DEFLATE_MAX_EXTRA_LENGTH_BITS)
28417 #define LENGTH_MAXFASTBITS (LITLEN_TABLEBITS + \
28418 DEFLATE_MAX_EXTRA_LENGTH_BITS)
28419  
28420  
28421 static const u32 litlen_decode_results[] = {
28422  
28423
28424 #define ENTRY(literal) (HUFFDEC_LITERAL | ((u32)literal << 16))
28425 ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,
28426 ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,
28427 ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,
28428 ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,
28429 ENTRY(16) , ENTRY(17) , ENTRY(18) , ENTRY(19) ,
28430 ENTRY(20) , ENTRY(21) , ENTRY(22) , ENTRY(23) ,
28431 ENTRY(24) , ENTRY(25) , ENTRY(26) , ENTRY(27) ,
28432 ENTRY(28) , ENTRY(29) , ENTRY(30) , ENTRY(31) ,
28433 ENTRY(32) , ENTRY(33) , ENTRY(34) , ENTRY(35) ,
28434 ENTRY(36) , ENTRY(37) , ENTRY(38) , ENTRY(39) ,
28435 ENTRY(40) , ENTRY(41) , ENTRY(42) , ENTRY(43) ,
28436 ENTRY(44) , ENTRY(45) , ENTRY(46) , ENTRY(47) ,
28437 ENTRY(48) , ENTRY(49) , ENTRY(50) , ENTRY(51) ,
28438 ENTRY(52) , ENTRY(53) , ENTRY(54) , ENTRY(55) ,
28439 ENTRY(56) , ENTRY(57) , ENTRY(58) , ENTRY(59) ,
28440 ENTRY(60) , ENTRY(61) , ENTRY(62) , ENTRY(63) ,
28441 ENTRY(64) , ENTRY(65) , ENTRY(66) , ENTRY(67) ,
28442 ENTRY(68) , ENTRY(69) , ENTRY(70) , ENTRY(71) ,
28443 ENTRY(72) , ENTRY(73) , ENTRY(74) , ENTRY(75) ,
28444 ENTRY(76) , ENTRY(77) , ENTRY(78) , ENTRY(79) ,
28445 ENTRY(80) , ENTRY(81) , ENTRY(82) , ENTRY(83) ,
28446 ENTRY(84) , ENTRY(85) , ENTRY(86) , ENTRY(87) ,
28447 ENTRY(88) , ENTRY(89) , ENTRY(90) , ENTRY(91) ,
28448 ENTRY(92) , ENTRY(93) , ENTRY(94) , ENTRY(95) ,
28449 ENTRY(96) , ENTRY(97) , ENTRY(98) , ENTRY(99) ,
28450 ENTRY(100) , ENTRY(101) , ENTRY(102) , ENTRY(103) ,
28451 ENTRY(104) , ENTRY(105) , ENTRY(106) , ENTRY(107) ,
28452 ENTRY(108) , ENTRY(109) , ENTRY(110) , ENTRY(111) ,
28453 ENTRY(112) , ENTRY(113) , ENTRY(114) , ENTRY(115) ,
28454 ENTRY(116) , ENTRY(117) , ENTRY(118) , ENTRY(119) ,
28455 ENTRY(120) , ENTRY(121) , ENTRY(122) , ENTRY(123) ,
28456 ENTRY(124) , ENTRY(125) , ENTRY(126) , ENTRY(127) ,
28457 ENTRY(128) , ENTRY(129) , ENTRY(130) , ENTRY(131) ,
28458 ENTRY(132) , ENTRY(133) , ENTRY(134) , ENTRY(135) ,
28459 ENTRY(136) , ENTRY(137) , ENTRY(138) , ENTRY(139) ,
28460 ENTRY(140) , ENTRY(141) , ENTRY(142) , ENTRY(143) ,
28461 ENTRY(144) , ENTRY(145) , ENTRY(146) , ENTRY(147) ,
28462 ENTRY(148) , ENTRY(149) , ENTRY(150) , ENTRY(151) ,
28463 ENTRY(152) , ENTRY(153) , ENTRY(154) , ENTRY(155) ,
28464 ENTRY(156) , ENTRY(157) , ENTRY(158) , ENTRY(159) ,
28465 ENTRY(160) , ENTRY(161) , ENTRY(162) , ENTRY(163) ,
28466 ENTRY(164) , ENTRY(165) , ENTRY(166) , ENTRY(167) ,
28467 ENTRY(168) , ENTRY(169) , ENTRY(170) , ENTRY(171) ,
28468 ENTRY(172) , ENTRY(173) , ENTRY(174) , ENTRY(175) ,
28469 ENTRY(176) , ENTRY(177) , ENTRY(178) , ENTRY(179) ,
28470 ENTRY(180) , ENTRY(181) , ENTRY(182) , ENTRY(183) ,
28471 ENTRY(184) , ENTRY(185) , ENTRY(186) , ENTRY(187) ,
28472 ENTRY(188) , ENTRY(189) , ENTRY(190) , ENTRY(191) ,
28473 ENTRY(192) , ENTRY(193) , ENTRY(194) , ENTRY(195) ,
28474 ENTRY(196) , ENTRY(197) , ENTRY(198) , ENTRY(199) ,
28475 ENTRY(200) , ENTRY(201) , ENTRY(202) , ENTRY(203) ,
28476 ENTRY(204) , ENTRY(205) , ENTRY(206) , ENTRY(207) ,
28477 ENTRY(208) , ENTRY(209) , ENTRY(210) , ENTRY(211) ,
28478 ENTRY(212) , ENTRY(213) , ENTRY(214) , ENTRY(215) ,
28479 ENTRY(216) , ENTRY(217) , ENTRY(218) , ENTRY(219) ,
28480 ENTRY(220) , ENTRY(221) , ENTRY(222) , ENTRY(223) ,
28481 ENTRY(224) , ENTRY(225) , ENTRY(226) , ENTRY(227) ,
28482 ENTRY(228) , ENTRY(229) , ENTRY(230) , ENTRY(231) ,
28483 ENTRY(232) , ENTRY(233) , ENTRY(234) , ENTRY(235) ,
28484 ENTRY(236) , ENTRY(237) , ENTRY(238) , ENTRY(239) ,
28485 ENTRY(240) , ENTRY(241) , ENTRY(242) , ENTRY(243) ,
28486 ENTRY(244) , ENTRY(245) , ENTRY(246) , ENTRY(247) ,
28487 ENTRY(248) , ENTRY(249) , ENTRY(250) , ENTRY(251) ,
28488 ENTRY(252) , ENTRY(253) , ENTRY(254) , ENTRY(255) ,
28489 #undef ENTRY
28490  
28491
28492 HUFFDEC_EXCEPTIONAL | HUFFDEC_END_OF_BLOCK,
28493  
28494
28495 #define ENTRY(length_base, num_extra_bits) \
28496 (((u32)(length_base) << 16) | (num_extra_bits))
28497 ENTRY(3 , 0) , ENTRY(4 , 0) , ENTRY(5 , 0) , ENTRY(6 , 0),
28498 ENTRY(7 , 0) , ENTRY(8 , 0) , ENTRY(9 , 0) , ENTRY(10 , 0),
28499 ENTRY(11 , 1) , ENTRY(13 , 1) , ENTRY(15 , 1) , ENTRY(17 , 1),
28500 ENTRY(19 , 2) , ENTRY(23 , 2) , ENTRY(27 , 2) , ENTRY(31 , 2),
28501 ENTRY(35 , 3) , ENTRY(43 , 3) , ENTRY(51 , 3) , ENTRY(59 , 3),
28502 ENTRY(67 , 4) , ENTRY(83 , 4) , ENTRY(99 , 4) , ENTRY(115, 4),
28503 ENTRY(131, 5) , ENTRY(163, 5) , ENTRY(195, 5) , ENTRY(227, 5),
28504 ENTRY(258, 0) , ENTRY(258, 0) , ENTRY(258, 0) ,
28505 #undef ENTRY
28506 };
28507  
28508  
28509 #define OFFSET_MAXBITS (DEFLATE_MAX_OFFSET_CODEWORD_LEN + \
28510 DEFLATE_MAX_EXTRA_OFFSET_BITS)
28511 #define OFFSET_MAXFASTBITS (OFFSET_TABLEBITS + \
28512 DEFLATE_MAX_EXTRA_OFFSET_BITS)
28513  
28514  
28515 static const u32 offset_decode_results[] = {
28516 #define ENTRY(offset_base, num_extra_bits) \
28517 (((u32)(offset_base) << 16) | (num_extra_bits))
28518 ENTRY(1 , 0) , ENTRY(2 , 0) , ENTRY(3 , 0) , ENTRY(4 , 0) ,
28519 ENTRY(5 , 1) , ENTRY(7 , 1) , ENTRY(9 , 2) , ENTRY(13 , 2) ,
28520 ENTRY(17 , 3) , ENTRY(25 , 3) , ENTRY(33 , 4) , ENTRY(49 , 4) ,
28521 ENTRY(65 , 5) , ENTRY(97 , 5) , ENTRY(129 , 6) , ENTRY(193 , 6) ,
28522 ENTRY(257 , 7) , ENTRY(385 , 7) , ENTRY(513 , 8) , ENTRY(769 , 8) ,
28523 ENTRY(1025 , 9) , ENTRY(1537 , 9) , ENTRY(2049 , 10) , ENTRY(3073 , 10) ,
28524 ENTRY(4097 , 11) , ENTRY(6145 , 11) , ENTRY(8193 , 12) , ENTRY(12289 , 12) ,
28525 ENTRY(16385 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) ,
28526 #undef ENTRY
28527 };
28528  
28529  
28530 struct libdeflate_decompressor {
28531  
28532
28533  
28534 union {
28535 u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];
28536  
28537 struct {
28538 u8 lens[DEFLATE_NUM_LITLEN_SYMS +
28539 DEFLATE_NUM_OFFSET_SYMS +
28540 DEFLATE_MAX_LENS_OVERRUN];
28541  
28542 u32 precode_decode_table[PRECODE_ENOUGH];
28543 } l;
28544  
28545 u32 litlen_decode_table[LITLEN_ENOUGH];
28546 } u;
28547  
28548 u32 offset_decode_table[OFFSET_ENOUGH];
28549  
28550
28551 u16 sorted_syms[DEFLATE_MAX_NUM_SYMS];
28552  
28553 bool static_codes_loaded;
28554 unsigned litlen_tablebits;
28555  
28556
28557 free_func_t free_func;
28558 };
28559  
28560  
28561 static bool
28562 87 build_decode_table(u32 decode_table[],
28563 const u8 lens[],
28564 const unsigned num_syms,
28565 const u32 decode_results[],
28566 unsigned table_bits,
28567 unsigned max_codeword_len,
28568 u16 *sorted_syms,
28569 unsigned *table_bits_ret)
28570 {
28571 unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
28572 unsigned offsets[DEFLATE_MAX_CODEWORD_LEN + 1];
28573 unsigned sym;
28574 unsigned codeword;
28575 unsigned len;
28576 unsigned count;
28577 u32 codespace_used;
28578 unsigned cur_table_end;
28579 unsigned subtable_prefix;
28580 unsigned subtable_start;
28581 unsigned subtable_bits;
28582  
28583
28584 1439 ▶ for (len = 0; len <= max_codeword_len; len++)
Branch: 1 branch
BranchTrueFalseTotal
for (len = 0; len <= max_codeword_len; len++) 1352 87 1439
28585 1352 len_counts[len] = 0;
28586 13235 ▶ for (sym = 0; sym < num_syms; sym++)
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < num_syms; sym++) 13148 87 13235
28587 13148 len_counts[lens[sym]]++;
28588  
28589
28590 715 ▶ while (max_codeword_len > 1 && len_counts[max_codeword_len] == 0)
Branch: 2 branches
BranchTrueFalseTotal
while (max_codeword_len > 1 && len_counts[max_codeword_len] == 0) 715 0 715
while (max_codeword_len > 1 && len_counts[max_codeword_len] == 0) 628 87 715
28591 628 max_codeword_len--;
28592 87 ▶ if (table_bits_ret != NULL) {
Branch: 1 branch
BranchTrueFalseTotal
if (table_bits_ret != NULL) { 41 46 87
28593 41 table_bits = MIN(table_bits, max_codeword_len);
28594 41 *table_bits_ret = table_bits;
28595 }
28596  
28597
28598  
28599
28600 STATIC_ASSERT(sizeof(codespace_used) == 4);
28601 STATIC_ASSERT(UINT32_MAX / (1U << (DEFLATE_MAX_CODEWORD_LEN - 1)) >=
28602 DEFLATE_MAX_NUM_SYMS);
28603  
28604 87 offsets[0] = 0;
28605 87 offsets[1] = len_counts[0];
28606 87 codespace_used = 0;
28607 637 ▶ for (len = 1; len < max_codeword_len; len++) {
Branch: 1 branch
BranchTrueFalseTotal
for (len = 1; len < max_codeword_len; len++) { 550 87 637
28608 550 offsets[len + 1] = offsets[len] + len_counts[len];
28609 550 codespace_used = (codespace_used << 1) + len_counts[len];
28610 }
28611 87 codespace_used = (codespace_used << 1) + len_counts[len];
28612  
28613 13235 ▶ for (sym = 0; sym < num_syms; sym++)
Branch: 1 branch
BranchTrueFalseTotal
for (sym = 0; sym < num_syms; sym++) 13148 87 13235
28614 13148 sorted_syms[offsets[lens[sym]]++] = sym;
28615  
28616 87 sorted_syms += offsets[0];
28617  
28618
28619  
28620
28621  
28622
28623 87 ▶ if (unlikely(codespace_used > (1U << max_codeword_len)))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(codespace_used > (1U << max_codeword_len))) 0 87 87
28624 0 return false;
28625  
28626
28627 87 ▶ if (unlikely(codespace_used < (1U << max_codeword_len))) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(codespace_used < (1U << max_codeword_len))) { 0 87 87
28628 u32 entry;
28629 unsigned i;
28630  
28631
28632 0 ▶ if (codespace_used == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (codespace_used == 0) { 0 0 0
28633 0 sym = 0;
28634 } else {
28635 0 ▶ if (codespace_used != (1U << (max_codeword_len - 1)) ||
Branch: 1 branch
BranchTrueFalseTotal
if (codespace_used != (1U << (max_codeword_len - 1)) || 0 0 0
28636 0 ▶ len_counts[1] != 1)
Branch: 1 branch
BranchTrueFalseTotal
len_counts[1] != 1) 0 0 0
28637 0 return false;
28638 0 sym = sorted_syms[0];
28639 }
28640 0 entry = make_decode_table_entry(decode_results, sym, 1);
28641 0 ▶ for (i = 0; i < (1U << table_bits); i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < (1U << table_bits); i++) 0 0 0
28642 0 decode_table[i] = entry;
28643 0 return true;
28644 }
28645  
28646
28647 87 codeword = 0;
28648 87 len = 1;
28649 480 ▶ while ((count = len_counts[len]) == 0)
Branch: 1 branch
BranchTrueFalseTotal
while ((count = len_counts[len]) == 0) 393 87 480
28650 393 len++;
28651 87 cur_table_end = 1U << len;
28652 241 ▶ while (len <= table_bits) {
Branch: 1 branch
BranchTrueFalseTotal
while (len <= table_bits) { 233 8 241
28653
28654 do {
28655 unsigned bit;
28656  
28657
28658 24320 decode_table[codeword] =
28659 24320 make_decode_table_entry(decode_results,
28660 12160 *sorted_syms++, len);
28661  
28662 12160 ▶ if (codeword == cur_table_end - 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (codeword == cur_table_end - 1) { 79 12081 12160
28663
28664 190 ▶ for (; len < table_bits; len++) {
Branch: 1 branch
BranchTrueFalseTotal
for (; len < table_bits; len++) { 111 79 190
28665 111 memcpy(&decode_table[cur_table_end],
28666 decode_table,
28667 cur_table_end *
28668 sizeof(decode_table[0]));
28669 111 cur_table_end <<= 1;
28670 }
28671 79 return true;
28672 }
28673
28674 12081 bit = 1U << bsr32(codeword ^ (cur_table_end - 1));
28675 12081 codeword &= bit - 1;
28676 12081 codeword |= bit;
28677 12081 ▶ } while (--count);
Branch: 1 branch
BranchTrueFalseTotal
} while (--count); 11927 154 12081
28678  
28679
28680 do {
28681 157 ▶ if (++len <= table_bits) {
Branch: 1 branch
BranchTrueFalseTotal
if (++len <= table_bits) { 149 8 157
28682 149 memcpy(&decode_table[cur_table_end],
28683 decode_table,
28684 cur_table_end * sizeof(decode_table[0]));
28685 149 cur_table_end <<= 1;
28686 }
28687 157 ▶ } while ((count = len_counts[len]) == 0);
Branch: 1 branch
BranchTrueFalseTotal
} while ((count = len_counts[len]) == 0); 3 154 157
28688 }
28689  
28690
28691 8 cur_table_end = 1U << table_bits;
28692 8 subtable_prefix = -1;
28693 8 subtable_start = 0;
28694 30 for (;;) {
28695 u32 entry;
28696 unsigned i;
28697 unsigned stride;
28698 unsigned bit;
28699  
28700
28701 38 ▶ if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) {
Branch: 1 branch
BranchTrueFalseTotal
if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) { 19 19 38
28702 19 subtable_prefix = (codeword & ((1U << table_bits) - 1));
28703 19 subtable_start = cur_table_end;
28704
28705 19 subtable_bits = len - table_bits;
28706 19 codespace_used = count;
28707 19 ▶ while (codespace_used < (1U << subtable_bits)) {
Branch: 1 branch
BranchTrueFalseTotal
while (codespace_used < (1U << subtable_bits)) { 0 19 19
28708 0 subtable_bits++;
28709 0 codespace_used = (codespace_used << 1) +
28710 0 len_counts[table_bits + subtable_bits];
28711 }
28712 19 cur_table_end = subtable_start + (1U << subtable_bits);
28713  
28714
28715 19 decode_table[subtable_prefix] =
28716 19 ((u32)subtable_start << 16) |
28717 HUFFDEC_EXCEPTIONAL |
28718 19 HUFFDEC_SUBTABLE_POINTER |
28719 19 (subtable_bits << 8) | table_bits;
28720 }
28721  
28722
28723 38 entry = make_decode_table_entry(decode_results, *sorted_syms++,
28724 len - table_bits);
28725 38 i = subtable_start + (codeword >> table_bits);
28726 38 stride = 1U << (len - table_bits);
28727 do {
28728 38 decode_table[i] = entry;
28729 38 i += stride;
28730 38 ▶ } while (i < cur_table_end);
Branch: 1 branch
BranchTrueFalseTotal
} while (i < cur_table_end); 0 38 38
28731  
28732
28733 38 ▶ if (codeword == (1U << len) - 1)
Branch: 1 branch
BranchTrueFalseTotal
if (codeword == (1U << len) - 1) 8 30 38
28734 8 return true;
28735 30 bit = 1U << bsr32(codeword ^ ((1U << len) - 1));
28736 30 codeword &= bit - 1;
28737 30 codeword |= bit;
28738 30 count--;
28739 30 ▶ while (count == 0)
Branch: 1 branch
BranchTrueFalseTotal
while (count == 0) 0 30 30
28740 0 count = len_counts[++len];
28741 }
28742 }
28743  
28744  
28745 static bool
28746 5 build_precode_decode_table(struct libdeflate_decompressor *d)
28747 {
28748
28749 STATIC_ASSERT(PRECODE_TABLEBITS == 7 && PRECODE_ENOUGH == 128);
28750  
28751 STATIC_ASSERT(ARRAY_LEN(precode_decode_results) ==
28752 DEFLATE_NUM_PRECODE_SYMS);
28753  
28754 10 return build_decode_table(d->u.l.precode_decode_table,
28755 5 d->u.precode_lens,
28756 DEFLATE_NUM_PRECODE_SYMS,
28757 precode_decode_results,
28758 PRECODE_TABLEBITS,
28759 DEFLATE_MAX_PRE_CODEWORD_LEN,
28760 5 d->sorted_syms,
28761 NULL);
28762 }
28763  
28764  
28765 static bool
28766 41 build_litlen_decode_table(struct libdeflate_decompressor *d,
28767 unsigned num_litlen_syms, unsigned num_offset_syms)
28768 {
28769
28770 STATIC_ASSERT(LITLEN_TABLEBITS == 11 && LITLEN_ENOUGH == 2342);
28771  
28772 STATIC_ASSERT(ARRAY_LEN(litlen_decode_results) ==
28773 DEFLATE_NUM_LITLEN_SYMS);
28774  
28775 82 return build_decode_table(d->u.litlen_decode_table,
28776 41 d->u.l.lens,
28777 num_litlen_syms,
28778 litlen_decode_results,
28779 LITLEN_TABLEBITS,
28780 DEFLATE_MAX_LITLEN_CODEWORD_LEN,
28781 41 d->sorted_syms,
28782 &d->litlen_tablebits);
28783 }
28784  
28785  
28786 static bool
28787 41 build_offset_decode_table(struct libdeflate_decompressor *d,
28788 unsigned num_litlen_syms, unsigned num_offset_syms)
28789 {
28790
28791 STATIC_ASSERT(OFFSET_TABLEBITS == 8 && OFFSET_ENOUGH == 402);
28792  
28793 STATIC_ASSERT(ARRAY_LEN(offset_decode_results) ==
28794 DEFLATE_NUM_OFFSET_SYMS);
28795  
28796 82 return build_decode_table(d->offset_decode_table,
28797 41 d->u.l.lens + num_litlen_syms,
28798 num_offset_syms,
28799 offset_decode_results,
28800 OFFSET_TABLEBITS,
28801 DEFLATE_MAX_OFFSET_CODEWORD_LEN,
28802 41 d->sorted_syms,
28803 NULL);
28804 }
28805  
28806  
28807  
28808 typedef enum libdeflate_result (*decompress_func_t)
28809 (struct libdeflate_decompressor * restrict d,
28810 const void * restrict in, size_t in_nbytes,
28811 void * restrict out, size_t out_nbytes_avail,
28812 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
28813  
28814 #define FUNCNAME deflate_decompress_default
28815 #undef ATTRIBUTES
28816 #undef EXTRACT_VARBITS
28817 #undef EXTRACT_VARBITS8
28818 /* #include "decompress_template.h" */
28819  
28820  
28821  
28822  
28823 #ifndef ATTRIBUTES
28824 # define ATTRIBUTES
28825 #endif
28826 #ifndef EXTRACT_VARBITS
28827 # define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))
28828 #endif
28829 #ifndef EXTRACT_VARBITS8
28830 # define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))
28831 #endif
28832  
28833 static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
28834 0 FUNCNAME(struct libdeflate_decompressor * restrict d,
28835 const void * restrict in, size_t in_nbytes,
28836 void * restrict out, size_t out_nbytes_avail,
28837 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
28838 {
28839 0 u8 *out_next = out;
28840 0 u8 * const out_end = out_next + out_nbytes_avail;
28841 0 u8 * const out_fastloop_end =
28842 0 ▶ out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
Branch: 1 branch
BranchTrueFalseTotal
out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN); 0 0 0
28843  
28844
28845 0 const u8 *in_next = in;
28846 0 const u8 * const in_end = in_next + in_nbytes;
28847 0 const u8 * const in_fastloop_end =
28848 0 ▶ in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
Branch: 1 branch
BranchTrueFalseTotal
in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ); 0 0 0
28849 0 bitbuf_t bitbuf = 0;
28850 bitbuf_t saved_bitbuf;
28851 0 u32 bitsleft = 0;
28852 0 size_t overread_count = 0;
28853  
28854 bool is_final_block;
28855 unsigned block_type;
28856 unsigned num_litlen_syms;
28857 unsigned num_offset_syms;
28858 bitbuf_t litlen_tablemask;
28859 u32 entry;
28860  
28861 0 next_block:
28862
28863 ;
28864  
28865 STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
28866 0 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
28867  
28868
28869 0 is_final_block = bitbuf & BITMASK(1);
28870  
28871
28872 0 block_type = (bitbuf >> 1) & BITMASK(2);
28873  
28874 0 ▶ if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
Branch: 1 branch
BranchTrueFalseTotal
if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) { 0 0 0
28875  
28876
28877  
28878
28879 static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
28880 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
28881 };
28882  
28883 unsigned num_explicit_precode_lens;
28884 unsigned i;
28885  
28886
28887  
28888 STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
28889 0 num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
28890  
28891 STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
28892 0 num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
28893  
28894 STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
28895 0 num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
28896  
28897 0 d->static_codes_loaded = false;
28898  
28899
28900 STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
28901 if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
28902 0 d->u.precode_lens[deflate_precode_lens_permutation[0]] =
28903 0 (bitbuf >> 17) & BITMASK(3);
28904 0 bitbuf >>= 20;
28905 0 bitsleft -= 20;
28906 0 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
28907 0 i = 1;
28908 do {
28909 0 d->u.precode_lens[deflate_precode_lens_permutation[i]] =
28910 0 bitbuf & BITMASK(3);
28911 0 bitbuf >>= 3;
28912 0 bitsleft -= 3;
28913 0 ▶ } while (++i < num_explicit_precode_lens);
Branch: 1 branch
BranchTrueFalseTotal
} while (++i < num_explicit_precode_lens); 0 0 0
28914 } else {
28915 bitbuf >>= 17;
28916 bitsleft -= 17;
28917 i = 0;
28918 do {
28919 if ((u8)bitsleft < 3)
28920 REFILL_BITS();
28921 d->u.precode_lens[deflate_precode_lens_permutation[i]] =
28922 bitbuf & BITMASK(3);
28923 bitbuf >>= 3;
28924 bitsleft -= 3;
28925 } while (++i < num_explicit_precode_lens);
28926 }
28927 0 ▶ for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < DEFLATE_NUM_PRECODE_SYMS; i++) 0 0 0
28928 0 d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
28929  
28930
28931 0 ▶ SAFETY_CHECK(build_precode_decode_table(d));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_precode_decode_table(d)); 0 0 0
28932  
28933
28934 0 i = 0;
28935 do {
28936 unsigned presym;
28937 u8 rep_val;
28938 unsigned rep_count;
28939  
28940 0 ▶ if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
Branch: 1 branch
BranchTrueFalseTotal
if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7) 0 0 0
28941 0 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
28942  
28943
28944 STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
28945  
28946
28947 0 entry = d->u.l.precode_decode_table[
28948 0 bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
28949 0 bitbuf >>= (u8)entry;
28950 0 bitsleft -= entry;
28951 0 presym = entry >> 16;
28952  
28953 0 ▶ if (presym < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (presym < 16) { 0 0 0
28954
28955 0 d->u.l.lens[i++] = presym;
28956 0 continue;
28957 }
28958  
28959
28960  
28961
28962 STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
28963  
28964 0 ▶ if (presym == 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (presym == 16) { 0 0 0
28965
28966 0 ▶ SAFETY_CHECK(i != 0);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(i != 0); 0 0 0
28967 0 rep_val = d->u.l.lens[i - 1];
28968 STATIC_ASSERT(3 + BITMASK(2) == 6);
28969 0 rep_count = 3 + (bitbuf & BITMASK(2));
28970 0 bitbuf >>= 2;
28971 0 bitsleft -= 2;
28972 0 d->u.l.lens[i + 0] = rep_val;
28973 0 d->u.l.lens[i + 1] = rep_val;
28974 0 d->u.l.lens[i + 2] = rep_val;
28975 0 d->u.l.lens[i + 3] = rep_val;
28976 0 d->u.l.lens[i + 4] = rep_val;
28977 0 d->u.l.lens[i + 5] = rep_val;
28978 0 i += rep_count;
28979 0 ▶ } else if (presym == 17) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (presym == 17) { 0 0 0
28980
28981 STATIC_ASSERT(3 + BITMASK(3) == 10);
28982 0 rep_count = 3 + (bitbuf & BITMASK(3));
28983 0 bitbuf >>= 3;
28984 0 bitsleft -= 3;
28985 0 d->u.l.lens[i + 0] = 0;
28986 0 d->u.l.lens[i + 1] = 0;
28987 0 d->u.l.lens[i + 2] = 0;
28988 0 d->u.l.lens[i + 3] = 0;
28989 0 d->u.l.lens[i + 4] = 0;
28990 0 d->u.l.lens[i + 5] = 0;
28991 0 d->u.l.lens[i + 6] = 0;
28992 0 d->u.l.lens[i + 7] = 0;
28993 0 d->u.l.lens[i + 8] = 0;
28994 0 d->u.l.lens[i + 9] = 0;
28995 0 i += rep_count;
28996 } else {
28997
28998 STATIC_ASSERT(11 + BITMASK(7) == 138);
28999 0 rep_count = 11 + (bitbuf & BITMASK(7));
29000 0 bitbuf >>= 7;
29001 0 bitsleft -= 7;
29002 0 memset(&d->u.l.lens[i], 0,
29003 rep_count * sizeof(d->u.l.lens[i]));
29004 0 i += rep_count;
29005 }
29006 0 ▶ } while (i < num_litlen_syms + num_offset_syms);
Branch: 1 branch
BranchTrueFalseTotal
} while (i < num_litlen_syms + num_offset_syms); 0 0 0
29007  
29008
29009 0 ▶ SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(i == num_litlen_syms + num_offset_syms); 0 0 0
29010  
29011 0 ▶ } else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) { 0 0 0
29012 u16 len, nlen;
29013  
29014
29015  
29016 0 bitsleft -= 3;
29017  
29018
29019 0 bitsleft = (u8)bitsleft;
29020 0 ▶ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(overread_count <= (bitsleft >> 3)); 0 0 0
29021 0 in_next -= (bitsleft >> 3) - overread_count;
29022 0 overread_count = 0;
29023 0 bitbuf = 0;
29024 0 bitsleft = 0;
29025  
29026 0 ▶ SAFETY_CHECK(in_end - in_next >= 4);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(in_end - in_next >= 4); 0 0 0
29027 0 len = get_unaligned_le16(in_next);
29028 0 nlen = get_unaligned_le16(in_next + 2);
29029 0 in_next += 4;
29030  
29031 0 ▶ SAFETY_CHECK(len == (u16)~nlen);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(len == (u16)~nlen); 0 0 0
29032 0 ▶ if (unlikely(len > out_end - out_next))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(len > out_end - out_next)) 0 0 0
29033 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
29034 0 ▶ SAFETY_CHECK(len <= in_end - in_next);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(len <= in_end - in_next); 0 0 0
29035  
29036 0 memcpy(out_next, in_next, len);
29037 0 in_next += len;
29038 0 out_next += len;
29039  
29040 0 goto block_done;
29041  
29042 } else {
29043 unsigned i;
29044  
29045 0 ▶ SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN); 0 0 0
29046  
29047
29048  
29049 0 bitbuf >>= 3;
29050 0 bitsleft -= 3;
29051  
29052 0 ▶ if (d->static_codes_loaded)
Branch: 1 branch
BranchTrueFalseTotal
if (d->static_codes_loaded) 0 0 0
29053 0 goto have_decode_tables;
29054  
29055 0 d->static_codes_loaded = true;
29056  
29057 STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
29058 STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
29059  
29060 0 ▶ for (i = 0; i < 144; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 144; i++) 0 0 0
29061 0 d->u.l.lens[i] = 8;
29062 0 ▶ for (; i < 256; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 256; i++) 0 0 0
29063 0 d->u.l.lens[i] = 9;
29064 0 ▶ for (; i < 280; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 280; i++) 0 0 0
29065 0 d->u.l.lens[i] = 7;
29066 0 ▶ for (; i < 288; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 288; i++) 0 0 0
29067 0 d->u.l.lens[i] = 8;
29068  
29069 0 ▶ for (; i < 288 + 32; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 288 + 32; i++) 0 0 0
29070 0 d->u.l.lens[i] = 5;
29071  
29072 0 num_litlen_syms = 288;
29073 0 num_offset_syms = 32;
29074 }
29075  
29076
29077  
29078 0 ▶ SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms)); 0 0 0
29079 0 ▶ SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms)); 0 0 0
29080 0 have_decode_tables:
29081 0 litlen_tablemask = BITMASK(d->litlen_tablebits);
29082  
29083
29084 0 ▶ if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
Branch: 2 branches
BranchTrueFalseTotal
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end) 0 0 0
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end) 0 0 0
29085 0 goto generic_loop;
29086 0 REFILL_BITS_IN_FASTLOOP();
29087 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29088 do {
29089 u32 length, offset, lit;
29090 const u8 *src;
29091 u8 *dst;
29092  
29093
29094 0 saved_bitbuf = bitbuf;
29095 0 bitbuf >>= (u8)entry;
29096 0 bitsleft -= entry;
29097  
29098
29099 0 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 0 0 0
29100
29101 if (
29102 CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
29103 LENGTH_MAXBITS,
29104 OFFSET_TABLEBITS) &&
29105
29106 CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
29107 DEFLATE_MAX_LITLEN_CODEWORD_LEN,
29108 LITLEN_TABLEBITS)) {
29109
29110 0 lit = entry >> 16;
29111 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29112 0 saved_bitbuf = bitbuf;
29113 0 bitbuf >>= (u8)entry;
29114 0 bitsleft -= entry;
29115 0 *out_next++ = lit;
29116 0 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 0 0 0
29117
29118 0 lit = entry >> 16;
29119 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29120 0 saved_bitbuf = bitbuf;
29121 0 bitbuf >>= (u8)entry;
29122 0 bitsleft -= entry;
29123 0 *out_next++ = lit;
29124 0 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 0 0 0
29125
29126 0 lit = entry >> 16;
29127 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29128 0 REFILL_BITS_IN_FASTLOOP();
29129 0 *out_next++ = lit;
29130 0 continue;
29131 }
29132 }
29133 } else {
29134
29135 STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
29136 LITLEN_TABLEBITS, LITLEN_TABLEBITS));
29137 lit = entry >> 16;
29138 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29139 REFILL_BITS_IN_FASTLOOP();
29140 *out_next++ = lit;
29141 continue;
29142 }
29143 }
29144  
29145
29146 0 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 0 0 0
29147
29148  
29149 0 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 0 0 0
29150 0 goto block_done;
29151  
29152
29153 0 entry = d->u.litlen_decode_table[(entry >> 16) +
29154 0 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
29155 0 saved_bitbuf = bitbuf;
29156 0 bitbuf >>= (u8)entry;
29157 0 bitsleft -= entry;
29158  
29159
29160 if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
29161 LITLEN_TABLEBITS) ||
29162 !CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
29163 OFFSET_TABLEBITS))
29164 REFILL_BITS_IN_FASTLOOP();
29165 0 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 0 0 0
29166
29167 0 lit = entry >> 16;
29168 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29169 0 REFILL_BITS_IN_FASTLOOP();
29170 0 *out_next++ = lit;
29171 0 continue;
29172 }
29173 0 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 0 0 0
29174 0 goto block_done;
29175
29176 }
29177  
29178
29179 0 length = entry >> 16;
29180 0 length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
29181  
29182
29183 STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
29184 OFFSET_TABLEBITS));
29185 0 entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
29186 if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
29187 LITLEN_TABLEBITS)) {
29188
29189 0 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 0 0 0
29190
29191 0 ▶ if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely((u8)bitsleft < OFFSET_MAXBITS + 0 0 0
29192 LITLEN_TABLEBITS - PRELOAD_SLACK))
29193 0 REFILL_BITS_IN_FASTLOOP();
29194 0 bitbuf >>= OFFSET_TABLEBITS;
29195 0 bitsleft -= OFFSET_TABLEBITS;
29196 0 entry = d->offset_decode_table[(entry >> 16) +
29197 0 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
29198 0 ▶ } else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
Branch: 1 branch
BranchTrueFalseTotal
} else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS + 0 0 0
29199 LITLEN_TABLEBITS - PRELOAD_SLACK))
29200 0 REFILL_BITS_IN_FASTLOOP();
29201 } else {
29202
29203 REFILL_BITS_IN_FASTLOOP();
29204 if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
29205
29206 bitbuf >>= OFFSET_TABLEBITS;
29207 bitsleft -= OFFSET_TABLEBITS;
29208 entry = d->offset_decode_table[(entry >> 16) +
29209 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
29210 REFILL_BITS_IN_FASTLOOP();
29211
29212 STATIC_ASSERT(CAN_CONSUME(
29213 OFFSET_MAXBITS - OFFSET_TABLEBITS));
29214 } else {
29215
29216 STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
29217 }
29218 }
29219 0 saved_bitbuf = bitbuf;
29220 0 bitbuf >>= (u8)entry;
29221 0 bitsleft -= entry;
29222 0 offset = entry >> 16;
29223 0 offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
29224  
29225
29226 0 ▶ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(offset <= out_next - (const u8 *)out); 0 0 0
29227 0 src = out_next - offset;
29228 0 dst = out_next;
29229 0 out_next += length;
29230  
29231
29232 if (!CAN_CONSUME_AND_THEN_PRELOAD(
29233 MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
29234 OFFSET_MAXFASTBITS),
29235 LITLEN_TABLEBITS) &&
29236 unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
29237 REFILL_BITS_IN_FASTLOOP();
29238 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29239 0 REFILL_BITS_IN_FASTLOOP();
29240  
29241
29242 0 ▶ if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
Branch: 1 branch
BranchTrueFalseTotal
if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) { 0 0 0
29243 0 store_word_unaligned(load_word_unaligned(src), dst);
29244 0 src += WORDBYTES;
29245 0 dst += WORDBYTES;
29246 0 store_word_unaligned(load_word_unaligned(src), dst);
29247 0 src += WORDBYTES;
29248 0 dst += WORDBYTES;
29249 0 store_word_unaligned(load_word_unaligned(src), dst);
29250 0 src += WORDBYTES;
29251 0 dst += WORDBYTES;
29252 0 store_word_unaligned(load_word_unaligned(src), dst);
29253 0 src += WORDBYTES;
29254 0 dst += WORDBYTES;
29255 0 store_word_unaligned(load_word_unaligned(src), dst);
29256 0 src += WORDBYTES;
29257 0 dst += WORDBYTES;
29258 0 ▶ while (dst < out_next) {
Branch: 1 branch
BranchTrueFalseTotal
while (dst < out_next) { 0 0 0
29259 0 store_word_unaligned(load_word_unaligned(src), dst);
29260 0 src += WORDBYTES;
29261 0 dst += WORDBYTES;
29262 0 store_word_unaligned(load_word_unaligned(src), dst);
29263 0 src += WORDBYTES;
29264 0 dst += WORDBYTES;
29265 0 store_word_unaligned(load_word_unaligned(src), dst);
29266 0 src += WORDBYTES;
29267 0 dst += WORDBYTES;
29268 0 store_word_unaligned(load_word_unaligned(src), dst);
29269 0 src += WORDBYTES;
29270 0 dst += WORDBYTES;
29271 0 store_word_unaligned(load_word_unaligned(src), dst);
29272 0 src += WORDBYTES;
29273 0 dst += WORDBYTES;
29274 }
29275 0 ▶ } else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) { 0 0 0
29276 machine_word_t v;
29277  
29278
29279 0 v = (machine_word_t)0x0101010101010101 * src[0];
29280 0 store_word_unaligned(v, dst);
29281 0 dst += WORDBYTES;
29282 0 store_word_unaligned(v, dst);
29283 0 dst += WORDBYTES;
29284 0 store_word_unaligned(v, dst);
29285 0 dst += WORDBYTES;
29286 0 store_word_unaligned(v, dst);
29287 0 dst += WORDBYTES;
29288 0 ▶ while (dst < out_next) {
Branch: 1 branch
BranchTrueFalseTotal
while (dst < out_next) { 0 0 0
29289 0 store_word_unaligned(v, dst);
29290 0 dst += WORDBYTES;
29291 0 store_word_unaligned(v, dst);
29292 0 dst += WORDBYTES;
29293 0 store_word_unaligned(v, dst);
29294 0 dst += WORDBYTES;
29295 0 store_word_unaligned(v, dst);
29296 0 dst += WORDBYTES;
29297 }
29298 } else if (UNALIGNED_ACCESS_IS_FAST) {
29299 0 store_word_unaligned(load_word_unaligned(src), dst);
29300 0 src += offset;
29301 0 dst += offset;
29302 0 store_word_unaligned(load_word_unaligned(src), dst);
29303 0 src += offset;
29304 0 dst += offset;
29305 do {
29306 0 store_word_unaligned(load_word_unaligned(src), dst);
29307 0 src += offset;
29308 0 dst += offset;
29309 0 store_word_unaligned(load_word_unaligned(src), dst);
29310 0 src += offset;
29311 0 dst += offset;
29312 0 ▶ } while (dst < out_next);
Branch: 1 branch
BranchTrueFalseTotal
} while (dst < out_next); 0 0 0
29313 } else {
29314 *dst++ = *src++;
29315 *dst++ = *src++;
29316 do {
29317 *dst++ = *src++;
29318 } while (dst < out_next);
29319 }
29320 0 ▶ } while (in_next < in_fastloop_end && out_next < out_fastloop_end);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next < in_fastloop_end && out_next < out_fastloop_end); 0 0 0
} while (in_next < in_fastloop_end && out_next < out_fastloop_end); 0 0 0
29321  
29322
29323 0 generic_loop:
29324 0 for (;;) {
29325 u32 length, offset;
29326 const u8 *src;
29327 u8 *dst;
29328  
29329 0 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
29330 0 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
29331 0 saved_bitbuf = bitbuf;
29332 0 bitbuf >>= (u8)entry;
29333 0 bitsleft -= entry;
29334 0 ▶ if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) { 0 0 0
29335 0 entry = d->u.litlen_decode_table[(entry >> 16) +
29336 0 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
29337 0 saved_bitbuf = bitbuf;
29338 0 bitbuf >>= (u8)entry;
29339 0 bitsleft -= entry;
29340 }
29341 0 length = entry >> 16;
29342 0 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 0 0 0
29343 0 ▶ if (unlikely(out_next == out_end))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(out_next == out_end)) 0 0 0
29344 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
29345 0 *out_next++ = length;
29346 0 continue;
29347 }
29348 0 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 0 0 0
29349 0 goto block_done;
29350 0 length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
29351 0 ▶ if (unlikely(length > out_end - out_next))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(length > out_end - out_next)) 0 0 0
29352 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
29353  
29354 if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
29355 REFILL_BITS();
29356 0 entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
29357 0 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 0 0 0
29358 0 bitbuf >>= OFFSET_TABLEBITS;
29359 0 bitsleft -= OFFSET_TABLEBITS;
29360 0 entry = d->offset_decode_table[(entry >> 16) +
29361 0 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
29362 if (!CAN_CONSUME(OFFSET_MAXBITS))
29363 REFILL_BITS();
29364 }
29365 0 offset = entry >> 16;
29366 0 offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
29367 0 bitbuf >>= (u8)entry;
29368 0 bitsleft -= entry;
29369  
29370 0 ▶ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(offset <= out_next - (const u8 *)out); 0 0 0
29371 0 src = out_next - offset;
29372 0 dst = out_next;
29373 0 out_next += length;
29374  
29375 STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
29376 0 *dst++ = *src++;
29377 0 *dst++ = *src++;
29378 do {
29379 0 *dst++ = *src++;
29380 0 ▶ } while (dst < out_next);
Branch: 1 branch
BranchTrueFalseTotal
} while (dst < out_next); 0 0 0
29381 }
29382  
29383 0 block_done:
29384
29385  
29386 0 ▶ if (!is_final_block)
Branch: 1 branch
BranchTrueFalseTotal
if (!is_final_block) 0 0 0
29387 0 goto next_block;
29388  
29389
29390  
29391 0 bitsleft = (u8)bitsleft;
29392  
29393
29394 0 ▶ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(overread_count <= (bitsleft >> 3)); 0 0 0
29395  
29396
29397 0 ▶ if (actual_in_nbytes_ret) {
Branch: 1 branch
BranchTrueFalseTotal
if (actual_in_nbytes_ret) { 0 0 0
29398
29399 0 in_next -= (bitsleft >> 3) - overread_count;
29400  
29401 0 *actual_in_nbytes_ret = in_next - (u8 *)in;
29402 }
29403  
29404
29405 0 ▶ if (actual_out_nbytes_ret) {
Branch: 1 branch
BranchTrueFalseTotal
if (actual_out_nbytes_ret) { 0 0 0
29406 0 *actual_out_nbytes_ret = out_next - (u8 *)out;
29407 } else {
29408 0 ▶ if (out_next != out_end)
Branch: 1 branch
BranchTrueFalseTotal
if (out_next != out_end) 0 0 0
29409 0 return LIBDEFLATE_SHORT_OUTPUT;
29410 }
29411 0 return LIBDEFLATE_SUCCESS;
29412 }
29413  
29414 #undef FUNCNAME
29415 #undef ATTRIBUTES
29416 #undef EXTRACT_VARBITS
29417 #undef EXTRACT_VARBITS8
29418  
29419  
29420  
29421 #undef DEFAULT_IMPL
29422 #undef arch_select_decompress_func
29423 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
29424 /* # include "x86/decompress_impl.h" */
29425 #ifndef LIB_X86_DECOMPRESS_IMPL_H
29426 #define LIB_X86_DECOMPRESS_IMPL_H
29427  
29428 /* #include "x86-cpu_features.h" */
29429  
29430  
29431 #ifndef LIB_X86_CPU_FEATURES_H
29432 #define LIB_X86_CPU_FEATURES_H
29433  
29434 /* #include "lib_common.h" */
29435  
29436  
29437 #ifndef LIB_LIB_COMMON_H
29438 #define LIB_LIB_COMMON_H
29439  
29440 #ifdef LIBDEFLATE_H
29441
29442 # error "lib_common.h must always be included before libdeflate.h"
29443 #endif
29444  
29445 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
29446 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
29447 #elif defined(__GNUC__)
29448 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
29449 #else
29450 # define LIBDEFLATE_EXPORT_SYM
29451 #endif
29452  
29453  
29454 #if defined(__GNUC__) && defined(__i386__)
29455 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
29456 #else
29457 # define LIBDEFLATE_ALIGN_STACK
29458 #endif
29459  
29460 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
29461  
29462 /* #include "../common_defs.h" */
29463  
29464  
29465 #ifndef COMMON_DEFS_H
29466 #define COMMON_DEFS_H
29467  
29468 /* #include "libdeflate.h" */
29469  
29470  
29471 #ifndef LIBDEFLATE_H
29472 #define LIBDEFLATE_H
29473  
29474 #include <stddef.h>
29475 #include <stdint.h>
29476  
29477 #ifdef __cplusplus
29478 extern "C" {
29479 #endif
29480  
29481 #define LIBDEFLATE_VERSION_MAJOR 1
29482 #define LIBDEFLATE_VERSION_MINOR 26
29483 #define LIBDEFLATE_VERSION_STRING "1.26"
29484  
29485  
29486 #ifndef LIBDEFLATEAPI
29487 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
29488 # define LIBDEFLATEAPI __declspec(dllimport)
29489 # else
29490 # define LIBDEFLATEAPI
29491 # endif
29492 #endif
29493  
29494  
29495  
29496  
29497  
29498 struct libdeflate_compressor;
29499 struct libdeflate_options;
29500  
29501  
29502 LIBDEFLATEAPI struct libdeflate_compressor *
29503 libdeflate_alloc_compressor(int compression_level);
29504  
29505  
29506 LIBDEFLATEAPI struct libdeflate_compressor *
29507 libdeflate_alloc_compressor_ex(int compression_level,
29508 const struct libdeflate_options *options);
29509  
29510  
29511 LIBDEFLATEAPI size_t
29512 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
29513 const void *in, size_t in_nbytes,
29514 void *out, size_t out_nbytes_avail);
29515  
29516  
29517 LIBDEFLATEAPI size_t
29518 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
29519 size_t in_nbytes);
29520  
29521  
29522 LIBDEFLATEAPI size_t
29523 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
29524 const void *in, size_t in_nbytes,
29525 void *out, size_t out_nbytes_avail);
29526  
29527  
29528 LIBDEFLATEAPI size_t
29529 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
29530 size_t in_nbytes);
29531  
29532  
29533 LIBDEFLATEAPI size_t
29534 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
29535 const void *in, size_t in_nbytes,
29536 void *out, size_t out_nbytes_avail);
29537  
29538  
29539 LIBDEFLATEAPI size_t
29540 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
29541 size_t in_nbytes);
29542  
29543  
29544 LIBDEFLATEAPI void
29545 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
29546  
29547  
29548  
29549  
29550  
29551 struct libdeflate_decompressor;
29552 struct libdeflate_options;
29553  
29554  
29555 LIBDEFLATEAPI struct libdeflate_decompressor *
29556 libdeflate_alloc_decompressor(void);
29557  
29558  
29559 LIBDEFLATEAPI struct libdeflate_decompressor *
29560 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
29561  
29562  
29563 enum libdeflate_result {
29564
29565 LIBDEFLATE_SUCCESS = 0,
29566  
29567
29568 LIBDEFLATE_BAD_DATA = 1,
29569  
29570
29571 LIBDEFLATE_SHORT_OUTPUT = 2,
29572  
29573
29574 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
29575 };
29576  
29577  
29578 LIBDEFLATEAPI enum libdeflate_result
29579 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
29580 const void *in, size_t in_nbytes,
29581 void *out, size_t out_nbytes_avail,
29582 size_t *actual_out_nbytes_ret);
29583  
29584  
29585 LIBDEFLATEAPI enum libdeflate_result
29586 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
29587 const void *in, size_t in_nbytes,
29588 void *out, size_t out_nbytes_avail,
29589 size_t *actual_in_nbytes_ret,
29590 size_t *actual_out_nbytes_ret);
29591  
29592  
29593 LIBDEFLATEAPI enum libdeflate_result
29594 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
29595 const void *in, size_t in_nbytes,
29596 void *out, size_t out_nbytes_avail,
29597 size_t *actual_out_nbytes_ret);
29598  
29599  
29600 LIBDEFLATEAPI enum libdeflate_result
29601 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
29602 const void *in, size_t in_nbytes,
29603 void *out, size_t out_nbytes_avail,
29604 size_t *actual_in_nbytes_ret,
29605 size_t *actual_out_nbytes_ret);
29606  
29607  
29608 LIBDEFLATEAPI enum libdeflate_result
29609 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
29610 const void *in, size_t in_nbytes,
29611 void *out, size_t out_nbytes_avail,
29612 size_t *actual_out_nbytes_ret);
29613  
29614  
29615 LIBDEFLATEAPI enum libdeflate_result
29616 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
29617 const void *in, size_t in_nbytes,
29618 void *out, size_t out_nbytes_avail,
29619 size_t *actual_in_nbytes_ret,
29620 size_t *actual_out_nbytes_ret);
29621  
29622  
29623 LIBDEFLATEAPI void
29624 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
29625  
29626  
29627  
29628  
29629  
29630  
29631 LIBDEFLATEAPI uint32_t
29632 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
29633  
29634  
29635  
29636 LIBDEFLATEAPI uint32_t
29637 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
29638  
29639  
29640  
29641  
29642  
29643  
29644 LIBDEFLATEAPI void
29645 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
29646 void (*free_func)(void *));
29647  
29648  
29649 struct libdeflate_options {
29650  
29651
29652 size_t sizeof_options;
29653  
29654
29655 void *(*malloc_func)(size_t);
29656 void (*free_func)(void *);
29657 };
29658  
29659 #ifdef __cplusplus
29660 }
29661 #endif
29662  
29663 #endif
29664  
29665  
29666 #include <stdbool.h>
29667 #include <stddef.h>
29668 #include <stdint.h>
29669 #ifdef _MSC_VER
29670 # include <intrin.h>
29671 # include <stdlib.h>
29672
29673
29674 # pragma warning(disable : 4146)
29675
29676 # pragma warning(disable : 4018)
29677 # pragma warning(disable : 4244)
29678 # pragma warning(disable : 4267)
29679 # pragma warning(disable : 4310)
29680
29681 # pragma warning(disable : 4100)
29682 # pragma warning(disable : 4127)
29683 # pragma warning(disable : 4189)
29684 # pragma warning(disable : 4232)
29685 # pragma warning(disable : 4245)
29686 # pragma warning(disable : 4295)
29687 #endif
29688 #ifndef FREESTANDING
29689 # include <string.h>
29690 #endif
29691  
29692  
29693  
29694  
29695  
29696  
29697 #undef ARCH_X86_64
29698 #undef ARCH_X86_32
29699 #undef ARCH_ARM64
29700 #undef ARCH_ARM32
29701 #undef ARCH_RISCV
29702 #ifdef _MSC_VER
29703
29704 # if defined(_M_X64) && !defined(_M_ARM64EC)
29705 # define ARCH_X86_64
29706 # elif defined(_M_IX86)
29707 # define ARCH_X86_32
29708 # elif defined(_M_ARM64)
29709 # define ARCH_ARM64
29710 # elif defined(_M_ARM)
29711 # define ARCH_ARM32
29712 # endif
29713 #else
29714 # if defined(__x86_64__)
29715 # define ARCH_X86_64
29716 # elif defined(__i386__)
29717 # define ARCH_X86_32
29718 # elif defined(__aarch64__)
29719 # define ARCH_ARM64
29720 # elif defined(__arm__)
29721 # define ARCH_ARM32
29722 # elif defined(__riscv)
29723 # define ARCH_RISCV
29724 # endif
29725 #endif
29726  
29727  
29728  
29729  
29730  
29731  
29732 typedef uint8_t u8;
29733 typedef uint16_t u16;
29734 typedef uint32_t u32;
29735 typedef uint64_t u64;
29736 typedef int8_t s8;
29737 typedef int16_t s16;
29738 typedef int32_t s32;
29739 typedef int64_t s64;
29740  
29741  
29742 #ifdef _MSC_VER
29743 # ifdef _WIN64
29744 typedef long long ssize_t;
29745 # else
29746 typedef long ssize_t;
29747 # endif
29748 #endif
29749  
29750  
29751 typedef size_t machine_word_t;
29752  
29753  
29754 #define WORDBYTES ((int)sizeof(machine_word_t))
29755  
29756  
29757 #define WORDBITS (8 * WORDBYTES)
29758  
29759  
29760  
29761  
29762  
29763  
29764 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
29765 # define GCC_PREREQ(major, minor) \
29766 (__GNUC__ > (major) || \
29767 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
29768 # if !GCC_PREREQ(4, 9)
29769 # error "gcc versions older than 4.9 are no longer supported"
29770 # endif
29771 #else
29772 # define GCC_PREREQ(major, minor) 0
29773 #endif
29774 #ifdef __clang__
29775 # ifdef __apple_build_version__
29776 # define CLANG_PREREQ(major, minor, apple_version) \
29777 (__apple_build_version__ >= (apple_version))
29778 # else
29779 # define CLANG_PREREQ(major, minor, apple_version) \
29780 (__clang_major__ > (major) || \
29781 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
29782 # endif
29783 # if !CLANG_PREREQ(3, 9, 8000000)
29784 # error "clang versions older than 3.9 are no longer supported"
29785 # endif
29786 #else
29787 # define CLANG_PREREQ(major, minor, apple_version) 0
29788 #endif
29789 #ifdef _MSC_VER
29790 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
29791 # if !MSVC_PREREQ(1928)
29792 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
29793 # endif
29794 #else
29795 # define MSVC_PREREQ(version) 0
29796 #endif
29797  
29798  
29799 #ifndef __has_attribute
29800 # define __has_attribute(attribute) 0
29801 #endif
29802  
29803  
29804 #ifndef __has_builtin
29805 # define __has_builtin(builtin) 0
29806 #endif
29807  
29808  
29809 #ifdef _MSC_VER
29810 # define inline __inline
29811 #endif
29812  
29813  
29814 #if defined(__GNUC__) || __has_attribute(always_inline)
29815 # define forceinline inline __attribute__((always_inline))
29816 #elif defined(_MSC_VER)
29817 # define forceinline __forceinline
29818 #else
29819 # define forceinline inline
29820 #endif
29821  
29822  
29823 #if defined(__GNUC__) || __has_attribute(unused)
29824 # define MAYBE_UNUSED __attribute__((unused))
29825 #else
29826 # define MAYBE_UNUSED
29827 #endif
29828  
29829  
29830 #if defined(__GNUC__) || __has_attribute(noreturn)
29831 # define NORETURN __attribute__((noreturn))
29832 #else
29833 # define NORETURN
29834 #endif
29835  
29836  
29837 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
29838 # define likely(expr) __builtin_expect(!!(expr), 1)
29839 #else
29840 # define likely(expr) (expr)
29841 #endif
29842  
29843  
29844 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
29845 # define unlikely(expr) __builtin_expect(!!(expr), 0)
29846 #else
29847 # define unlikely(expr) (expr)
29848 #endif
29849  
29850  
29851 #undef prefetchr
29852 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
29853 # define prefetchr(addr) __builtin_prefetch((addr), 0)
29854 #elif defined(_MSC_VER)
29855 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
29856 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
29857 # elif defined(ARCH_ARM64)
29858 # define prefetchr(addr) __prefetch2((addr), 0x00 )
29859 # elif defined(ARCH_ARM32)
29860 # define prefetchr(addr) __prefetch(addr)
29861 # endif
29862 #endif
29863 #ifndef prefetchr
29864 # define prefetchr(addr)
29865 #endif
29866  
29867  
29868 #undef prefetchw
29869 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
29870 # define prefetchw(addr) __builtin_prefetch((addr), 1)
29871 #elif defined(_MSC_VER)
29872 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
29873 # define prefetchw(addr) _m_prefetchw(addr)
29874 # elif defined(ARCH_ARM64)
29875 # define prefetchw(addr) __prefetch2((addr), 0x10 )
29876 # elif defined(ARCH_ARM32)
29877 # define prefetchw(addr) __prefetchw(addr)
29878 # endif
29879 #endif
29880 #ifndef prefetchw
29881 # define prefetchw(addr)
29882 #endif
29883  
29884  
29885 #undef _aligned_attribute
29886 #if defined(__GNUC__) || __has_attribute(aligned)
29887 # define _aligned_attribute(n) __attribute__((aligned(n)))
29888 #elif defined(_MSC_VER)
29889 # define _aligned_attribute(n) __declspec(align(n))
29890 #endif
29891  
29892  
29893 #if defined(__GNUC__) || __has_attribute(target)
29894 # define _target_attribute(attrs) __attribute__((target(attrs)))
29895 #else
29896 # define _target_attribute(attrs)
29897 #endif
29898  
29899  
29900  
29901  
29902  
29903 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
29904 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
29905 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
29906 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
29907 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
29908 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
29909 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
29910  
29911  
29912  
29913  
29914  
29915  
29916 #if defined(__BYTE_ORDER__)
29917 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
29918 #elif defined(_MSC_VER)
29919 # define CPU_IS_LITTLE_ENDIAN() true
29920 #else
29921 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
29922 {
29923 union {
29924 u32 w;
29925 u8 b;
29926 } u;
29927  
29928 u.w = 1;
29929 return u.b;
29930 }
29931 #endif
29932  
29933  
29934 static forceinline u16 bswap16(u16 v)
29935 {
29936 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
29937 return __builtin_bswap16(v);
29938 #elif defined(_MSC_VER)
29939 return _byteswap_ushort(v);
29940 #else
29941 return (v << 8) | (v >> 8);
29942 #endif
29943 }
29944  
29945  
29946 static forceinline u32 bswap32(u32 v)
29947 {
29948 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
29949 return __builtin_bswap32(v);
29950 #elif defined(_MSC_VER)
29951 return _byteswap_ulong(v);
29952 #else
29953 return ((v & 0x000000FF) << 24) |
29954 ((v & 0x0000FF00) << 8) |
29955 ((v & 0x00FF0000) >> 8) |
29956 ((v & 0xFF000000) >> 24);
29957 #endif
29958 }
29959  
29960  
29961 static forceinline u64 bswap64(u64 v)
29962 {
29963 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
29964 return __builtin_bswap64(v);
29965 #elif defined(_MSC_VER)
29966 return _byteswap_uint64(v);
29967 #else
29968 return ((v & 0x00000000000000FF) << 56) |
29969 ((v & 0x000000000000FF00) << 40) |
29970 ((v & 0x0000000000FF0000) << 24) |
29971 ((v & 0x00000000FF000000) << 8) |
29972 ((v & 0x000000FF00000000) >> 8) |
29973 ((v & 0x0000FF0000000000) >> 24) |
29974 ((v & 0x00FF000000000000) >> 40) |
29975 ((v & 0xFF00000000000000) >> 56);
29976 #endif
29977 }
29978  
29979 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
29980 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
29981 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
29982 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
29983 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
29984 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
29985  
29986  
29987  
29988  
29989  
29990  
29991 #if (defined(__GNUC__) || defined(__clang__)) && \
29992 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
29993 defined(__ARM_FEATURE_UNALIGNED) || \
29994 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
29995 defined(__riscv_misaligned_fast) || \
29996 defined(__wasm__))
29997 # define UNALIGNED_ACCESS_IS_FAST 1
29998 #elif defined(_MSC_VER)
29999 # define UNALIGNED_ACCESS_IS_FAST 1
30000 #else
30001 # define UNALIGNED_ACCESS_IS_FAST 0
30002 #endif
30003  
30004  
30005  
30006 #ifdef FREESTANDING
30007 # define MEMCOPY __builtin_memcpy
30008 #else
30009 # define MEMCOPY memcpy
30010 #endif
30011  
30012  
30013  
30014 #define DEFINE_UNALIGNED_TYPE(type) \
30015 static forceinline type \
30016 load_##type##_unaligned(const void *p) \
30017 { \
30018 type v; \
30019 \
30020 MEMCOPY(&v, p, sizeof(v)); \
30021 return v; \
30022 } \
30023 \
30024 static forceinline void \
30025 store_##type##_unaligned(type v, void *p) \
30026 { \
30027 MEMCOPY(p, &v, sizeof(v)); \
30028 }
30029  
30030 DEFINE_UNALIGNED_TYPE(u16)
30031 DEFINE_UNALIGNED_TYPE(u32)
30032 DEFINE_UNALIGNED_TYPE(u64)
30033 DEFINE_UNALIGNED_TYPE(machine_word_t)
30034  
30035 #undef MEMCOPY
30036  
30037 #define load_word_unaligned load_machine_word_t_unaligned
30038 #define store_word_unaligned store_machine_word_t_unaligned
30039  
30040  
30041  
30042 static forceinline u16
30043 get_unaligned_le16(const u8 *p)
30044 {
30045 if (UNALIGNED_ACCESS_IS_FAST)
30046 return le16_bswap(load_u16_unaligned(p));
30047 else
30048 return ((u16)p[1] << 8) | p[0];
30049 }
30050  
30051 static forceinline u16
30052 get_unaligned_be16(const u8 *p)
30053 {
30054 if (UNALIGNED_ACCESS_IS_FAST)
30055 return be16_bswap(load_u16_unaligned(p));
30056 else
30057 return ((u16)p[0] << 8) | p[1];
30058 }
30059  
30060 static forceinline u32
30061 get_unaligned_le32(const u8 *p)
30062 {
30063 if (UNALIGNED_ACCESS_IS_FAST)
30064 return le32_bswap(load_u32_unaligned(p));
30065 else
30066 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
30067 ((u32)p[1] << 8) | p[0];
30068 }
30069  
30070 static forceinline u32
30071 get_unaligned_be32(const u8 *p)
30072 {
30073 if (UNALIGNED_ACCESS_IS_FAST)
30074 return be32_bswap(load_u32_unaligned(p));
30075 else
30076 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
30077 ((u32)p[2] << 8) | p[3];
30078 }
30079  
30080 static forceinline u64
30081 get_unaligned_le64(const u8 *p)
30082 {
30083 if (UNALIGNED_ACCESS_IS_FAST)
30084 return le64_bswap(load_u64_unaligned(p));
30085 else
30086 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
30087 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
30088 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
30089 ((u64)p[1] << 8) | p[0];
30090 }
30091  
30092 static forceinline machine_word_t
30093 get_unaligned_leword(const u8 *p)
30094 {
30095 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
30096 if (WORDBITS == 32)
30097 return get_unaligned_le32(p);
30098 else
30099 return get_unaligned_le64(p);
30100 }
30101  
30102  
30103  
30104 static forceinline void
30105 put_unaligned_le16(u16 v, u8 *p)
30106 {
30107 if (UNALIGNED_ACCESS_IS_FAST) {
30108 store_u16_unaligned(le16_bswap(v), p);
30109 } else {
30110 p[0] = (u8)(v >> 0);
30111 p[1] = (u8)(v >> 8);
30112 }
30113 }
30114  
30115 static forceinline void
30116 put_unaligned_be16(u16 v, u8 *p)
30117 {
30118 if (UNALIGNED_ACCESS_IS_FAST) {
30119 store_u16_unaligned(be16_bswap(v), p);
30120 } else {
30121 p[0] = (u8)(v >> 8);
30122 p[1] = (u8)(v >> 0);
30123 }
30124 }
30125  
30126 static forceinline void
30127 put_unaligned_le32(u32 v, u8 *p)
30128 {
30129 if (UNALIGNED_ACCESS_IS_FAST) {
30130 store_u32_unaligned(le32_bswap(v), p);
30131 } else {
30132 p[0] = (u8)(v >> 0);
30133 p[1] = (u8)(v >> 8);
30134 p[2] = (u8)(v >> 16);
30135 p[3] = (u8)(v >> 24);
30136 }
30137 }
30138  
30139 static forceinline void
30140 put_unaligned_be32(u32 v, u8 *p)
30141 {
30142 if (UNALIGNED_ACCESS_IS_FAST) {
30143 store_u32_unaligned(be32_bswap(v), p);
30144 } else {
30145 p[0] = (u8)(v >> 24);
30146 p[1] = (u8)(v >> 16);
30147 p[2] = (u8)(v >> 8);
30148 p[3] = (u8)(v >> 0);
30149 }
30150 }
30151  
30152 static forceinline void
30153 put_unaligned_le64(u64 v, u8 *p)
30154 {
30155 if (UNALIGNED_ACCESS_IS_FAST) {
30156 store_u64_unaligned(le64_bswap(v), p);
30157 } else {
30158 p[0] = (u8)(v >> 0);
30159 p[1] = (u8)(v >> 8);
30160 p[2] = (u8)(v >> 16);
30161 p[3] = (u8)(v >> 24);
30162 p[4] = (u8)(v >> 32);
30163 p[5] = (u8)(v >> 40);
30164 p[6] = (u8)(v >> 48);
30165 p[7] = (u8)(v >> 56);
30166 }
30167 }
30168  
30169 static forceinline void
30170 put_unaligned_leword(machine_word_t v, u8 *p)
30171 {
30172 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
30173 if (WORDBITS == 32)
30174 put_unaligned_le32(v, p);
30175 else
30176 put_unaligned_le64(v, p);
30177 }
30178  
30179  
30180  
30181  
30182  
30183  
30184  
30185 static forceinline unsigned
30186 bsr32(u32 v)
30187 {
30188 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
30189 return 31 - __builtin_clz(v);
30190 #elif defined(_MSC_VER)
30191 unsigned long i;
30192  
30193 _BitScanReverse(&i, v);
30194 return i;
30195 #else
30196 unsigned i = 0;
30197  
30198 while ((v >>= 1) != 0)
30199 i++;
30200 return i;
30201 #endif
30202 }
30203  
30204 static forceinline unsigned
30205 bsr64(u64 v)
30206 {
30207 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
30208 return 63 - __builtin_clzll(v);
30209 #elif defined(_MSC_VER) && defined(_WIN64)
30210 unsigned long i;
30211  
30212 _BitScanReverse64(&i, v);
30213 return i;
30214 #else
30215 unsigned i = 0;
30216  
30217 while ((v >>= 1) != 0)
30218 i++;
30219 return i;
30220 #endif
30221 }
30222  
30223 static forceinline unsigned
30224 bsrw(machine_word_t v)
30225 {
30226 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
30227 if (WORDBITS == 32)
30228 return bsr32(v);
30229 else
30230 return bsr64(v);
30231 }
30232  
30233  
30234  
30235 static forceinline unsigned
30236 bsf32(u32 v)
30237 {
30238 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
30239 return __builtin_ctz(v);
30240 #elif defined(_MSC_VER)
30241 unsigned long i;
30242  
30243 _BitScanForward(&i, v);
30244 return i;
30245 #else
30246 unsigned i = 0;
30247  
30248 for (; (v & 1) == 0; v >>= 1)
30249 i++;
30250 return i;
30251 #endif
30252 }
30253  
30254 static forceinline unsigned
30255 bsf64(u64 v)
30256 {
30257 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
30258 return __builtin_ctzll(v);
30259 #elif defined(_MSC_VER) && defined(_WIN64)
30260 unsigned long i;
30261  
30262 _BitScanForward64(&i, v);
30263 return i;
30264 #else
30265 unsigned i = 0;
30266  
30267 for (; (v & 1) == 0; v >>= 1)
30268 i++;
30269 return i;
30270 #endif
30271 }
30272  
30273 static forceinline unsigned
30274 bsfw(machine_word_t v)
30275 {
30276 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
30277 if (WORDBITS == 32)
30278 return bsf32(v);
30279 else
30280 return bsf64(v);
30281 }
30282  
30283  
30284 #undef rbit32
30285 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
30286 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
30287 static forceinline u32
30288 rbit32(u32 v)
30289 {
30290 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
30291 return v;
30292 }
30293 #define rbit32 rbit32
30294 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
30295 static forceinline u32
30296 rbit32(u32 v)
30297 {
30298 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
30299 return v;
30300 }
30301 #define rbit32 rbit32
30302 #endif
30303  
30304 #endif
30305  
30306  
30307 typedef void *(*malloc_func_t)(size_t);
30308 typedef void (*free_func_t)(void *);
30309  
30310 extern malloc_func_t libdeflate_default_malloc_func;
30311 extern free_func_t libdeflate_default_free_func;
30312  
30313 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
30314 size_t alignment, size_t size);
30315 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
30316  
30317 #ifdef FREESTANDING
30318  
30319 void *memset(void *s, int c, size_t n);
30320 #define memset(s, c, n) __builtin_memset((s), (c), (n))
30321  
30322 void *memcpy(void *dest, const void *src, size_t n);
30323 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
30324  
30325 void *memmove(void *dest, const void *src, size_t n);
30326 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
30327  
30328 int memcmp(const void *s1, const void *s2, size_t n);
30329 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
30330  
30331 #undef LIBDEFLATE_ENABLE_ASSERTIONS
30332 #else
30333 # include <string.h>
30334
30335 # ifdef __clang_analyzer__
30336 # define LIBDEFLATE_ENABLE_ASSERTIONS
30337 # endif
30338 #endif
30339  
30340  
30341 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
30342 NORETURN void
30343 libdeflate_assertion_failed(const char *expr, const char *file, int line);
30344 #define ASSERT(expr) { if (unlikely(!(expr))) \
30345 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
30346 #else
30347 #define ASSERT(expr) (void)(expr)
30348 #endif
30349  
30350 #define CONCAT_IMPL(a, b) a##b
30351 #define CONCAT(a, b) CONCAT_IMPL(a, b)
30352 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
30353  
30354 #endif
30355  
30356  
30357 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
30358  
30359 #define X86_CPU_FEATURE_SSE2 (1 << 0)
30360 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
30361 #define X86_CPU_FEATURE_AVX (1 << 2)
30362 #define X86_CPU_FEATURE_AVX2 (1 << 3)
30363 #define X86_CPU_FEATURE_BMI2 (1 << 4)
30364  
30365 #define X86_CPU_FEATURE_ZMM (1 << 5)
30366 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
30367 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
30368 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
30369 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
30370 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
30371  
30372 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
30373  
30374 # define X86_CPU_FEATURES_KNOWN (1U << 31)
30375 extern volatile u32 libdeflate_x86_cpu_features;
30376  
30377 void libdeflate_init_x86_cpu_features(void);
30378  
30379 static inline u32 get_x86_cpu_features(void)
30380 {
30381 if (libdeflate_x86_cpu_features == 0)
30382 libdeflate_init_x86_cpu_features();
30383 return libdeflate_x86_cpu_features;
30384 }
30385  
30386 # include <immintrin.h>
30387 # if defined(_MSC_VER) && defined(__clang__)
30388 # include <tmmintrin.h>
30389 # include <smmintrin.h>
30390 # include <wmmintrin.h>
30391 # include <avxintrin.h>
30392 # include <avx2intrin.h>
30393 # include <avx512fintrin.h>
30394 # include <avx512bwintrin.h>
30395 # include <avx512vlintrin.h>
30396 # if __has_include(<avx512vlbwintrin.h>)
30397 # include <avx512vlbwintrin.h>
30398 # endif
30399 # if __has_include(<vpclmulqdqintrin.h>)
30400 # include <vpclmulqdqintrin.h>
30401 # endif
30402 # if __has_include(<avx512vnniintrin.h>)
30403 # include <avx512vnniintrin.h>
30404 # endif
30405 # if __has_include(<avx512vlvnniintrin.h>)
30406 # include <avx512vlvnniintrin.h>
30407 # endif
30408 # if __has_include(<avxvnniintrin.h>)
30409 # include <avxvnniintrin.h>
30410 # endif
30411 # endif
30412 #else
30413 static inline u32 get_x86_cpu_features(void) { return 0; }
30414 #endif
30415  
30416 #if defined(__SSE2__) || \
30417 (defined(_MSC_VER) && \
30418 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
30419 # define HAVE_SSE2(features) 1
30420 # define HAVE_SSE2_NATIVE 1
30421 #else
30422 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
30423 # define HAVE_SSE2_NATIVE 0
30424 #endif
30425  
30426 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
30427 (defined(_MSC_VER) && defined(__AVX2__))
30428 # define HAVE_PCLMULQDQ(features) 1
30429 #else
30430 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
30431 #endif
30432  
30433 #ifdef __AVX__
30434 # define HAVE_AVX(features) 1
30435 #else
30436 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
30437 #endif
30438  
30439 #ifdef __AVX2__
30440 # define HAVE_AVX2(features) 1
30441 #else
30442 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
30443 #endif
30444  
30445 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
30446 # define HAVE_BMI2(features) 1
30447 # define HAVE_BMI2_NATIVE 1
30448 #else
30449 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
30450 # define HAVE_BMI2_NATIVE 0
30451 #endif
30452  
30453 #ifdef __AVX512BW__
30454 # define HAVE_AVX512BW(features) 1
30455 #else
30456 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
30457 #endif
30458  
30459 #ifdef __AVX512VL__
30460 # define HAVE_AVX512VL(features) 1
30461 #else
30462 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
30463 #endif
30464  
30465 #ifdef __VPCLMULQDQ__
30466 # define HAVE_VPCLMULQDQ(features) 1
30467 #else
30468 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
30469 #endif
30470  
30471 #ifdef __AVX512VNNI__
30472 # define HAVE_AVX512VNNI(features) 1
30473 #else
30474 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
30475 #endif
30476  
30477 #ifdef __AVXVNNI__
30478 # define HAVE_AVXVNNI(features) 1
30479 #else
30480 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
30481 #endif
30482  
30483 #endif
30484  
30485 #endif
30486  
30487  
30488  
30489 #if defined(__GNUC__) || defined(__clang__) || MSVC_PREREQ(1930)
30490 # define deflate_decompress_bmi2 deflate_decompress_bmi2
30491 # define FUNCNAME deflate_decompress_bmi2
30492 # define ATTRIBUTES _target_attribute("bmi2")
30493
30494 # ifndef __clang__
30495 # ifdef ARCH_X86_64
30496 # define EXTRACT_VARBITS(word, count) _bzhi_u64((word), (count))
30497 # define EXTRACT_VARBITS8(word, count) _bzhi_u64((word), (count))
30498 # else
30499 # define EXTRACT_VARBITS(word, count) _bzhi_u32((word), (count))
30500 # define EXTRACT_VARBITS8(word, count) _bzhi_u32((word), (count))
30501 # endif
30502 # endif
30503 /* #include "decompress_template.h" */
30504  
30505  
30506  
30507  
30508 #ifndef ATTRIBUTES
30509 # define ATTRIBUTES
30510 #endif
30511 #ifndef EXTRACT_VARBITS
30512 # define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))
30513 #endif
30514 #ifndef EXTRACT_VARBITS8
30515 # define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))
30516 #endif
30517  
30518 static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
30519 40 FUNCNAME(struct libdeflate_decompressor * restrict d,
30520 const void * restrict in, size_t in_nbytes,
30521 void * restrict out, size_t out_nbytes_avail,
30522 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
30523 {
30524 40 u8 *out_next = out;
30525 40 u8 * const out_end = out_next + out_nbytes_avail;
30526 40 u8 * const out_fastloop_end =
30527 40 ▶ out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
Branch: 1 branch
BranchTrueFalseTotal
out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN); 0 40 40
30528  
30529
30530 40 const u8 *in_next = in;
30531 40 const u8 * const in_end = in_next + in_nbytes;
30532 40 const u8 * const in_fastloop_end =
30533 40 ▶ in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
Branch: 1 branch
BranchTrueFalseTotal
in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ); 0 40 40
30534 40 bitbuf_t bitbuf = 0;
30535 bitbuf_t saved_bitbuf;
30536 40 u32 bitsleft = 0;
30537 40 size_t overread_count = 0;
30538  
30539 bool is_final_block;
30540 unsigned block_type;
30541 unsigned num_litlen_syms;
30542 unsigned num_offset_syms;
30543 bitbuf_t litlen_tablemask;
30544 u32 entry;
30545  
30546 41 next_block:
30547
30548 ;
30549  
30550 STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
30551 82 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 41 0 41
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
30552  
30553
30554 41 is_final_block = bitbuf & BITMASK(1);
30555  
30556
30557 41 block_type = (bitbuf >> 1) & BITMASK(2);
30558  
30559 41 ▶ if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
Branch: 1 branch
BranchTrueFalseTotal
if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) { 5 36 41
30560  
30561
30562  
30563
30564 static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
30565 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
30566 };
30567  
30568 unsigned num_explicit_precode_lens;
30569 unsigned i;
30570  
30571
30572  
30573 STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
30574 5 num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
30575  
30576 STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
30577 5 num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
30578  
30579 STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
30580 5 num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
30581  
30582 5 d->static_codes_loaded = false;
30583  
30584
30585 STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
30586 if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
30587 5 d->u.precode_lens[deflate_precode_lens_permutation[0]] =
30588 5 (bitbuf >> 17) & BITMASK(3);
30589 5 bitbuf >>= 20;
30590 5 bitsleft -= 20;
30591 10 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 5 0 5
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
30592 5 i = 1;
30593 do {
30594 65 d->u.precode_lens[deflate_precode_lens_permutation[i]] =
30595 65 bitbuf & BITMASK(3);
30596 65 bitbuf >>= 3;
30597 65 bitsleft -= 3;
30598 65 ▶ } while (++i < num_explicit_precode_lens);
Branch: 1 branch
BranchTrueFalseTotal
} while (++i < num_explicit_precode_lens); 60 5 65
30599 } else {
30600 bitbuf >>= 17;
30601 bitsleft -= 17;
30602 i = 0;
30603 do {
30604 if ((u8)bitsleft < 3)
30605 REFILL_BITS();
30606 d->u.precode_lens[deflate_precode_lens_permutation[i]] =
30607 bitbuf & BITMASK(3);
30608 bitbuf >>= 3;
30609 bitsleft -= 3;
30610 } while (++i < num_explicit_precode_lens);
30611 }
30612 30 ▶ for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < DEFLATE_NUM_PRECODE_SYMS; i++) 25 5 30
30613 25 d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
30614  
30615
30616 5 ▶ SAFETY_CHECK(build_precode_decode_table(d));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_precode_decode_table(d)); 0 5 5
30617  
30618
30619 5 i = 0;
30620 do {
30621 unsigned presym;
30622 u8 rep_val;
30623 unsigned rep_count;
30624  
30625 674 ▶ if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
Branch: 1 branch
BranchTrueFalseTotal
if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7) 55 619 674
30626 110 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 55 0 55
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
REFILL_BITS(); 0 0 0
30627  
30628
30629 STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
30630  
30631
30632 674 entry = d->u.l.precode_decode_table[
30633 674 bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
30634 674 bitbuf >>= (u8)entry;
30635 674 bitsleft -= entry;
30636 674 presym = entry >> 16;
30637  
30638 674 ▶ if (presym < 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (presym < 16) { 626 48 674
30639
30640 626 d->u.l.lens[i++] = presym;
30641 626 continue;
30642 }
30643  
30644
30645  
30646
30647 STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
30648  
30649 48 ▶ if (presym == 16) {
Branch: 1 branch
BranchTrueFalseTotal
if (presym == 16) { 15 33 48
30650
30651 15 ▶ SAFETY_CHECK(i != 0);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(i != 0); 0 15 15
30652 15 rep_val = d->u.l.lens[i - 1];
30653 STATIC_ASSERT(3 + BITMASK(2) == 6);
30654 15 rep_count = 3 + (bitbuf & BITMASK(2));
30655 15 bitbuf >>= 2;
30656 15 bitsleft -= 2;
30657 15 d->u.l.lens[i + 0] = rep_val;
30658 15 d->u.l.lens[i + 1] = rep_val;
30659 15 d->u.l.lens[i + 2] = rep_val;
30660 15 d->u.l.lens[i + 3] = rep_val;
30661 15 d->u.l.lens[i + 4] = rep_val;
30662 15 d->u.l.lens[i + 5] = rep_val;
30663 15 i += rep_count;
30664 33 ▶ } else if (presym == 17) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (presym == 17) { 15 18 33
30665
30666 STATIC_ASSERT(3 + BITMASK(3) == 10);
30667 15 rep_count = 3 + (bitbuf & BITMASK(3));
30668 15 bitbuf >>= 3;
30669 15 bitsleft -= 3;
30670 15 d->u.l.lens[i + 0] = 0;
30671 15 d->u.l.lens[i + 1] = 0;
30672 15 d->u.l.lens[i + 2] = 0;
30673 15 d->u.l.lens[i + 3] = 0;
30674 15 d->u.l.lens[i + 4] = 0;
30675 15 d->u.l.lens[i + 5] = 0;
30676 15 d->u.l.lens[i + 6] = 0;
30677 15 d->u.l.lens[i + 7] = 0;
30678 15 d->u.l.lens[i + 8] = 0;
30679 15 d->u.l.lens[i + 9] = 0;
30680 15 i += rep_count;
30681 } else {
30682
30683 STATIC_ASSERT(11 + BITMASK(7) == 138);
30684 18 rep_count = 11 + (bitbuf & BITMASK(7));
30685 18 bitbuf >>= 7;
30686 18 bitsleft -= 7;
30687 18 memset(&d->u.l.lens[i], 0,
30688 rep_count * sizeof(d->u.l.lens[i]));
30689 18 i += rep_count;
30690 }
30691 674 ▶ } while (i < num_litlen_syms + num_offset_syms);
Branch: 1 branch
BranchTrueFalseTotal
} while (i < num_litlen_syms + num_offset_syms); 669 5 674
30692  
30693
30694 5 ▶ SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(i == num_litlen_syms + num_offset_syms); 0 5 5
30695  
30696 36 ▶ } else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) { 0 36 36
30697 u16 len, nlen;
30698  
30699
30700  
30701 0 bitsleft -= 3;
30702  
30703
30704 0 bitsleft = (u8)bitsleft;
30705 0 ▶ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(overread_count <= (bitsleft >> 3)); 0 0 0
30706 0 in_next -= (bitsleft >> 3) - overread_count;
30707 0 overread_count = 0;
30708 0 bitbuf = 0;
30709 0 bitsleft = 0;
30710  
30711 0 ▶ SAFETY_CHECK(in_end - in_next >= 4);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(in_end - in_next >= 4); 0 0 0
30712 0 len = get_unaligned_le16(in_next);
30713 0 nlen = get_unaligned_le16(in_next + 2);
30714 0 in_next += 4;
30715  
30716 0 ▶ SAFETY_CHECK(len == (u16)~nlen);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(len == (u16)~nlen); 0 0 0
30717 0 ▶ if (unlikely(len > out_end - out_next))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(len > out_end - out_next)) 0 0 0
30718 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
30719 0 ▶ SAFETY_CHECK(len <= in_end - in_next);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(len <= in_end - in_next); 0 0 0
30720  
30721 0 memcpy(out_next, in_next, len);
30722 0 in_next += len;
30723 0 out_next += len;
30724  
30725 0 goto block_done;
30726  
30727 } else {
30728 unsigned i;
30729  
30730 36 ▶ SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN); 0 36 36
30731  
30732
30733  
30734 36 bitbuf >>= 3;
30735 36 bitsleft -= 3;
30736  
30737 36 ▶ if (d->static_codes_loaded)
Branch: 1 branch
BranchTrueFalseTotal
if (d->static_codes_loaded) 0 36 36
30738 0 goto have_decode_tables;
30739  
30740 36 d->static_codes_loaded = true;
30741  
30742 STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
30743 STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
30744  
30745 5220 ▶ for (i = 0; i < 144; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (i = 0; i < 144; i++) 5184 36 5220
30746 5184 d->u.l.lens[i] = 8;
30747 4068 ▶ for (; i < 256; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 256; i++) 4032 36 4068
30748 4032 d->u.l.lens[i] = 9;
30749 900 ▶ for (; i < 280; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 280; i++) 864 36 900
30750 864 d->u.l.lens[i] = 7;
30751 324 ▶ for (; i < 288; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 288; i++) 288 36 324
30752 288 d->u.l.lens[i] = 8;
30753  
30754 1188 ▶ for (; i < 288 + 32; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (; i < 288 + 32; i++) 1152 36 1188
30755 1152 d->u.l.lens[i] = 5;
30756  
30757 36 num_litlen_syms = 288;
30758 36 num_offset_syms = 32;
30759 }
30760  
30761
30762  
30763 41 ▶ SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms)); 0 41 41
30764 41 ▶ SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms)); 0 41 41
30765 41 have_decode_tables:
30766 41 litlen_tablemask = BITMASK(d->litlen_tablebits);
30767  
30768
30769 41 ▶ if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
Branch: 2 branches
BranchTrueFalseTotal
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end) 41 0 41
if (in_next >= in_fastloop_end || out_next >= out_fastloop_end) 0 41 41
30770 0 goto generic_loop;
30771 41 REFILL_BITS_IN_FASTLOOP();
30772 41 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30773 do {
30774 u32 length, offset, lit;
30775 const u8 *src;
30776 u8 *dst;
30777  
30778
30779 10151 saved_bitbuf = bitbuf;
30780 10151 bitbuf >>= (u8)entry;
30781 10151 bitsleft -= entry;
30782  
30783
30784 10151 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 3886 6265 10151
30785
30786 if (
30787 CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
30788 LENGTH_MAXBITS,
30789 OFFSET_TABLEBITS) &&
30790
30791 CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
30792 DEFLATE_MAX_LITLEN_CODEWORD_LEN,
30793 LITLEN_TABLEBITS)) {
30794
30795 3886 lit = entry >> 16;
30796 3886 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30797 3886 saved_bitbuf = bitbuf;
30798 3886 bitbuf >>= (u8)entry;
30799 3886 bitsleft -= entry;
30800 3886 *out_next++ = lit;
30801 3886 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 2617 1269 3886
30802
30803 2617 lit = entry >> 16;
30804 2617 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30805 2617 saved_bitbuf = bitbuf;
30806 2617 bitbuf >>= (u8)entry;
30807 2617 bitsleft -= entry;
30808 2617 *out_next++ = lit;
30809 2617 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 1951 666 2617
30810
30811 1951 lit = entry >> 16;
30812 1951 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30813 1951 REFILL_BITS_IN_FASTLOOP();
30814 1951 *out_next++ = lit;
30815 1951 continue;
30816 }
30817 }
30818 } else {
30819
30820 STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
30821 LITLEN_TABLEBITS, LITLEN_TABLEBITS));
30822 lit = entry >> 16;
30823 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30824 REFILL_BITS_IN_FASTLOOP();
30825 *out_next++ = lit;
30826 continue;
30827 }
30828 }
30829  
30830
30831 8200 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 28 8172 8200
30832
30833  
30834 28 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 1 27 28
30835 1 goto block_done;
30836  
30837
30838 54 entry = d->u.litlen_decode_table[(entry >> 16) +
30839 27 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
30840 27 saved_bitbuf = bitbuf;
30841 27 bitbuf >>= (u8)entry;
30842 27 bitsleft -= entry;
30843  
30844
30845 if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
30846 LITLEN_TABLEBITS) ||
30847 !CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
30848 OFFSET_TABLEBITS))
30849 REFILL_BITS_IN_FASTLOOP();
30850 27 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 27 0 27
30851
30852 27 lit = entry >> 16;
30853 27 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30854 27 REFILL_BITS_IN_FASTLOOP();
30855 27 *out_next++ = lit;
30856 27 continue;
30857 }
30858 0 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 0 0 0
30859 0 goto block_done;
30860
30861 }
30862  
30863
30864 8172 length = entry >> 16;
30865 8172 length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
30866  
30867
30868 STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
30869 OFFSET_TABLEBITS));
30870 8172 entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
30871 if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
30872 LITLEN_TABLEBITS)) {
30873
30874 8172 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 17 8155 8172
30875
30876 17 ▶ if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely((u8)bitsleft < OFFSET_MAXBITS + 0 17 17
30877 LITLEN_TABLEBITS - PRELOAD_SLACK))
30878 0 REFILL_BITS_IN_FASTLOOP();
30879 17 bitbuf >>= OFFSET_TABLEBITS;
30880 17 bitsleft -= OFFSET_TABLEBITS;
30881 34 entry = d->offset_decode_table[(entry >> 16) +
30882 34 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
30883 8155 ▶ } else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
Branch: 1 branch
BranchTrueFalseTotal
} else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS + 1 8154 8155
30884 LITLEN_TABLEBITS - PRELOAD_SLACK))
30885 1 REFILL_BITS_IN_FASTLOOP();
30886 } else {
30887
30888 REFILL_BITS_IN_FASTLOOP();
30889 if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
30890
30891 bitbuf >>= OFFSET_TABLEBITS;
30892 bitsleft -= OFFSET_TABLEBITS;
30893 entry = d->offset_decode_table[(entry >> 16) +
30894 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
30895 REFILL_BITS_IN_FASTLOOP();
30896
30897 STATIC_ASSERT(CAN_CONSUME(
30898 OFFSET_MAXBITS - OFFSET_TABLEBITS));
30899 } else {
30900
30901 STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
30902 }
30903 }
30904 8172 saved_bitbuf = bitbuf;
30905 8172 bitbuf >>= (u8)entry;
30906 8172 bitsleft -= entry;
30907 8172 offset = entry >> 16;
30908 8172 offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
30909  
30910
30911 8172 ▶ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(offset <= out_next - (const u8 *)out); 0 8172 8172
30912 8172 src = out_next - offset;
30913 8172 dst = out_next;
30914 8172 out_next += length;
30915  
30916
30917 if (!CAN_CONSUME_AND_THEN_PRELOAD(
30918 MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
30919 OFFSET_MAXFASTBITS),
30920 LITLEN_TABLEBITS) &&
30921 unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
30922 REFILL_BITS_IN_FASTLOOP();
30923 8172 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
30924 8172 REFILL_BITS_IN_FASTLOOP();
30925  
30926
30927 8172 ▶ if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
Branch: 1 branch
BranchTrueFalseTotal
if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) { 8163 9 8172
30928 8163 store_word_unaligned(load_word_unaligned(src), dst);
30929 8163 src += WORDBYTES;
30930 8163 dst += WORDBYTES;
30931 8163 store_word_unaligned(load_word_unaligned(src), dst);
30932 8163 src += WORDBYTES;
30933 8163 dst += WORDBYTES;
30934 8163 store_word_unaligned(load_word_unaligned(src), dst);
30935 8163 src += WORDBYTES;
30936 8163 dst += WORDBYTES;
30937 8163 store_word_unaligned(load_word_unaligned(src), dst);
30938 8163 src += WORDBYTES;
30939 8163 dst += WORDBYTES;
30940 8163 store_word_unaligned(load_word_unaligned(src), dst);
30941 8163 src += WORDBYTES;
30942 8163 dst += WORDBYTES;
30943 8430 ▶ while (dst < out_next) {
Branch: 1 branch
BranchTrueFalseTotal
while (dst < out_next) { 267 8163 8430
30944 267 store_word_unaligned(load_word_unaligned(src), dst);
30945 267 src += WORDBYTES;
30946 267 dst += WORDBYTES;
30947 267 store_word_unaligned(load_word_unaligned(src), dst);
30948 267 src += WORDBYTES;
30949 267 dst += WORDBYTES;
30950 267 store_word_unaligned(load_word_unaligned(src), dst);
30951 267 src += WORDBYTES;
30952 267 dst += WORDBYTES;
30953 267 store_word_unaligned(load_word_unaligned(src), dst);
30954 267 src += WORDBYTES;
30955 267 dst += WORDBYTES;
30956 267 store_word_unaligned(load_word_unaligned(src), dst);
30957 267 src += WORDBYTES;
30958 267 dst += WORDBYTES;
30959 }
30960 9 ▶ } else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) { 0 9 9
30961 machine_word_t v;
30962  
30963
30964 0 v = (machine_word_t)0x0101010101010101 * src[0];
30965 0 store_word_unaligned(v, dst);
30966 0 dst += WORDBYTES;
30967 0 store_word_unaligned(v, dst);
30968 0 dst += WORDBYTES;
30969 0 store_word_unaligned(v, dst);
30970 0 dst += WORDBYTES;
30971 0 store_word_unaligned(v, dst);
30972 0 dst += WORDBYTES;
30973 0 ▶ while (dst < out_next) {
Branch: 1 branch
BranchTrueFalseTotal
while (dst < out_next) { 0 0 0
30974 0 store_word_unaligned(v, dst);
30975 0 dst += WORDBYTES;
30976 0 store_word_unaligned(v, dst);
30977 0 dst += WORDBYTES;
30978 0 store_word_unaligned(v, dst);
30979 0 dst += WORDBYTES;
30980 0 store_word_unaligned(v, dst);
30981 0 dst += WORDBYTES;
30982 }
30983 } else if (UNALIGNED_ACCESS_IS_FAST) {
30984 9 store_word_unaligned(load_word_unaligned(src), dst);
30985 9 src += offset;
30986 9 dst += offset;
30987 9 store_word_unaligned(load_word_unaligned(src), dst);
30988 9 src += offset;
30989 9 dst += offset;
30990 do {
30991 9 store_word_unaligned(load_word_unaligned(src), dst);
30992 9 src += offset;
30993 9 dst += offset;
30994 9 store_word_unaligned(load_word_unaligned(src), dst);
30995 9 src += offset;
30996 9 dst += offset;
30997 9 ▶ } while (dst < out_next);
Branch: 1 branch
BranchTrueFalseTotal
} while (dst < out_next); 0 9 9
30998 } else {
30999 *dst++ = *src++;
31000 *dst++ = *src++;
31001 do {
31002 *dst++ = *src++;
31003 } while (dst < out_next);
31004 }
31005 10150 ▶ } while (in_next < in_fastloop_end && out_next < out_fastloop_end);
Branch: 2 branches
BranchTrueFalseTotal
} while (in_next < in_fastloop_end && out_next < out_fastloop_end); 36 10114 10150
} while (in_next < in_fastloop_end && out_next < out_fastloop_end); 10110 4 10114
31006  
31007
31008 40 generic_loop:
31009 930 for (;;) {
31010 u32 length, offset;
31011 const u8 *src;
31012 u8 *dst;
31013  
31014 2172 ▶ REFILL_BITS();
Branch: 4 branches
BranchTrueFalseTotal
REFILL_BITS(); 722 248 970
REFILL_BITS(); 240 240 480
REFILL_BITS(); 0 240 240
REFILL_BITS(); 480 248 728
31015 970 entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
31016 970 saved_bitbuf = bitbuf;
31017 970 bitbuf >>= (u8)entry;
31018 970 bitsleft -= entry;
31019 970 ▶ if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) { 3 967 970
31020 6 entry = d->u.litlen_decode_table[(entry >> 16) +
31021 3 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
31022 3 saved_bitbuf = bitbuf;
31023 3 bitbuf >>= (u8)entry;
31024 3 bitsleft -= entry;
31025 }
31026 970 length = entry >> 16;
31027 970 ▶ if (entry & HUFFDEC_LITERAL) {
Branch: 1 branch
BranchTrueFalseTotal
if (entry & HUFFDEC_LITERAL) { 588 382 970
31028 588 ▶ if (unlikely(out_next == out_end))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(out_next == out_end)) 0 588 588
31029 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
31030 588 *out_next++ = length;
31031 588 continue;
31032 }
31033 382 ▶ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_END_OF_BLOCK)) 40 342 382
31034 40 goto block_done;
31035 342 length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
31036 342 ▶ if (unlikely(length > out_end - out_next))
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(length > out_end - out_next)) 0 342 342
31037 0 return LIBDEFLATE_INSUFFICIENT_SPACE;
31038  
31039 if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
31040 REFILL_BITS();
31041 342 entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
31042 342 ▶ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
Branch: 1 branch
BranchTrueFalseTotal
if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) { 0 342 342
31043 0 bitbuf >>= OFFSET_TABLEBITS;
31044 0 bitsleft -= OFFSET_TABLEBITS;
31045 0 entry = d->offset_decode_table[(entry >> 16) +
31046 0 EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
31047 if (!CAN_CONSUME(OFFSET_MAXBITS))
31048 REFILL_BITS();
31049 }
31050 342 offset = entry >> 16;
31051 342 offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
31052 342 bitbuf >>= (u8)entry;
31053 342 bitsleft -= entry;
31054  
31055 342 ▶ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(offset <= out_next - (const u8 *)out); 0 342 342
31056 342 src = out_next - offset;
31057 342 dst = out_next;
31058 342 out_next += length;
31059  
31060 STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
31061 342 *dst++ = *src++;
31062 342 *dst++ = *src++;
31063 do {
31064 60992 *dst++ = *src++;
31065 60992 ▶ } while (dst < out_next);
Branch: 1 branch
BranchTrueFalseTotal
} while (dst < out_next); 60650 342 60992
31066 }
31067  
31068 41 block_done:
31069
31070  
31071 41 ▶ if (!is_final_block)
Branch: 1 branch
BranchTrueFalseTotal
if (!is_final_block) 1 40 41
31072 1 goto next_block;
31073  
31074
31075  
31076 40 bitsleft = (u8)bitsleft;
31077  
31078
31079 40 ▶ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
Branch: 1 branch
BranchTrueFalseTotal
SAFETY_CHECK(overread_count <= (bitsleft >> 3)); 0 40 40
31080  
31081
31082 40 ▶ if (actual_in_nbytes_ret) {
Branch: 1 branch
BranchTrueFalseTotal
if (actual_in_nbytes_ret) { 40 0 40
31083
31084 40 in_next -= (bitsleft >> 3) - overread_count;
31085  
31086 40 *actual_in_nbytes_ret = in_next - (u8 *)in;
31087 }
31088  
31089
31090 40 ▶ if (actual_out_nbytes_ret) {
Branch: 1 branch
BranchTrueFalseTotal
if (actual_out_nbytes_ret) { 40 0 40
31091 40 *actual_out_nbytes_ret = out_next - (u8 *)out;
31092 } else {
31093 0 ▶ if (out_next != out_end)
Branch: 1 branch
BranchTrueFalseTotal
if (out_next != out_end) 0 0 0
31094 0 return LIBDEFLATE_SHORT_OUTPUT;
31095 }
31096 40 return LIBDEFLATE_SUCCESS;
31097 }
31098  
31099 #undef FUNCNAME
31100 #undef ATTRIBUTES
31101 #undef EXTRACT_VARBITS
31102 #undef EXTRACT_VARBITS8
31103  
31104 #endif
31105  
31106 #if defined(deflate_decompress_bmi2) && HAVE_BMI2_NATIVE
31107 #define DEFAULT_IMPL deflate_decompress_bmi2
31108 #else
31109 static inline decompress_func_t
31110 2 arch_select_decompress_func(void)
31111 {
31112 #ifdef deflate_decompress_bmi2
31113 2 ▶ if (HAVE_BMI2(get_x86_cpu_features()))
Branch: 1 branch
BranchTrueFalseTotal
if (HAVE_BMI2(get_x86_cpu_features())) 2 0 2
31114 2 return deflate_decompress_bmi2;
31115 #endif
31116 0 return NULL;
31117 }
31118 #define arch_select_decompress_func arch_select_decompress_func
31119 #endif
31120  
31121 #endif
31122  
31123 #endif
31124  
31125 #ifndef DEFAULT_IMPL
31126 # define DEFAULT_IMPL deflate_decompress_default
31127 #endif
31128  
31129 #ifdef arch_select_decompress_func
31130 static enum libdeflate_result
31131 dispatch_decomp(struct libdeflate_decompressor * restrict d,
31132 const void * restrict in, size_t in_nbytes,
31133 void * restrict out, size_t out_nbytes_avail,
31134 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
31135  
31136 static volatile decompress_func_t decompress_impl = dispatch_decomp;
31137  
31138  
31139 static enum libdeflate_result
31140 2 dispatch_decomp(struct libdeflate_decompressor * restrict d,
31141 const void * restrict in, size_t in_nbytes,
31142 void * restrict out, size_t out_nbytes_avail,
31143 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
31144 {
31145 2 decompress_func_t f = arch_select_decompress_func();
31146  
31147 2 ▶ if (f == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (f == NULL) 0 2 2
31148 0 f = DEFAULT_IMPL;
31149  
31150 2 decompress_impl = f;
31151 2 return f(d, in, in_nbytes, out, out_nbytes_avail,
31152 actual_in_nbytes_ret, actual_out_nbytes_ret);
31153 }
31154 #else
31155  
31156 # define decompress_impl DEFAULT_IMPL
31157 #endif
31158  
31159  
31160 LIBDEFLATEAPI enum libdeflate_result
31161 40 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *d,
31162 const void *in, size_t in_nbytes,
31163 void *out, size_t out_nbytes_avail,
31164 size_t *actual_in_nbytes_ret,
31165 size_t *actual_out_nbytes_ret)
31166 {
31167 40 return decompress_impl(d, in, in_nbytes, out, out_nbytes_avail,
31168 actual_in_nbytes_ret, actual_out_nbytes_ret);
31169 }
31170  
31171 LIBDEFLATEAPI enum libdeflate_result
31172 0 libdeflate_deflate_decompress(struct libdeflate_decompressor *d,
31173 const void *in, size_t in_nbytes,
31174 void *out, size_t out_nbytes_avail,
31175 size_t *actual_out_nbytes_ret)
31176 {
31177 0 return libdeflate_deflate_decompress_ex(d, in, in_nbytes,
31178 out, out_nbytes_avail,
31179 NULL, actual_out_nbytes_ret);
31180 }
31181  
31182 LIBDEFLATEAPI struct libdeflate_decompressor *
31183 38 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options)
31184 {
31185 struct libdeflate_decompressor *d;
31186  
31187
31188 38 ▶ if (options->sizeof_options != sizeof(*options))
Branch: 1 branch
BranchTrueFalseTotal
if (options->sizeof_options != sizeof(*options)) 0 38 38
31189 0 return NULL;
31190  
31191 38 ▶ d = (options->malloc_func ? options->malloc_func :
Branch: 1 branch
BranchTrueFalseTotal
d = (options->malloc_func ? options->malloc_func : 0 38 38
31192 libdeflate_default_malloc_func)(sizeof(*d));
31193 38 ▶ if (d == NULL)
Branch: 1 branch
BranchTrueFalseTotal
if (d == NULL) 0 38 38
31194 0 return NULL;
31195
31196 38 memset(d, 0, sizeof(*d));
31197 76 d->free_func = options->free_func ?
31198 38 ▶ options->free_func : libdeflate_default_free_func;
Branch: 1 branch
BranchTrueFalseTotal
options->free_func : libdeflate_default_free_func; 0 38 38
31199 38 return d;
31200 }
31201  
31202 LIBDEFLATEAPI struct libdeflate_decompressor *
31203 38 libdeflate_alloc_decompressor(void)
31204 {
31205 static const struct libdeflate_options defaults = {
31206 .sizeof_options = sizeof(defaults),
31207 };
31208 38 return libdeflate_alloc_decompressor_ex(&defaults);
31209 }
31210  
31211 LIBDEFLATEAPI void
31212 38 libdeflate_free_decompressor(struct libdeflate_decompressor *d)
31213 {
31214 38 ▶ if (d)
Branch: 1 branch
BranchTrueFalseTotal
if (d) 38 0 38
31215 38 d->free_func(d);
31216 38 }
31217 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/gzip_compress.c */
31218  
31219  
31220 /* #include "deflate_compress.h" */
31221 #ifndef LIB_DEFLATE_COMPRESS_H
31222 #define LIB_DEFLATE_COMPRESS_H
31223  
31224 /* #include "lib_common.h" */
31225  
31226  
31227 #ifndef LIB_LIB_COMMON_H
31228 #define LIB_LIB_COMMON_H
31229  
31230 #ifdef LIBDEFLATE_H
31231
31232 # error "lib_common.h must always be included before libdeflate.h"
31233 #endif
31234  
31235 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
31236 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
31237 #elif defined(__GNUC__)
31238 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
31239 #else
31240 # define LIBDEFLATE_EXPORT_SYM
31241 #endif
31242  
31243  
31244 #if defined(__GNUC__) && defined(__i386__)
31245 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
31246 #else
31247 # define LIBDEFLATE_ALIGN_STACK
31248 #endif
31249  
31250 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
31251  
31252 /* #include "../common_defs.h" */
31253  
31254  
31255 #ifndef COMMON_DEFS_H
31256 #define COMMON_DEFS_H
31257  
31258 /* #include "libdeflate.h" */
31259  
31260  
31261 #ifndef LIBDEFLATE_H
31262 #define LIBDEFLATE_H
31263  
31264 #include <stddef.h>
31265 #include <stdint.h>
31266  
31267 #ifdef __cplusplus
31268 extern "C" {
31269 #endif
31270  
31271 #define LIBDEFLATE_VERSION_MAJOR 1
31272 #define LIBDEFLATE_VERSION_MINOR 26
31273 #define LIBDEFLATE_VERSION_STRING "1.26"
31274  
31275  
31276 #ifndef LIBDEFLATEAPI
31277 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
31278 # define LIBDEFLATEAPI __declspec(dllimport)
31279 # else
31280 # define LIBDEFLATEAPI
31281 # endif
31282 #endif
31283  
31284  
31285  
31286  
31287  
31288 struct libdeflate_compressor;
31289 struct libdeflate_options;
31290  
31291  
31292 LIBDEFLATEAPI struct libdeflate_compressor *
31293 libdeflate_alloc_compressor(int compression_level);
31294  
31295  
31296 LIBDEFLATEAPI struct libdeflate_compressor *
31297 libdeflate_alloc_compressor_ex(int compression_level,
31298 const struct libdeflate_options *options);
31299  
31300  
31301 LIBDEFLATEAPI size_t
31302 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
31303 const void *in, size_t in_nbytes,
31304 void *out, size_t out_nbytes_avail);
31305  
31306  
31307 LIBDEFLATEAPI size_t
31308 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
31309 size_t in_nbytes);
31310  
31311  
31312 LIBDEFLATEAPI size_t
31313 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
31314 const void *in, size_t in_nbytes,
31315 void *out, size_t out_nbytes_avail);
31316  
31317  
31318 LIBDEFLATEAPI size_t
31319 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
31320 size_t in_nbytes);
31321  
31322  
31323 LIBDEFLATEAPI size_t
31324 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
31325 const void *in, size_t in_nbytes,
31326 void *out, size_t out_nbytes_avail);
31327  
31328  
31329 LIBDEFLATEAPI size_t
31330 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
31331 size_t in_nbytes);
31332  
31333  
31334 LIBDEFLATEAPI void
31335 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
31336  
31337  
31338  
31339  
31340  
31341 struct libdeflate_decompressor;
31342 struct libdeflate_options;
31343  
31344  
31345 LIBDEFLATEAPI struct libdeflate_decompressor *
31346 libdeflate_alloc_decompressor(void);
31347  
31348  
31349 LIBDEFLATEAPI struct libdeflate_decompressor *
31350 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
31351  
31352  
31353 enum libdeflate_result {
31354
31355 LIBDEFLATE_SUCCESS = 0,
31356  
31357
31358 LIBDEFLATE_BAD_DATA = 1,
31359  
31360
31361 LIBDEFLATE_SHORT_OUTPUT = 2,
31362  
31363
31364 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
31365 };
31366  
31367  
31368 LIBDEFLATEAPI enum libdeflate_result
31369 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
31370 const void *in, size_t in_nbytes,
31371 void *out, size_t out_nbytes_avail,
31372 size_t *actual_out_nbytes_ret);
31373  
31374  
31375 LIBDEFLATEAPI enum libdeflate_result
31376 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
31377 const void *in, size_t in_nbytes,
31378 void *out, size_t out_nbytes_avail,
31379 size_t *actual_in_nbytes_ret,
31380 size_t *actual_out_nbytes_ret);
31381  
31382  
31383 LIBDEFLATEAPI enum libdeflate_result
31384 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
31385 const void *in, size_t in_nbytes,
31386 void *out, size_t out_nbytes_avail,
31387 size_t *actual_out_nbytes_ret);
31388  
31389  
31390 LIBDEFLATEAPI enum libdeflate_result
31391 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
31392 const void *in, size_t in_nbytes,
31393 void *out, size_t out_nbytes_avail,
31394 size_t *actual_in_nbytes_ret,
31395 size_t *actual_out_nbytes_ret);
31396  
31397  
31398 LIBDEFLATEAPI enum libdeflate_result
31399 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
31400 const void *in, size_t in_nbytes,
31401 void *out, size_t out_nbytes_avail,
31402 size_t *actual_out_nbytes_ret);
31403  
31404  
31405 LIBDEFLATEAPI enum libdeflate_result
31406 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
31407 const void *in, size_t in_nbytes,
31408 void *out, size_t out_nbytes_avail,
31409 size_t *actual_in_nbytes_ret,
31410 size_t *actual_out_nbytes_ret);
31411  
31412  
31413 LIBDEFLATEAPI void
31414 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
31415  
31416  
31417  
31418  
31419  
31420  
31421 LIBDEFLATEAPI uint32_t
31422 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
31423  
31424  
31425  
31426 LIBDEFLATEAPI uint32_t
31427 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
31428  
31429  
31430  
31431  
31432  
31433  
31434 LIBDEFLATEAPI void
31435 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
31436 void (*free_func)(void *));
31437  
31438  
31439 struct libdeflate_options {
31440  
31441
31442 size_t sizeof_options;
31443  
31444
31445 void *(*malloc_func)(size_t);
31446 void (*free_func)(void *);
31447 };
31448  
31449 #ifdef __cplusplus
31450 }
31451 #endif
31452  
31453 #endif
31454  
31455  
31456 #include <stdbool.h>
31457 #include <stddef.h>
31458 #include <stdint.h>
31459 #ifdef _MSC_VER
31460 # include <intrin.h>
31461 # include <stdlib.h>
31462
31463
31464 # pragma warning(disable : 4146)
31465
31466 # pragma warning(disable : 4018)
31467 # pragma warning(disable : 4244)
31468 # pragma warning(disable : 4267)
31469 # pragma warning(disable : 4310)
31470
31471 # pragma warning(disable : 4100)
31472 # pragma warning(disable : 4127)
31473 # pragma warning(disable : 4189)
31474 # pragma warning(disable : 4232)
31475 # pragma warning(disable : 4245)
31476 # pragma warning(disable : 4295)
31477 #endif
31478 #ifndef FREESTANDING
31479 # include <string.h>
31480 #endif
31481  
31482  
31483  
31484  
31485  
31486  
31487 #undef ARCH_X86_64
31488 #undef ARCH_X86_32
31489 #undef ARCH_ARM64
31490 #undef ARCH_ARM32
31491 #undef ARCH_RISCV
31492 #ifdef _MSC_VER
31493
31494 # if defined(_M_X64) && !defined(_M_ARM64EC)
31495 # define ARCH_X86_64
31496 # elif defined(_M_IX86)
31497 # define ARCH_X86_32
31498 # elif defined(_M_ARM64)
31499 # define ARCH_ARM64
31500 # elif defined(_M_ARM)
31501 # define ARCH_ARM32
31502 # endif
31503 #else
31504 # if defined(__x86_64__)
31505 # define ARCH_X86_64
31506 # elif defined(__i386__)
31507 # define ARCH_X86_32
31508 # elif defined(__aarch64__)
31509 # define ARCH_ARM64
31510 # elif defined(__arm__)
31511 # define ARCH_ARM32
31512 # elif defined(__riscv)
31513 # define ARCH_RISCV
31514 # endif
31515 #endif
31516  
31517  
31518  
31519  
31520  
31521  
31522 typedef uint8_t u8;
31523 typedef uint16_t u16;
31524 typedef uint32_t u32;
31525 typedef uint64_t u64;
31526 typedef int8_t s8;
31527 typedef int16_t s16;
31528 typedef int32_t s32;
31529 typedef int64_t s64;
31530  
31531  
31532 #ifdef _MSC_VER
31533 # ifdef _WIN64
31534 typedef long long ssize_t;
31535 # else
31536 typedef long ssize_t;
31537 # endif
31538 #endif
31539  
31540  
31541 typedef size_t machine_word_t;
31542  
31543  
31544 #define WORDBYTES ((int)sizeof(machine_word_t))
31545  
31546  
31547 #define WORDBITS (8 * WORDBYTES)
31548  
31549  
31550  
31551  
31552  
31553  
31554 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
31555 # define GCC_PREREQ(major, minor) \
31556 (__GNUC__ > (major) || \
31557 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
31558 # if !GCC_PREREQ(4, 9)
31559 # error "gcc versions older than 4.9 are no longer supported"
31560 # endif
31561 #else
31562 # define GCC_PREREQ(major, minor) 0
31563 #endif
31564 #ifdef __clang__
31565 # ifdef __apple_build_version__
31566 # define CLANG_PREREQ(major, minor, apple_version) \
31567 (__apple_build_version__ >= (apple_version))
31568 # else
31569 # define CLANG_PREREQ(major, minor, apple_version) \
31570 (__clang_major__ > (major) || \
31571 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
31572 # endif
31573 # if !CLANG_PREREQ(3, 9, 8000000)
31574 # error "clang versions older than 3.9 are no longer supported"
31575 # endif
31576 #else
31577 # define CLANG_PREREQ(major, minor, apple_version) 0
31578 #endif
31579 #ifdef _MSC_VER
31580 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
31581 # if !MSVC_PREREQ(1928)
31582 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
31583 # endif
31584 #else
31585 # define MSVC_PREREQ(version) 0
31586 #endif
31587  
31588  
31589 #ifndef __has_attribute
31590 # define __has_attribute(attribute) 0
31591 #endif
31592  
31593  
31594 #ifndef __has_builtin
31595 # define __has_builtin(builtin) 0
31596 #endif
31597  
31598  
31599 #ifdef _MSC_VER
31600 # define inline __inline
31601 #endif
31602  
31603  
31604 #if defined(__GNUC__) || __has_attribute(always_inline)
31605 # define forceinline inline __attribute__((always_inline))
31606 #elif defined(_MSC_VER)
31607 # define forceinline __forceinline
31608 #else
31609 # define forceinline inline
31610 #endif
31611  
31612  
31613 #if defined(__GNUC__) || __has_attribute(unused)
31614 # define MAYBE_UNUSED __attribute__((unused))
31615 #else
31616 # define MAYBE_UNUSED
31617 #endif
31618  
31619  
31620 #if defined(__GNUC__) || __has_attribute(noreturn)
31621 # define NORETURN __attribute__((noreturn))
31622 #else
31623 # define NORETURN
31624 #endif
31625  
31626  
31627 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
31628 # define likely(expr) __builtin_expect(!!(expr), 1)
31629 #else
31630 # define likely(expr) (expr)
31631 #endif
31632  
31633  
31634 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
31635 # define unlikely(expr) __builtin_expect(!!(expr), 0)
31636 #else
31637 # define unlikely(expr) (expr)
31638 #endif
31639  
31640  
31641 #undef prefetchr
31642 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
31643 # define prefetchr(addr) __builtin_prefetch((addr), 0)
31644 #elif defined(_MSC_VER)
31645 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
31646 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
31647 # elif defined(ARCH_ARM64)
31648 # define prefetchr(addr) __prefetch2((addr), 0x00 )
31649 # elif defined(ARCH_ARM32)
31650 # define prefetchr(addr) __prefetch(addr)
31651 # endif
31652 #endif
31653 #ifndef prefetchr
31654 # define prefetchr(addr)
31655 #endif
31656  
31657  
31658 #undef prefetchw
31659 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
31660 # define prefetchw(addr) __builtin_prefetch((addr), 1)
31661 #elif defined(_MSC_VER)
31662 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
31663 # define prefetchw(addr) _m_prefetchw(addr)
31664 # elif defined(ARCH_ARM64)
31665 # define prefetchw(addr) __prefetch2((addr), 0x10 )
31666 # elif defined(ARCH_ARM32)
31667 # define prefetchw(addr) __prefetchw(addr)
31668 # endif
31669 #endif
31670 #ifndef prefetchw
31671 # define prefetchw(addr)
31672 #endif
31673  
31674  
31675 #undef _aligned_attribute
31676 #if defined(__GNUC__) || __has_attribute(aligned)
31677 # define _aligned_attribute(n) __attribute__((aligned(n)))
31678 #elif defined(_MSC_VER)
31679 # define _aligned_attribute(n) __declspec(align(n))
31680 #endif
31681  
31682  
31683 #if defined(__GNUC__) || __has_attribute(target)
31684 # define _target_attribute(attrs) __attribute__((target(attrs)))
31685 #else
31686 # define _target_attribute(attrs)
31687 #endif
31688  
31689  
31690  
31691  
31692  
31693 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
31694 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
31695 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
31696 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
31697 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
31698 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
31699 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
31700  
31701  
31702  
31703  
31704  
31705  
31706 #if defined(__BYTE_ORDER__)
31707 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
31708 #elif defined(_MSC_VER)
31709 # define CPU_IS_LITTLE_ENDIAN() true
31710 #else
31711 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
31712 {
31713 union {
31714 u32 w;
31715 u8 b;
31716 } u;
31717  
31718 u.w = 1;
31719 return u.b;
31720 }
31721 #endif
31722  
31723  
31724 static forceinline u16 bswap16(u16 v)
31725 {
31726 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
31727 return __builtin_bswap16(v);
31728 #elif defined(_MSC_VER)
31729 return _byteswap_ushort(v);
31730 #else
31731 return (v << 8) | (v >> 8);
31732 #endif
31733 }
31734  
31735  
31736 static forceinline u32 bswap32(u32 v)
31737 {
31738 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
31739 return __builtin_bswap32(v);
31740 #elif defined(_MSC_VER)
31741 return _byteswap_ulong(v);
31742 #else
31743 return ((v & 0x000000FF) << 24) |
31744 ((v & 0x0000FF00) << 8) |
31745 ((v & 0x00FF0000) >> 8) |
31746 ((v & 0xFF000000) >> 24);
31747 #endif
31748 }
31749  
31750  
31751 static forceinline u64 bswap64(u64 v)
31752 {
31753 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
31754 return __builtin_bswap64(v);
31755 #elif defined(_MSC_VER)
31756 return _byteswap_uint64(v);
31757 #else
31758 return ((v & 0x00000000000000FF) << 56) |
31759 ((v & 0x000000000000FF00) << 40) |
31760 ((v & 0x0000000000FF0000) << 24) |
31761 ((v & 0x00000000FF000000) << 8) |
31762 ((v & 0x000000FF00000000) >> 8) |
31763 ((v & 0x0000FF0000000000) >> 24) |
31764 ((v & 0x00FF000000000000) >> 40) |
31765 ((v & 0xFF00000000000000) >> 56);
31766 #endif
31767 }
31768  
31769 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
31770 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
31771 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
31772 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
31773 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
31774 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
31775  
31776  
31777  
31778  
31779  
31780  
31781 #if (defined(__GNUC__) || defined(__clang__)) && \
31782 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
31783 defined(__ARM_FEATURE_UNALIGNED) || \
31784 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
31785 defined(__riscv_misaligned_fast) || \
31786 defined(__wasm__))
31787 # define UNALIGNED_ACCESS_IS_FAST 1
31788 #elif defined(_MSC_VER)
31789 # define UNALIGNED_ACCESS_IS_FAST 1
31790 #else
31791 # define UNALIGNED_ACCESS_IS_FAST 0
31792 #endif
31793  
31794  
31795  
31796 #ifdef FREESTANDING
31797 # define MEMCOPY __builtin_memcpy
31798 #else
31799 # define MEMCOPY memcpy
31800 #endif
31801  
31802  
31803  
31804 #define DEFINE_UNALIGNED_TYPE(type) \
31805 static forceinline type \
31806 load_##type##_unaligned(const void *p) \
31807 { \
31808 type v; \
31809 \
31810 MEMCOPY(&v, p, sizeof(v)); \
31811 return v; \
31812 } \
31813 \
31814 static forceinline void \
31815 store_##type##_unaligned(type v, void *p) \
31816 { \
31817 MEMCOPY(p, &v, sizeof(v)); \
31818 }
31819  
31820 DEFINE_UNALIGNED_TYPE(u16)
31821 DEFINE_UNALIGNED_TYPE(u32)
31822 DEFINE_UNALIGNED_TYPE(u64)
31823 DEFINE_UNALIGNED_TYPE(machine_word_t)
31824  
31825 #undef MEMCOPY
31826  
31827 #define load_word_unaligned load_machine_word_t_unaligned
31828 #define store_word_unaligned store_machine_word_t_unaligned
31829  
31830  
31831  
31832 static forceinline u16
31833 get_unaligned_le16(const u8 *p)
31834 {
31835 if (UNALIGNED_ACCESS_IS_FAST)
31836 return le16_bswap(load_u16_unaligned(p));
31837 else
31838 return ((u16)p[1] << 8) | p[0];
31839 }
31840  
31841 static forceinline u16
31842 get_unaligned_be16(const u8 *p)
31843 {
31844 if (UNALIGNED_ACCESS_IS_FAST)
31845 return be16_bswap(load_u16_unaligned(p));
31846 else
31847 return ((u16)p[0] << 8) | p[1];
31848 }
31849  
31850 static forceinline u32
31851 get_unaligned_le32(const u8 *p)
31852 {
31853 if (UNALIGNED_ACCESS_IS_FAST)
31854 return le32_bswap(load_u32_unaligned(p));
31855 else
31856 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
31857 ((u32)p[1] << 8) | p[0];
31858 }
31859  
31860 static forceinline u32
31861 get_unaligned_be32(const u8 *p)
31862 {
31863 if (UNALIGNED_ACCESS_IS_FAST)
31864 return be32_bswap(load_u32_unaligned(p));
31865 else
31866 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
31867 ((u32)p[2] << 8) | p[3];
31868 }
31869  
31870 static forceinline u64
31871 get_unaligned_le64(const u8 *p)
31872 {
31873 if (UNALIGNED_ACCESS_IS_FAST)
31874 return le64_bswap(load_u64_unaligned(p));
31875 else
31876 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
31877 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
31878 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
31879 ((u64)p[1] << 8) | p[0];
31880 }
31881  
31882 static forceinline machine_word_t
31883 get_unaligned_leword(const u8 *p)
31884 {
31885 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
31886 if (WORDBITS == 32)
31887 return get_unaligned_le32(p);
31888 else
31889 return get_unaligned_le64(p);
31890 }
31891  
31892  
31893  
31894 static forceinline void
31895 put_unaligned_le16(u16 v, u8 *p)
31896 {
31897 if (UNALIGNED_ACCESS_IS_FAST) {
31898 store_u16_unaligned(le16_bswap(v), p);
31899 } else {
31900 p[0] = (u8)(v >> 0);
31901 p[1] = (u8)(v >> 8);
31902 }
31903 }
31904  
31905 static forceinline void
31906 put_unaligned_be16(u16 v, u8 *p)
31907 {
31908 if (UNALIGNED_ACCESS_IS_FAST) {
31909 store_u16_unaligned(be16_bswap(v), p);
31910 } else {
31911 p[0] = (u8)(v >> 8);
31912 p[1] = (u8)(v >> 0);
31913 }
31914 }
31915  
31916 static forceinline void
31917 put_unaligned_le32(u32 v, u8 *p)
31918 {
31919 if (UNALIGNED_ACCESS_IS_FAST) {
31920 store_u32_unaligned(le32_bswap(v), p);
31921 } else {
31922 p[0] = (u8)(v >> 0);
31923 p[1] = (u8)(v >> 8);
31924 p[2] = (u8)(v >> 16);
31925 p[3] = (u8)(v >> 24);
31926 }
31927 }
31928  
31929 static forceinline void
31930 put_unaligned_be32(u32 v, u8 *p)
31931 {
31932 if (UNALIGNED_ACCESS_IS_FAST) {
31933 store_u32_unaligned(be32_bswap(v), p);
31934 } else {
31935 p[0] = (u8)(v >> 24);
31936 p[1] = (u8)(v >> 16);
31937 p[2] = (u8)(v >> 8);
31938 p[3] = (u8)(v >> 0);
31939 }
31940 }
31941  
31942 static forceinline void
31943 put_unaligned_le64(u64 v, u8 *p)
31944 {
31945 if (UNALIGNED_ACCESS_IS_FAST) {
31946 store_u64_unaligned(le64_bswap(v), p);
31947 } else {
31948 p[0] = (u8)(v >> 0);
31949 p[1] = (u8)(v >> 8);
31950 p[2] = (u8)(v >> 16);
31951 p[3] = (u8)(v >> 24);
31952 p[4] = (u8)(v >> 32);
31953 p[5] = (u8)(v >> 40);
31954 p[6] = (u8)(v >> 48);
31955 p[7] = (u8)(v >> 56);
31956 }
31957 }
31958  
31959 static forceinline void
31960 put_unaligned_leword(machine_word_t v, u8 *p)
31961 {
31962 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
31963 if (WORDBITS == 32)
31964 put_unaligned_le32(v, p);
31965 else
31966 put_unaligned_le64(v, p);
31967 }
31968  
31969  
31970  
31971  
31972  
31973  
31974  
31975 static forceinline unsigned
31976 bsr32(u32 v)
31977 {
31978 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
31979 return 31 - __builtin_clz(v);
31980 #elif defined(_MSC_VER)
31981 unsigned long i;
31982  
31983 _BitScanReverse(&i, v);
31984 return i;
31985 #else
31986 unsigned i = 0;
31987  
31988 while ((v >>= 1) != 0)
31989 i++;
31990 return i;
31991 #endif
31992 }
31993  
31994 static forceinline unsigned
31995 bsr64(u64 v)
31996 {
31997 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
31998 return 63 - __builtin_clzll(v);
31999 #elif defined(_MSC_VER) && defined(_WIN64)
32000 unsigned long i;
32001  
32002 _BitScanReverse64(&i, v);
32003 return i;
32004 #else
32005 unsigned i = 0;
32006  
32007 while ((v >>= 1) != 0)
32008 i++;
32009 return i;
32010 #endif
32011 }
32012  
32013 static forceinline unsigned
32014 bsrw(machine_word_t v)
32015 {
32016 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
32017 if (WORDBITS == 32)
32018 return bsr32(v);
32019 else
32020 return bsr64(v);
32021 }
32022  
32023  
32024  
32025 static forceinline unsigned
32026 bsf32(u32 v)
32027 {
32028 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
32029 return __builtin_ctz(v);
32030 #elif defined(_MSC_VER)
32031 unsigned long i;
32032  
32033 _BitScanForward(&i, v);
32034 return i;
32035 #else
32036 unsigned i = 0;
32037  
32038 for (; (v & 1) == 0; v >>= 1)
32039 i++;
32040 return i;
32041 #endif
32042 }
32043  
32044 static forceinline unsigned
32045 bsf64(u64 v)
32046 {
32047 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
32048 return __builtin_ctzll(v);
32049 #elif defined(_MSC_VER) && defined(_WIN64)
32050 unsigned long i;
32051  
32052 _BitScanForward64(&i, v);
32053 return i;
32054 #else
32055 unsigned i = 0;
32056  
32057 for (; (v & 1) == 0; v >>= 1)
32058 i++;
32059 return i;
32060 #endif
32061 }
32062  
32063 static forceinline unsigned
32064 bsfw(machine_word_t v)
32065 {
32066 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
32067 if (WORDBITS == 32)
32068 return bsf32(v);
32069 else
32070 return bsf64(v);
32071 }
32072  
32073  
32074 #undef rbit32
32075 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
32076 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
32077 static forceinline u32
32078 rbit32(u32 v)
32079 {
32080 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
32081 return v;
32082 }
32083 #define rbit32 rbit32
32084 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
32085 static forceinline u32
32086 rbit32(u32 v)
32087 {
32088 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
32089 return v;
32090 }
32091 #define rbit32 rbit32
32092 #endif
32093  
32094 #endif
32095  
32096  
32097 typedef void *(*malloc_func_t)(size_t);
32098 typedef void (*free_func_t)(void *);
32099  
32100 extern malloc_func_t libdeflate_default_malloc_func;
32101 extern free_func_t libdeflate_default_free_func;
32102  
32103 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
32104 size_t alignment, size_t size);
32105 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
32106  
32107 #ifdef FREESTANDING
32108  
32109 void *memset(void *s, int c, size_t n);
32110 #define memset(s, c, n) __builtin_memset((s), (c), (n))
32111  
32112 void *memcpy(void *dest, const void *src, size_t n);
32113 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
32114  
32115 void *memmove(void *dest, const void *src, size_t n);
32116 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
32117  
32118 int memcmp(const void *s1, const void *s2, size_t n);
32119 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
32120  
32121 #undef LIBDEFLATE_ENABLE_ASSERTIONS
32122 #else
32123 # include <string.h>
32124
32125 # ifdef __clang_analyzer__
32126 # define LIBDEFLATE_ENABLE_ASSERTIONS
32127 # endif
32128 #endif
32129  
32130  
32131 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
32132 NORETURN void
32133 libdeflate_assertion_failed(const char *expr, const char *file, int line);
32134 #define ASSERT(expr) { if (unlikely(!(expr))) \
32135 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
32136 #else
32137 #define ASSERT(expr) (void)(expr)
32138 #endif
32139  
32140 #define CONCAT_IMPL(a, b) a##b
32141 #define CONCAT(a, b) CONCAT_IMPL(a, b)
32142 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
32143  
32144 #endif
32145  
32146  
32147  
32148  
32149 struct libdeflate_compressor;
32150  
32151 unsigned int libdeflate_get_compression_level(struct libdeflate_compressor *c);
32152  
32153 #endif
32154  
32155 /* #include "gzip_constants.h" */
32156  
32157  
32158 #ifndef LIB_GZIP_CONSTANTS_H
32159 #define LIB_GZIP_CONSTANTS_H
32160  
32161 #define GZIP_MIN_HEADER_SIZE 10
32162 #define GZIP_FOOTER_SIZE 8
32163 #define GZIP_MIN_OVERHEAD (GZIP_MIN_HEADER_SIZE + GZIP_FOOTER_SIZE)
32164  
32165 #define GZIP_ID1 0x1F
32166 #define GZIP_ID2 0x8B
32167  
32168 #define GZIP_CM_DEFLATE 8
32169  
32170 #define GZIP_FTEXT 0x01
32171 #define GZIP_FHCRC 0x02
32172 #define GZIP_FEXTRA 0x04
32173 #define GZIP_FNAME 0x08
32174 #define GZIP_FCOMMENT 0x10
32175 #define GZIP_FRESERVED 0xE0
32176  
32177 #define GZIP_MTIME_UNAVAILABLE 0
32178  
32179 #define GZIP_XFL_SLOWEST_COMPRESSION 0x02
32180 #define GZIP_XFL_FASTEST_COMPRESSION 0x04
32181  
32182 #define GZIP_OS_FAT 0
32183 #define GZIP_OS_AMIGA 1
32184 #define GZIP_OS_VMS 2
32185 #define GZIP_OS_UNIX 3
32186 #define GZIP_OS_VM_CMS 4
32187 #define GZIP_OS_ATARI_TOS 5
32188 #define GZIP_OS_HPFS 6
32189 #define GZIP_OS_MACINTOSH 7
32190 #define GZIP_OS_Z_SYSTEM 8
32191 #define GZIP_OS_CP_M 9
32192 #define GZIP_OS_TOPS_20 10
32193 #define GZIP_OS_NTFS 11
32194 #define GZIP_OS_QDOS 12
32195 #define GZIP_OS_RISCOS 13
32196 #define GZIP_OS_UNKNOWN 255
32197  
32198 #endif
32199  
32200  
32201 LIBDEFLATEAPI size_t
32202 13 libdeflate_gzip_compress(struct libdeflate_compressor *c,
32203 const void *in, size_t in_nbytes,
32204 void *out, size_t out_nbytes_avail)
32205 {
32206 13 u8 *out_next = out;
32207 unsigned compression_level;
32208 u8 xfl;
32209 size_t deflate_size;
32210  
32211 13 ▶ if (out_nbytes_avail <= GZIP_MIN_OVERHEAD)
Branch: 1 branch
BranchTrueFalseTotal
if (out_nbytes_avail <= GZIP_MIN_OVERHEAD) 0 13 13
32212 0 return 0;
32213  
32214
32215 13 *out_next++ = GZIP_ID1;
32216
32217 13 *out_next++ = GZIP_ID2;
32218
32219 13 *out_next++ = GZIP_CM_DEFLATE;
32220
32221 13 *out_next++ = 0;
32222
32223 put_unaligned_le32(GZIP_MTIME_UNAVAILABLE, out_next);
32224 13 out_next += 4;
32225
32226 13 xfl = 0;
32227 13 compression_level = libdeflate_get_compression_level(c);
32228 13 ▶ if (compression_level < 2)
Branch: 1 branch
BranchTrueFalseTotal
if (compression_level < 2) 1 12 13
32229 1 xfl |= GZIP_XFL_FASTEST_COMPRESSION;
32230 12 ▶ else if (compression_level >= 8)
Branch: 1 branch
BranchTrueFalseTotal
else if (compression_level >= 8) 5 7 12
32231 5 xfl |= GZIP_XFL_SLOWEST_COMPRESSION;
32232 13 *out_next++ = xfl;
32233
32234 13 *out_next++ = GZIP_OS_UNKNOWN;
32235  
32236
32237 13 deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
32238 out_nbytes_avail - GZIP_MIN_OVERHEAD);
32239 13 ▶ if (deflate_size == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (deflate_size == 0) 0 13 13
32240 0 return 0;
32241 13 out_next += deflate_size;
32242  
32243
32244 13 put_unaligned_le32(libdeflate_crc32(0, in, in_nbytes), out_next);
32245 13 out_next += 4;
32246  
32247
32248 13 put_unaligned_le32((u32)in_nbytes, out_next);
32249 13 out_next += 4;
32250  
32251 13 return out_next - (u8 *)out;
32252 }
32253  
32254 LIBDEFLATEAPI size_t
32255 13 libdeflate_gzip_compress_bound(struct libdeflate_compressor *c,
32256 size_t in_nbytes)
32257 {
32258 13 return GZIP_MIN_OVERHEAD +
32259 13 libdeflate_deflate_compress_bound(c, in_nbytes);
32260 }
32261 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/gzip_decompress.c */
32262  
32263  
32264 /* #include "lib_common.h" */
32265  
32266  
32267 #ifndef LIB_LIB_COMMON_H
32268 #define LIB_LIB_COMMON_H
32269  
32270 #ifdef LIBDEFLATE_H
32271
32272 # error "lib_common.h must always be included before libdeflate.h"
32273 #endif
32274  
32275 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
32276 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
32277 #elif defined(__GNUC__)
32278 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
32279 #else
32280 # define LIBDEFLATE_EXPORT_SYM
32281 #endif
32282  
32283  
32284 #if defined(__GNUC__) && defined(__i386__)
32285 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
32286 #else
32287 # define LIBDEFLATE_ALIGN_STACK
32288 #endif
32289  
32290 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
32291  
32292 /* #include "../common_defs.h" */
32293  
32294  
32295 #ifndef COMMON_DEFS_H
32296 #define COMMON_DEFS_H
32297  
32298 /* #include "libdeflate.h" */
32299  
32300  
32301 #ifndef LIBDEFLATE_H
32302 #define LIBDEFLATE_H
32303  
32304 #include <stddef.h>
32305 #include <stdint.h>
32306  
32307 #ifdef __cplusplus
32308 extern "C" {
32309 #endif
32310  
32311 #define LIBDEFLATE_VERSION_MAJOR 1
32312 #define LIBDEFLATE_VERSION_MINOR 26
32313 #define LIBDEFLATE_VERSION_STRING "1.26"
32314  
32315  
32316 #ifndef LIBDEFLATEAPI
32317 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
32318 # define LIBDEFLATEAPI __declspec(dllimport)
32319 # else
32320 # define LIBDEFLATEAPI
32321 # endif
32322 #endif
32323  
32324  
32325  
32326  
32327  
32328 struct libdeflate_compressor;
32329 struct libdeflate_options;
32330  
32331  
32332 LIBDEFLATEAPI struct libdeflate_compressor *
32333 libdeflate_alloc_compressor(int compression_level);
32334  
32335  
32336 LIBDEFLATEAPI struct libdeflate_compressor *
32337 libdeflate_alloc_compressor_ex(int compression_level,
32338 const struct libdeflate_options *options);
32339  
32340  
32341 LIBDEFLATEAPI size_t
32342 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
32343 const void *in, size_t in_nbytes,
32344 void *out, size_t out_nbytes_avail);
32345  
32346  
32347 LIBDEFLATEAPI size_t
32348 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
32349 size_t in_nbytes);
32350  
32351  
32352 LIBDEFLATEAPI size_t
32353 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
32354 const void *in, size_t in_nbytes,
32355 void *out, size_t out_nbytes_avail);
32356  
32357  
32358 LIBDEFLATEAPI size_t
32359 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
32360 size_t in_nbytes);
32361  
32362  
32363 LIBDEFLATEAPI size_t
32364 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
32365 const void *in, size_t in_nbytes,
32366 void *out, size_t out_nbytes_avail);
32367  
32368  
32369 LIBDEFLATEAPI size_t
32370 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
32371 size_t in_nbytes);
32372  
32373  
32374 LIBDEFLATEAPI void
32375 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
32376  
32377  
32378  
32379  
32380  
32381 struct libdeflate_decompressor;
32382 struct libdeflate_options;
32383  
32384  
32385 LIBDEFLATEAPI struct libdeflate_decompressor *
32386 libdeflate_alloc_decompressor(void);
32387  
32388  
32389 LIBDEFLATEAPI struct libdeflate_decompressor *
32390 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
32391  
32392  
32393 enum libdeflate_result {
32394
32395 LIBDEFLATE_SUCCESS = 0,
32396  
32397
32398 LIBDEFLATE_BAD_DATA = 1,
32399  
32400
32401 LIBDEFLATE_SHORT_OUTPUT = 2,
32402  
32403
32404 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
32405 };
32406  
32407  
32408 LIBDEFLATEAPI enum libdeflate_result
32409 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
32410 const void *in, size_t in_nbytes,
32411 void *out, size_t out_nbytes_avail,
32412 size_t *actual_out_nbytes_ret);
32413  
32414  
32415 LIBDEFLATEAPI enum libdeflate_result
32416 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
32417 const void *in, size_t in_nbytes,
32418 void *out, size_t out_nbytes_avail,
32419 size_t *actual_in_nbytes_ret,
32420 size_t *actual_out_nbytes_ret);
32421  
32422  
32423 LIBDEFLATEAPI enum libdeflate_result
32424 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
32425 const void *in, size_t in_nbytes,
32426 void *out, size_t out_nbytes_avail,
32427 size_t *actual_out_nbytes_ret);
32428  
32429  
32430 LIBDEFLATEAPI enum libdeflate_result
32431 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
32432 const void *in, size_t in_nbytes,
32433 void *out, size_t out_nbytes_avail,
32434 size_t *actual_in_nbytes_ret,
32435 size_t *actual_out_nbytes_ret);
32436  
32437  
32438 LIBDEFLATEAPI enum libdeflate_result
32439 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
32440 const void *in, size_t in_nbytes,
32441 void *out, size_t out_nbytes_avail,
32442 size_t *actual_out_nbytes_ret);
32443  
32444  
32445 LIBDEFLATEAPI enum libdeflate_result
32446 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
32447 const void *in, size_t in_nbytes,
32448 void *out, size_t out_nbytes_avail,
32449 size_t *actual_in_nbytes_ret,
32450 size_t *actual_out_nbytes_ret);
32451  
32452  
32453 LIBDEFLATEAPI void
32454 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
32455  
32456  
32457  
32458  
32459  
32460  
32461 LIBDEFLATEAPI uint32_t
32462 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
32463  
32464  
32465  
32466 LIBDEFLATEAPI uint32_t
32467 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
32468  
32469  
32470  
32471  
32472  
32473  
32474 LIBDEFLATEAPI void
32475 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
32476 void (*free_func)(void *));
32477  
32478  
32479 struct libdeflate_options {
32480  
32481
32482 size_t sizeof_options;
32483  
32484
32485 void *(*malloc_func)(size_t);
32486 void (*free_func)(void *);
32487 };
32488  
32489 #ifdef __cplusplus
32490 }
32491 #endif
32492  
32493 #endif
32494  
32495  
32496 #include <stdbool.h>
32497 #include <stddef.h>
32498 #include <stdint.h>
32499 #ifdef _MSC_VER
32500 # include <intrin.h>
32501 # include <stdlib.h>
32502
32503
32504 # pragma warning(disable : 4146)
32505
32506 # pragma warning(disable : 4018)
32507 # pragma warning(disable : 4244)
32508 # pragma warning(disable : 4267)
32509 # pragma warning(disable : 4310)
32510
32511 # pragma warning(disable : 4100)
32512 # pragma warning(disable : 4127)
32513 # pragma warning(disable : 4189)
32514 # pragma warning(disable : 4232)
32515 # pragma warning(disable : 4245)
32516 # pragma warning(disable : 4295)
32517 #endif
32518 #ifndef FREESTANDING
32519 # include <string.h>
32520 #endif
32521  
32522  
32523  
32524  
32525  
32526  
32527 #undef ARCH_X86_64
32528 #undef ARCH_X86_32
32529 #undef ARCH_ARM64
32530 #undef ARCH_ARM32
32531 #undef ARCH_RISCV
32532 #ifdef _MSC_VER
32533
32534 # if defined(_M_X64) && !defined(_M_ARM64EC)
32535 # define ARCH_X86_64
32536 # elif defined(_M_IX86)
32537 # define ARCH_X86_32
32538 # elif defined(_M_ARM64)
32539 # define ARCH_ARM64
32540 # elif defined(_M_ARM)
32541 # define ARCH_ARM32
32542 # endif
32543 #else
32544 # if defined(__x86_64__)
32545 # define ARCH_X86_64
32546 # elif defined(__i386__)
32547 # define ARCH_X86_32
32548 # elif defined(__aarch64__)
32549 # define ARCH_ARM64
32550 # elif defined(__arm__)
32551 # define ARCH_ARM32
32552 # elif defined(__riscv)
32553 # define ARCH_RISCV
32554 # endif
32555 #endif
32556  
32557  
32558  
32559  
32560  
32561  
32562 typedef uint8_t u8;
32563 typedef uint16_t u16;
32564 typedef uint32_t u32;
32565 typedef uint64_t u64;
32566 typedef int8_t s8;
32567 typedef int16_t s16;
32568 typedef int32_t s32;
32569 typedef int64_t s64;
32570  
32571  
32572 #ifdef _MSC_VER
32573 # ifdef _WIN64
32574 typedef long long ssize_t;
32575 # else
32576 typedef long ssize_t;
32577 # endif
32578 #endif
32579  
32580  
32581 typedef size_t machine_word_t;
32582  
32583  
32584 #define WORDBYTES ((int)sizeof(machine_word_t))
32585  
32586  
32587 #define WORDBITS (8 * WORDBYTES)
32588  
32589  
32590  
32591  
32592  
32593  
32594 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
32595 # define GCC_PREREQ(major, minor) \
32596 (__GNUC__ > (major) || \
32597 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
32598 # if !GCC_PREREQ(4, 9)
32599 # error "gcc versions older than 4.9 are no longer supported"
32600 # endif
32601 #else
32602 # define GCC_PREREQ(major, minor) 0
32603 #endif
32604 #ifdef __clang__
32605 # ifdef __apple_build_version__
32606 # define CLANG_PREREQ(major, minor, apple_version) \
32607 (__apple_build_version__ >= (apple_version))
32608 # else
32609 # define CLANG_PREREQ(major, minor, apple_version) \
32610 (__clang_major__ > (major) || \
32611 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
32612 # endif
32613 # if !CLANG_PREREQ(3, 9, 8000000)
32614 # error "clang versions older than 3.9 are no longer supported"
32615 # endif
32616 #else
32617 # define CLANG_PREREQ(major, minor, apple_version) 0
32618 #endif
32619 #ifdef _MSC_VER
32620 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
32621 # if !MSVC_PREREQ(1928)
32622 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
32623 # endif
32624 #else
32625 # define MSVC_PREREQ(version) 0
32626 #endif
32627  
32628  
32629 #ifndef __has_attribute
32630 # define __has_attribute(attribute) 0
32631 #endif
32632  
32633  
32634 #ifndef __has_builtin
32635 # define __has_builtin(builtin) 0
32636 #endif
32637  
32638  
32639 #ifdef _MSC_VER
32640 # define inline __inline
32641 #endif
32642  
32643  
32644 #if defined(__GNUC__) || __has_attribute(always_inline)
32645 # define forceinline inline __attribute__((always_inline))
32646 #elif defined(_MSC_VER)
32647 # define forceinline __forceinline
32648 #else
32649 # define forceinline inline
32650 #endif
32651  
32652  
32653 #if defined(__GNUC__) || __has_attribute(unused)
32654 # define MAYBE_UNUSED __attribute__((unused))
32655 #else
32656 # define MAYBE_UNUSED
32657 #endif
32658  
32659  
32660 #if defined(__GNUC__) || __has_attribute(noreturn)
32661 # define NORETURN __attribute__((noreturn))
32662 #else
32663 # define NORETURN
32664 #endif
32665  
32666  
32667 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
32668 # define likely(expr) __builtin_expect(!!(expr), 1)
32669 #else
32670 # define likely(expr) (expr)
32671 #endif
32672  
32673  
32674 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
32675 # define unlikely(expr) __builtin_expect(!!(expr), 0)
32676 #else
32677 # define unlikely(expr) (expr)
32678 #endif
32679  
32680  
32681 #undef prefetchr
32682 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
32683 # define prefetchr(addr) __builtin_prefetch((addr), 0)
32684 #elif defined(_MSC_VER)
32685 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
32686 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
32687 # elif defined(ARCH_ARM64)
32688 # define prefetchr(addr) __prefetch2((addr), 0x00 )
32689 # elif defined(ARCH_ARM32)
32690 # define prefetchr(addr) __prefetch(addr)
32691 # endif
32692 #endif
32693 #ifndef prefetchr
32694 # define prefetchr(addr)
32695 #endif
32696  
32697  
32698 #undef prefetchw
32699 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
32700 # define prefetchw(addr) __builtin_prefetch((addr), 1)
32701 #elif defined(_MSC_VER)
32702 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
32703 # define prefetchw(addr) _m_prefetchw(addr)
32704 # elif defined(ARCH_ARM64)
32705 # define prefetchw(addr) __prefetch2((addr), 0x10 )
32706 # elif defined(ARCH_ARM32)
32707 # define prefetchw(addr) __prefetchw(addr)
32708 # endif
32709 #endif
32710 #ifndef prefetchw
32711 # define prefetchw(addr)
32712 #endif
32713  
32714  
32715 #undef _aligned_attribute
32716 #if defined(__GNUC__) || __has_attribute(aligned)
32717 # define _aligned_attribute(n) __attribute__((aligned(n)))
32718 #elif defined(_MSC_VER)
32719 # define _aligned_attribute(n) __declspec(align(n))
32720 #endif
32721  
32722  
32723 #if defined(__GNUC__) || __has_attribute(target)
32724 # define _target_attribute(attrs) __attribute__((target(attrs)))
32725 #else
32726 # define _target_attribute(attrs)
32727 #endif
32728  
32729  
32730  
32731  
32732  
32733 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
32734 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
32735 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
32736 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
32737 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
32738 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
32739 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
32740  
32741  
32742  
32743  
32744  
32745  
32746 #if defined(__BYTE_ORDER__)
32747 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
32748 #elif defined(_MSC_VER)
32749 # define CPU_IS_LITTLE_ENDIAN() true
32750 #else
32751 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
32752 {
32753 union {
32754 u32 w;
32755 u8 b;
32756 } u;
32757  
32758 u.w = 1;
32759 return u.b;
32760 }
32761 #endif
32762  
32763  
32764 static forceinline u16 bswap16(u16 v)
32765 {
32766 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
32767 return __builtin_bswap16(v);
32768 #elif defined(_MSC_VER)
32769 return _byteswap_ushort(v);
32770 #else
32771 return (v << 8) | (v >> 8);
32772 #endif
32773 }
32774  
32775  
32776 static forceinline u32 bswap32(u32 v)
32777 {
32778 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
32779 return __builtin_bswap32(v);
32780 #elif defined(_MSC_VER)
32781 return _byteswap_ulong(v);
32782 #else
32783 return ((v & 0x000000FF) << 24) |
32784 ((v & 0x0000FF00) << 8) |
32785 ((v & 0x00FF0000) >> 8) |
32786 ((v & 0xFF000000) >> 24);
32787 #endif
32788 }
32789  
32790  
32791 static forceinline u64 bswap64(u64 v)
32792 {
32793 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
32794 return __builtin_bswap64(v);
32795 #elif defined(_MSC_VER)
32796 return _byteswap_uint64(v);
32797 #else
32798 return ((v & 0x00000000000000FF) << 56) |
32799 ((v & 0x000000000000FF00) << 40) |
32800 ((v & 0x0000000000FF0000) << 24) |
32801 ((v & 0x00000000FF000000) << 8) |
32802 ((v & 0x000000FF00000000) >> 8) |
32803 ((v & 0x0000FF0000000000) >> 24) |
32804 ((v & 0x00FF000000000000) >> 40) |
32805 ((v & 0xFF00000000000000) >> 56);
32806 #endif
32807 }
32808  
32809 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
32810 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
32811 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
32812 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
32813 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
32814 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
32815  
32816  
32817  
32818  
32819  
32820  
32821 #if (defined(__GNUC__) || defined(__clang__)) && \
32822 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
32823 defined(__ARM_FEATURE_UNALIGNED) || \
32824 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
32825 defined(__riscv_misaligned_fast) || \
32826 defined(__wasm__))
32827 # define UNALIGNED_ACCESS_IS_FAST 1
32828 #elif defined(_MSC_VER)
32829 # define UNALIGNED_ACCESS_IS_FAST 1
32830 #else
32831 # define UNALIGNED_ACCESS_IS_FAST 0
32832 #endif
32833  
32834  
32835  
32836 #ifdef FREESTANDING
32837 # define MEMCOPY __builtin_memcpy
32838 #else
32839 # define MEMCOPY memcpy
32840 #endif
32841  
32842  
32843  
32844 #define DEFINE_UNALIGNED_TYPE(type) \
32845 static forceinline type \
32846 load_##type##_unaligned(const void *p) \
32847 { \
32848 type v; \
32849 \
32850 MEMCOPY(&v, p, sizeof(v)); \
32851 return v; \
32852 } \
32853 \
32854 static forceinline void \
32855 store_##type##_unaligned(type v, void *p) \
32856 { \
32857 MEMCOPY(p, &v, sizeof(v)); \
32858 }
32859  
32860 DEFINE_UNALIGNED_TYPE(u16)
32861 DEFINE_UNALIGNED_TYPE(u32)
32862 DEFINE_UNALIGNED_TYPE(u64)
32863 DEFINE_UNALIGNED_TYPE(machine_word_t)
32864  
32865 #undef MEMCOPY
32866  
32867 #define load_word_unaligned load_machine_word_t_unaligned
32868 #define store_word_unaligned store_machine_word_t_unaligned
32869  
32870  
32871  
32872 static forceinline u16
32873 get_unaligned_le16(const u8 *p)
32874 {
32875 if (UNALIGNED_ACCESS_IS_FAST)
32876 return le16_bswap(load_u16_unaligned(p));
32877 else
32878 return ((u16)p[1] << 8) | p[0];
32879 }
32880  
32881 static forceinline u16
32882 get_unaligned_be16(const u8 *p)
32883 {
32884 if (UNALIGNED_ACCESS_IS_FAST)
32885 return be16_bswap(load_u16_unaligned(p));
32886 else
32887 return ((u16)p[0] << 8) | p[1];
32888 }
32889  
32890 static forceinline u32
32891 get_unaligned_le32(const u8 *p)
32892 {
32893 if (UNALIGNED_ACCESS_IS_FAST)
32894 return le32_bswap(load_u32_unaligned(p));
32895 else
32896 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
32897 ((u32)p[1] << 8) | p[0];
32898 }
32899  
32900 static forceinline u32
32901 get_unaligned_be32(const u8 *p)
32902 {
32903 if (UNALIGNED_ACCESS_IS_FAST)
32904 return be32_bswap(load_u32_unaligned(p));
32905 else
32906 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
32907 ((u32)p[2] << 8) | p[3];
32908 }
32909  
32910 static forceinline u64
32911 get_unaligned_le64(const u8 *p)
32912 {
32913 if (UNALIGNED_ACCESS_IS_FAST)
32914 return le64_bswap(load_u64_unaligned(p));
32915 else
32916 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
32917 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
32918 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
32919 ((u64)p[1] << 8) | p[0];
32920 }
32921  
32922 static forceinline machine_word_t
32923 get_unaligned_leword(const u8 *p)
32924 {
32925 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
32926 if (WORDBITS == 32)
32927 return get_unaligned_le32(p);
32928 else
32929 return get_unaligned_le64(p);
32930 }
32931  
32932  
32933  
32934 static forceinline void
32935 put_unaligned_le16(u16 v, u8 *p)
32936 {
32937 if (UNALIGNED_ACCESS_IS_FAST) {
32938 store_u16_unaligned(le16_bswap(v), p);
32939 } else {
32940 p[0] = (u8)(v >> 0);
32941 p[1] = (u8)(v >> 8);
32942 }
32943 }
32944  
32945 static forceinline void
32946 put_unaligned_be16(u16 v, u8 *p)
32947 {
32948 if (UNALIGNED_ACCESS_IS_FAST) {
32949 store_u16_unaligned(be16_bswap(v), p);
32950 } else {
32951 p[0] = (u8)(v >> 8);
32952 p[1] = (u8)(v >> 0);
32953 }
32954 }
32955  
32956 static forceinline void
32957 put_unaligned_le32(u32 v, u8 *p)
32958 {
32959 if (UNALIGNED_ACCESS_IS_FAST) {
32960 store_u32_unaligned(le32_bswap(v), p);
32961 } else {
32962 p[0] = (u8)(v >> 0);
32963 p[1] = (u8)(v >> 8);
32964 p[2] = (u8)(v >> 16);
32965 p[3] = (u8)(v >> 24);
32966 }
32967 }
32968  
32969 static forceinline void
32970 put_unaligned_be32(u32 v, u8 *p)
32971 {
32972 if (UNALIGNED_ACCESS_IS_FAST) {
32973 store_u32_unaligned(be32_bswap(v), p);
32974 } else {
32975 p[0] = (u8)(v >> 24);
32976 p[1] = (u8)(v >> 16);
32977 p[2] = (u8)(v >> 8);
32978 p[3] = (u8)(v >> 0);
32979 }
32980 }
32981  
32982 static forceinline void
32983 put_unaligned_le64(u64 v, u8 *p)
32984 {
32985 if (UNALIGNED_ACCESS_IS_FAST) {
32986 store_u64_unaligned(le64_bswap(v), p);
32987 } else {
32988 p[0] = (u8)(v >> 0);
32989 p[1] = (u8)(v >> 8);
32990 p[2] = (u8)(v >> 16);
32991 p[3] = (u8)(v >> 24);
32992 p[4] = (u8)(v >> 32);
32993 p[5] = (u8)(v >> 40);
32994 p[6] = (u8)(v >> 48);
32995 p[7] = (u8)(v >> 56);
32996 }
32997 }
32998  
32999 static forceinline void
33000 put_unaligned_leword(machine_word_t v, u8 *p)
33001 {
33002 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
33003 if (WORDBITS == 32)
33004 put_unaligned_le32(v, p);
33005 else
33006 put_unaligned_le64(v, p);
33007 }
33008  
33009  
33010  
33011  
33012  
33013  
33014  
33015 static forceinline unsigned
33016 bsr32(u32 v)
33017 {
33018 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
33019 return 31 - __builtin_clz(v);
33020 #elif defined(_MSC_VER)
33021 unsigned long i;
33022  
33023 _BitScanReverse(&i, v);
33024 return i;
33025 #else
33026 unsigned i = 0;
33027  
33028 while ((v >>= 1) != 0)
33029 i++;
33030 return i;
33031 #endif
33032 }
33033  
33034 static forceinline unsigned
33035 bsr64(u64 v)
33036 {
33037 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
33038 return 63 - __builtin_clzll(v);
33039 #elif defined(_MSC_VER) && defined(_WIN64)
33040 unsigned long i;
33041  
33042 _BitScanReverse64(&i, v);
33043 return i;
33044 #else
33045 unsigned i = 0;
33046  
33047 while ((v >>= 1) != 0)
33048 i++;
33049 return i;
33050 #endif
33051 }
33052  
33053 static forceinline unsigned
33054 bsrw(machine_word_t v)
33055 {
33056 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
33057 if (WORDBITS == 32)
33058 return bsr32(v);
33059 else
33060 return bsr64(v);
33061 }
33062  
33063  
33064  
33065 static forceinline unsigned
33066 bsf32(u32 v)
33067 {
33068 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
33069 return __builtin_ctz(v);
33070 #elif defined(_MSC_VER)
33071 unsigned long i;
33072  
33073 _BitScanForward(&i, v);
33074 return i;
33075 #else
33076 unsigned i = 0;
33077  
33078 for (; (v & 1) == 0; v >>= 1)
33079 i++;
33080 return i;
33081 #endif
33082 }
33083  
33084 static forceinline unsigned
33085 bsf64(u64 v)
33086 {
33087 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
33088 return __builtin_ctzll(v);
33089 #elif defined(_MSC_VER) && defined(_WIN64)
33090 unsigned long i;
33091  
33092 _BitScanForward64(&i, v);
33093 return i;
33094 #else
33095 unsigned i = 0;
33096  
33097 for (; (v & 1) == 0; v >>= 1)
33098 i++;
33099 return i;
33100 #endif
33101 }
33102  
33103 static forceinline unsigned
33104 bsfw(machine_word_t v)
33105 {
33106 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
33107 if (WORDBITS == 32)
33108 return bsf32(v);
33109 else
33110 return bsf64(v);
33111 }
33112  
33113  
33114 #undef rbit32
33115 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
33116 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
33117 static forceinline u32
33118 rbit32(u32 v)
33119 {
33120 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
33121 return v;
33122 }
33123 #define rbit32 rbit32
33124 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
33125 static forceinline u32
33126 rbit32(u32 v)
33127 {
33128 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
33129 return v;
33130 }
33131 #define rbit32 rbit32
33132 #endif
33133  
33134 #endif
33135  
33136  
33137 typedef void *(*malloc_func_t)(size_t);
33138 typedef void (*free_func_t)(void *);
33139  
33140 extern malloc_func_t libdeflate_default_malloc_func;
33141 extern free_func_t libdeflate_default_free_func;
33142  
33143 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
33144 size_t alignment, size_t size);
33145 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
33146  
33147 #ifdef FREESTANDING
33148  
33149 void *memset(void *s, int c, size_t n);
33150 #define memset(s, c, n) __builtin_memset((s), (c), (n))
33151  
33152 void *memcpy(void *dest, const void *src, size_t n);
33153 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
33154  
33155 void *memmove(void *dest, const void *src, size_t n);
33156 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
33157  
33158 int memcmp(const void *s1, const void *s2, size_t n);
33159 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
33160  
33161 #undef LIBDEFLATE_ENABLE_ASSERTIONS
33162 #else
33163 # include <string.h>
33164
33165 # ifdef __clang_analyzer__
33166 # define LIBDEFLATE_ENABLE_ASSERTIONS
33167 # endif
33168 #endif
33169  
33170  
33171 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
33172 NORETURN void
33173 libdeflate_assertion_failed(const char *expr, const char *file, int line);
33174 #define ASSERT(expr) { if (unlikely(!(expr))) \
33175 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
33176 #else
33177 #define ASSERT(expr) (void)(expr)
33178 #endif
33179  
33180 #define CONCAT_IMPL(a, b) a##b
33181 #define CONCAT(a, b) CONCAT_IMPL(a, b)
33182 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
33183  
33184 #endif
33185  
33186 /* #include "gzip_constants.h" */
33187  
33188  
33189 #ifndef LIB_GZIP_CONSTANTS_H
33190 #define LIB_GZIP_CONSTANTS_H
33191  
33192 #define GZIP_MIN_HEADER_SIZE 10
33193 #define GZIP_FOOTER_SIZE 8
33194 #define GZIP_MIN_OVERHEAD (GZIP_MIN_HEADER_SIZE + GZIP_FOOTER_SIZE)
33195  
33196 #define GZIP_ID1 0x1F
33197 #define GZIP_ID2 0x8B
33198  
33199 #define GZIP_CM_DEFLATE 8
33200  
33201 #define GZIP_FTEXT 0x01
33202 #define GZIP_FHCRC 0x02
33203 #define GZIP_FEXTRA 0x04
33204 #define GZIP_FNAME 0x08
33205 #define GZIP_FCOMMENT 0x10
33206 #define GZIP_FRESERVED 0xE0
33207  
33208 #define GZIP_MTIME_UNAVAILABLE 0
33209  
33210 #define GZIP_XFL_SLOWEST_COMPRESSION 0x02
33211 #define GZIP_XFL_FASTEST_COMPRESSION 0x04
33212  
33213 #define GZIP_OS_FAT 0
33214 #define GZIP_OS_AMIGA 1
33215 #define GZIP_OS_VMS 2
33216 #define GZIP_OS_UNIX 3
33217 #define GZIP_OS_VM_CMS 4
33218 #define GZIP_OS_ATARI_TOS 5
33219 #define GZIP_OS_HPFS 6
33220 #define GZIP_OS_MACINTOSH 7
33221 #define GZIP_OS_Z_SYSTEM 8
33222 #define GZIP_OS_CP_M 9
33223 #define GZIP_OS_TOPS_20 10
33224 #define GZIP_OS_NTFS 11
33225 #define GZIP_OS_QDOS 12
33226 #define GZIP_OS_RISCOS 13
33227 #define GZIP_OS_UNKNOWN 255
33228  
33229 #endif
33230  
33231  
33232 LIBDEFLATEAPI enum libdeflate_result
33233 16 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *d,
33234 const void *in, size_t in_nbytes,
33235 void *out, size_t out_nbytes_avail,
33236 size_t *actual_in_nbytes_ret,
33237 size_t *actual_out_nbytes_ret)
33238 {
33239 16 const u8 *in_next = in;
33240 16 const u8 * const in_end = in_next + in_nbytes;
33241 u8 flg;
33242 size_t actual_in_nbytes;
33243 size_t actual_out_nbytes;
33244 enum libdeflate_result result;
33245  
33246 16 ▶ if (in_nbytes < GZIP_MIN_OVERHEAD)
Branch: 1 branch
BranchTrueFalseTotal
if (in_nbytes < GZIP_MIN_OVERHEAD) 0 16 16
33247 0 return LIBDEFLATE_BAD_DATA;
33248  
33249
33250 16 ▶ if (*in_next++ != GZIP_ID1)
Branch: 1 branch
BranchTrueFalseTotal
if (*in_next++ != GZIP_ID1) 0 16 16
33251 0 return LIBDEFLATE_BAD_DATA;
33252
33253 16 ▶ if (*in_next++ != GZIP_ID2)
Branch: 1 branch
BranchTrueFalseTotal
if (*in_next++ != GZIP_ID2) 0 16 16
33254 0 return LIBDEFLATE_BAD_DATA;
33255
33256 16 ▶ if (*in_next++ != GZIP_CM_DEFLATE)
Branch: 1 branch
BranchTrueFalseTotal
if (*in_next++ != GZIP_CM_DEFLATE) 0 16 16
33257 0 return LIBDEFLATE_BAD_DATA;
33258 16 flg = *in_next++;
33259
33260 16 in_next += 4;
33261
33262 16 in_next += 1;
33263
33264 16 in_next += 1;
33265  
33266 16 ▶ if (flg & GZIP_FRESERVED)
Branch: 1 branch
BranchTrueFalseTotal
if (flg & GZIP_FRESERVED) 0 16 16
33267 0 return LIBDEFLATE_BAD_DATA;
33268  
33269
33270 16 ▶ if (flg & GZIP_FEXTRA) {
Branch: 1 branch
BranchTrueFalseTotal
if (flg & GZIP_FEXTRA) { 0 16 16
33271 0 u16 xlen = get_unaligned_le16(in_next);
33272 0 in_next += 2;
33273  
33274 0 ▶ if (in_end - in_next < (u32)xlen + GZIP_FOOTER_SIZE)
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next < (u32)xlen + GZIP_FOOTER_SIZE) 0 0 0
33275 0 return LIBDEFLATE_BAD_DATA;
33276  
33277 0 in_next += xlen;
33278 }
33279  
33280
33281 16 ▶ if (flg & GZIP_FNAME) {
Branch: 1 branch
BranchTrueFalseTotal
if (flg & GZIP_FNAME) { 3 13 16
33282 33 ▶ while (*in_next++ != 0 && in_next != in_end)
Branch: 2 branches
BranchTrueFalseTotal
while (*in_next++ != 0 && in_next != in_end) 30 3 33
while (*in_next++ != 0 && in_next != in_end) 30 0 30
33283 ;
33284 3 ▶ if (in_end - in_next < GZIP_FOOTER_SIZE)
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next < GZIP_FOOTER_SIZE) 0 3 3
33285 0 return LIBDEFLATE_BAD_DATA;
33286 }
33287  
33288
33289 16 ▶ if (flg & GZIP_FCOMMENT) {
Branch: 1 branch
BranchTrueFalseTotal
if (flg & GZIP_FCOMMENT) { 0 16 16
33290 0 ▶ while (*in_next++ != 0 && in_next != in_end)
Branch: 2 branches
BranchTrueFalseTotal
while (*in_next++ != 0 && in_next != in_end) 0 0 0
while (*in_next++ != 0 && in_next != in_end) 0 0 0
33291 ;
33292 0 ▶ if (in_end - in_next < GZIP_FOOTER_SIZE)
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next < GZIP_FOOTER_SIZE) 0 0 0
33293 0 return LIBDEFLATE_BAD_DATA;
33294 }
33295  
33296
33297 16 ▶ if (flg & GZIP_FHCRC) {
Branch: 1 branch
BranchTrueFalseTotal
if (flg & GZIP_FHCRC) { 0 16 16
33298 0 in_next += 2;
33299 0 ▶ if (in_end - in_next < GZIP_FOOTER_SIZE)
Branch: 1 branch
BranchTrueFalseTotal
if (in_end - in_next < GZIP_FOOTER_SIZE) 0 0 0
33300 0 return LIBDEFLATE_BAD_DATA;
33301 }
33302  
33303
33304 16 result = libdeflate_deflate_decompress_ex(d, in_next,
33305 16 in_end - GZIP_FOOTER_SIZE - in_next,
33306 out, out_nbytes_avail,
33307 &actual_in_nbytes,
33308 actual_out_nbytes_ret);
33309 16 ▶ if (result != LIBDEFLATE_SUCCESS)
Branch: 1 branch
BranchTrueFalseTotal
if (result != LIBDEFLATE_SUCCESS) 0 16 16
33310 0 return result;
33311  
33312 16 ▶ if (actual_out_nbytes_ret)
Branch: 1 branch
BranchTrueFalseTotal
if (actual_out_nbytes_ret) 16 0 16
33313 16 actual_out_nbytes = *actual_out_nbytes_ret;
33314 else
33315 0 actual_out_nbytes = out_nbytes_avail;
33316  
33317 16 in_next += actual_in_nbytes;
33318  
33319
33320 16 if (libdeflate_crc32(0, out, actual_out_nbytes) !=
33321 16 ▶ get_unaligned_le32(in_next))
Branch: 1 branch
BranchTrueFalseTotal
get_unaligned_le32(in_next)) 0 16 16
33322 0 return LIBDEFLATE_BAD_DATA;
33323 16 in_next += 4;
33324  
33325
33326 32 ▶ if ((u32)actual_out_nbytes != get_unaligned_le32(in_next))
Branch: 1 branch
BranchTrueFalseTotal
if ((u32)actual_out_nbytes != get_unaligned_le32(in_next)) 0 16 16
33327 0 return LIBDEFLATE_BAD_DATA;
33328 16 in_next += 4;
33329  
33330 16 ▶ if (actual_in_nbytes_ret)
Branch: 1 branch
BranchTrueFalseTotal
if (actual_in_nbytes_ret) 16 0 16
33331 16 *actual_in_nbytes_ret = in_next - (u8 *)in;
33332  
33333 16 return LIBDEFLATE_SUCCESS;
33334 }
33335  
33336 LIBDEFLATEAPI enum libdeflate_result
33337 0 libdeflate_gzip_decompress(struct libdeflate_decompressor *d,
33338 const void *in, size_t in_nbytes,
33339 void *out, size_t out_nbytes_avail,
33340 size_t *actual_out_nbytes_ret)
33341 {
33342 0 return libdeflate_gzip_decompress_ex(d, in, in_nbytes,
33343 out, out_nbytes_avail,
33344 NULL, actual_out_nbytes_ret);
33345 }
33346 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/utils.c */
33347  
33348  
33349 /* #include "lib_common.h" */
33350  
33351  
33352 #ifndef LIB_LIB_COMMON_H
33353 #define LIB_LIB_COMMON_H
33354  
33355 #ifdef LIBDEFLATE_H
33356
33357 # error "lib_common.h must always be included before libdeflate.h"
33358 #endif
33359  
33360 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
33361 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
33362 #elif defined(__GNUC__)
33363 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
33364 #else
33365 # define LIBDEFLATE_EXPORT_SYM
33366 #endif
33367  
33368  
33369 #if defined(__GNUC__) && defined(__i386__)
33370 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
33371 #else
33372 # define LIBDEFLATE_ALIGN_STACK
33373 #endif
33374  
33375 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
33376  
33377 /* #include "../common_defs.h" */
33378  
33379  
33380 #ifndef COMMON_DEFS_H
33381 #define COMMON_DEFS_H
33382  
33383 /* #include "libdeflate.h" */
33384  
33385  
33386 #ifndef LIBDEFLATE_H
33387 #define LIBDEFLATE_H
33388  
33389 #include <stddef.h>
33390 #include <stdint.h>
33391  
33392 #ifdef __cplusplus
33393 extern "C" {
33394 #endif
33395  
33396 #define LIBDEFLATE_VERSION_MAJOR 1
33397 #define LIBDEFLATE_VERSION_MINOR 26
33398 #define LIBDEFLATE_VERSION_STRING "1.26"
33399  
33400  
33401 #ifndef LIBDEFLATEAPI
33402 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
33403 # define LIBDEFLATEAPI __declspec(dllimport)
33404 # else
33405 # define LIBDEFLATEAPI
33406 # endif
33407 #endif
33408  
33409  
33410  
33411  
33412  
33413 struct libdeflate_compressor;
33414 struct libdeflate_options;
33415  
33416  
33417 LIBDEFLATEAPI struct libdeflate_compressor *
33418 libdeflate_alloc_compressor(int compression_level);
33419  
33420  
33421 LIBDEFLATEAPI struct libdeflate_compressor *
33422 libdeflate_alloc_compressor_ex(int compression_level,
33423 const struct libdeflate_options *options);
33424  
33425  
33426 LIBDEFLATEAPI size_t
33427 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
33428 const void *in, size_t in_nbytes,
33429 void *out, size_t out_nbytes_avail);
33430  
33431  
33432 LIBDEFLATEAPI size_t
33433 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
33434 size_t in_nbytes);
33435  
33436  
33437 LIBDEFLATEAPI size_t
33438 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
33439 const void *in, size_t in_nbytes,
33440 void *out, size_t out_nbytes_avail);
33441  
33442  
33443 LIBDEFLATEAPI size_t
33444 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
33445 size_t in_nbytes);
33446  
33447  
33448 LIBDEFLATEAPI size_t
33449 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
33450 const void *in, size_t in_nbytes,
33451 void *out, size_t out_nbytes_avail);
33452  
33453  
33454 LIBDEFLATEAPI size_t
33455 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
33456 size_t in_nbytes);
33457  
33458  
33459 LIBDEFLATEAPI void
33460 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
33461  
33462  
33463  
33464  
33465  
33466 struct libdeflate_decompressor;
33467 struct libdeflate_options;
33468  
33469  
33470 LIBDEFLATEAPI struct libdeflate_decompressor *
33471 libdeflate_alloc_decompressor(void);
33472  
33473  
33474 LIBDEFLATEAPI struct libdeflate_decompressor *
33475 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
33476  
33477  
33478 enum libdeflate_result {
33479
33480 LIBDEFLATE_SUCCESS = 0,
33481  
33482
33483 LIBDEFLATE_BAD_DATA = 1,
33484  
33485
33486 LIBDEFLATE_SHORT_OUTPUT = 2,
33487  
33488
33489 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
33490 };
33491  
33492  
33493 LIBDEFLATEAPI enum libdeflate_result
33494 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
33495 const void *in, size_t in_nbytes,
33496 void *out, size_t out_nbytes_avail,
33497 size_t *actual_out_nbytes_ret);
33498  
33499  
33500 LIBDEFLATEAPI enum libdeflate_result
33501 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
33502 const void *in, size_t in_nbytes,
33503 void *out, size_t out_nbytes_avail,
33504 size_t *actual_in_nbytes_ret,
33505 size_t *actual_out_nbytes_ret);
33506  
33507  
33508 LIBDEFLATEAPI enum libdeflate_result
33509 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
33510 const void *in, size_t in_nbytes,
33511 void *out, size_t out_nbytes_avail,
33512 size_t *actual_out_nbytes_ret);
33513  
33514  
33515 LIBDEFLATEAPI enum libdeflate_result
33516 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
33517 const void *in, size_t in_nbytes,
33518 void *out, size_t out_nbytes_avail,
33519 size_t *actual_in_nbytes_ret,
33520 size_t *actual_out_nbytes_ret);
33521  
33522  
33523 LIBDEFLATEAPI enum libdeflate_result
33524 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
33525 const void *in, size_t in_nbytes,
33526 void *out, size_t out_nbytes_avail,
33527 size_t *actual_out_nbytes_ret);
33528  
33529  
33530 LIBDEFLATEAPI enum libdeflate_result
33531 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
33532 const void *in, size_t in_nbytes,
33533 void *out, size_t out_nbytes_avail,
33534 size_t *actual_in_nbytes_ret,
33535 size_t *actual_out_nbytes_ret);
33536  
33537  
33538 LIBDEFLATEAPI void
33539 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
33540  
33541  
33542  
33543  
33544  
33545  
33546 LIBDEFLATEAPI uint32_t
33547 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
33548  
33549  
33550  
33551 LIBDEFLATEAPI uint32_t
33552 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
33553  
33554  
33555  
33556  
33557  
33558  
33559 LIBDEFLATEAPI void
33560 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
33561 void (*free_func)(void *));
33562  
33563  
33564 struct libdeflate_options {
33565  
33566
33567 size_t sizeof_options;
33568  
33569
33570 void *(*malloc_func)(size_t);
33571 void (*free_func)(void *);
33572 };
33573  
33574 #ifdef __cplusplus
33575 }
33576 #endif
33577  
33578 #endif
33579  
33580  
33581 #include <stdbool.h>
33582 #include <stddef.h>
33583 #include <stdint.h>
33584 #ifdef _MSC_VER
33585 # include <intrin.h>
33586 # include <stdlib.h>
33587
33588
33589 # pragma warning(disable : 4146)
33590
33591 # pragma warning(disable : 4018)
33592 # pragma warning(disable : 4244)
33593 # pragma warning(disable : 4267)
33594 # pragma warning(disable : 4310)
33595
33596 # pragma warning(disable : 4100)
33597 # pragma warning(disable : 4127)
33598 # pragma warning(disable : 4189)
33599 # pragma warning(disable : 4232)
33600 # pragma warning(disable : 4245)
33601 # pragma warning(disable : 4295)
33602 #endif
33603 #ifndef FREESTANDING
33604 # include <string.h>
33605 #endif
33606  
33607  
33608  
33609  
33610  
33611  
33612 #undef ARCH_X86_64
33613 #undef ARCH_X86_32
33614 #undef ARCH_ARM64
33615 #undef ARCH_ARM32
33616 #undef ARCH_RISCV
33617 #ifdef _MSC_VER
33618
33619 # if defined(_M_X64) && !defined(_M_ARM64EC)
33620 # define ARCH_X86_64
33621 # elif defined(_M_IX86)
33622 # define ARCH_X86_32
33623 # elif defined(_M_ARM64)
33624 # define ARCH_ARM64
33625 # elif defined(_M_ARM)
33626 # define ARCH_ARM32
33627 # endif
33628 #else
33629 # if defined(__x86_64__)
33630 # define ARCH_X86_64
33631 # elif defined(__i386__)
33632 # define ARCH_X86_32
33633 # elif defined(__aarch64__)
33634 # define ARCH_ARM64
33635 # elif defined(__arm__)
33636 # define ARCH_ARM32
33637 # elif defined(__riscv)
33638 # define ARCH_RISCV
33639 # endif
33640 #endif
33641  
33642  
33643  
33644  
33645  
33646  
33647 typedef uint8_t u8;
33648 typedef uint16_t u16;
33649 typedef uint32_t u32;
33650 typedef uint64_t u64;
33651 typedef int8_t s8;
33652 typedef int16_t s16;
33653 typedef int32_t s32;
33654 typedef int64_t s64;
33655  
33656  
33657 #ifdef _MSC_VER
33658 # ifdef _WIN64
33659 typedef long long ssize_t;
33660 # else
33661 typedef long ssize_t;
33662 # endif
33663 #endif
33664  
33665  
33666 typedef size_t machine_word_t;
33667  
33668  
33669 #define WORDBYTES ((int)sizeof(machine_word_t))
33670  
33671  
33672 #define WORDBITS (8 * WORDBYTES)
33673  
33674  
33675  
33676  
33677  
33678  
33679 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
33680 # define GCC_PREREQ(major, minor) \
33681 (__GNUC__ > (major) || \
33682 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
33683 # if !GCC_PREREQ(4, 9)
33684 # error "gcc versions older than 4.9 are no longer supported"
33685 # endif
33686 #else
33687 # define GCC_PREREQ(major, minor) 0
33688 #endif
33689 #ifdef __clang__
33690 # ifdef __apple_build_version__
33691 # define CLANG_PREREQ(major, minor, apple_version) \
33692 (__apple_build_version__ >= (apple_version))
33693 # else
33694 # define CLANG_PREREQ(major, minor, apple_version) \
33695 (__clang_major__ > (major) || \
33696 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
33697 # endif
33698 # if !CLANG_PREREQ(3, 9, 8000000)
33699 # error "clang versions older than 3.9 are no longer supported"
33700 # endif
33701 #else
33702 # define CLANG_PREREQ(major, minor, apple_version) 0
33703 #endif
33704 #ifdef _MSC_VER
33705 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
33706 # if !MSVC_PREREQ(1928)
33707 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
33708 # endif
33709 #else
33710 # define MSVC_PREREQ(version) 0
33711 #endif
33712  
33713  
33714 #ifndef __has_attribute
33715 # define __has_attribute(attribute) 0
33716 #endif
33717  
33718  
33719 #ifndef __has_builtin
33720 # define __has_builtin(builtin) 0
33721 #endif
33722  
33723  
33724 #ifdef _MSC_VER
33725 # define inline __inline
33726 #endif
33727  
33728  
33729 #if defined(__GNUC__) || __has_attribute(always_inline)
33730 # define forceinline inline __attribute__((always_inline))
33731 #elif defined(_MSC_VER)
33732 # define forceinline __forceinline
33733 #else
33734 # define forceinline inline
33735 #endif
33736  
33737  
33738 #if defined(__GNUC__) || __has_attribute(unused)
33739 # define MAYBE_UNUSED __attribute__((unused))
33740 #else
33741 # define MAYBE_UNUSED
33742 #endif
33743  
33744  
33745 #if defined(__GNUC__) || __has_attribute(noreturn)
33746 # define NORETURN __attribute__((noreturn))
33747 #else
33748 # define NORETURN
33749 #endif
33750  
33751  
33752 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
33753 # define likely(expr) __builtin_expect(!!(expr), 1)
33754 #else
33755 # define likely(expr) (expr)
33756 #endif
33757  
33758  
33759 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
33760 # define unlikely(expr) __builtin_expect(!!(expr), 0)
33761 #else
33762 # define unlikely(expr) (expr)
33763 #endif
33764  
33765  
33766 #undef prefetchr
33767 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
33768 # define prefetchr(addr) __builtin_prefetch((addr), 0)
33769 #elif defined(_MSC_VER)
33770 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
33771 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
33772 # elif defined(ARCH_ARM64)
33773 # define prefetchr(addr) __prefetch2((addr), 0x00 )
33774 # elif defined(ARCH_ARM32)
33775 # define prefetchr(addr) __prefetch(addr)
33776 # endif
33777 #endif
33778 #ifndef prefetchr
33779 # define prefetchr(addr)
33780 #endif
33781  
33782  
33783 #undef prefetchw
33784 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
33785 # define prefetchw(addr) __builtin_prefetch((addr), 1)
33786 #elif defined(_MSC_VER)
33787 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
33788 # define prefetchw(addr) _m_prefetchw(addr)
33789 # elif defined(ARCH_ARM64)
33790 # define prefetchw(addr) __prefetch2((addr), 0x10 )
33791 # elif defined(ARCH_ARM32)
33792 # define prefetchw(addr) __prefetchw(addr)
33793 # endif
33794 #endif
33795 #ifndef prefetchw
33796 # define prefetchw(addr)
33797 #endif
33798  
33799  
33800 #undef _aligned_attribute
33801 #if defined(__GNUC__) || __has_attribute(aligned)
33802 # define _aligned_attribute(n) __attribute__((aligned(n)))
33803 #elif defined(_MSC_VER)
33804 # define _aligned_attribute(n) __declspec(align(n))
33805 #endif
33806  
33807  
33808 #if defined(__GNUC__) || __has_attribute(target)
33809 # define _target_attribute(attrs) __attribute__((target(attrs)))
33810 #else
33811 # define _target_attribute(attrs)
33812 #endif
33813  
33814  
33815  
33816  
33817  
33818 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
33819 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
33820 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
33821 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
33822 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
33823 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
33824 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
33825  
33826  
33827  
33828  
33829  
33830  
33831 #if defined(__BYTE_ORDER__)
33832 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
33833 #elif defined(_MSC_VER)
33834 # define CPU_IS_LITTLE_ENDIAN() true
33835 #else
33836 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
33837 {
33838 union {
33839 u32 w;
33840 u8 b;
33841 } u;
33842  
33843 u.w = 1;
33844 return u.b;
33845 }
33846 #endif
33847  
33848  
33849 static forceinline u16 bswap16(u16 v)
33850 {
33851 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
33852 return __builtin_bswap16(v);
33853 #elif defined(_MSC_VER)
33854 return _byteswap_ushort(v);
33855 #else
33856 return (v << 8) | (v >> 8);
33857 #endif
33858 }
33859  
33860  
33861 static forceinline u32 bswap32(u32 v)
33862 {
33863 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
33864 return __builtin_bswap32(v);
33865 #elif defined(_MSC_VER)
33866 return _byteswap_ulong(v);
33867 #else
33868 return ((v & 0x000000FF) << 24) |
33869 ((v & 0x0000FF00) << 8) |
33870 ((v & 0x00FF0000) >> 8) |
33871 ((v & 0xFF000000) >> 24);
33872 #endif
33873 }
33874  
33875  
33876 static forceinline u64 bswap64(u64 v)
33877 {
33878 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
33879 return __builtin_bswap64(v);
33880 #elif defined(_MSC_VER)
33881 return _byteswap_uint64(v);
33882 #else
33883 return ((v & 0x00000000000000FF) << 56) |
33884 ((v & 0x000000000000FF00) << 40) |
33885 ((v & 0x0000000000FF0000) << 24) |
33886 ((v & 0x00000000FF000000) << 8) |
33887 ((v & 0x000000FF00000000) >> 8) |
33888 ((v & 0x0000FF0000000000) >> 24) |
33889 ((v & 0x00FF000000000000) >> 40) |
33890 ((v & 0xFF00000000000000) >> 56);
33891 #endif
33892 }
33893  
33894 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
33895 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
33896 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
33897 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
33898 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
33899 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
33900  
33901  
33902  
33903  
33904  
33905  
33906 #if (defined(__GNUC__) || defined(__clang__)) && \
33907 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
33908 defined(__ARM_FEATURE_UNALIGNED) || \
33909 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
33910 defined(__riscv_misaligned_fast) || \
33911 defined(__wasm__))
33912 # define UNALIGNED_ACCESS_IS_FAST 1
33913 #elif defined(_MSC_VER)
33914 # define UNALIGNED_ACCESS_IS_FAST 1
33915 #else
33916 # define UNALIGNED_ACCESS_IS_FAST 0
33917 #endif
33918  
33919  
33920  
33921 #ifdef FREESTANDING
33922 # define MEMCOPY __builtin_memcpy
33923 #else
33924 # define MEMCOPY memcpy
33925 #endif
33926  
33927  
33928  
33929 #define DEFINE_UNALIGNED_TYPE(type) \
33930 static forceinline type \
33931 load_##type##_unaligned(const void *p) \
33932 { \
33933 type v; \
33934 \
33935 MEMCOPY(&v, p, sizeof(v)); \
33936 return v; \
33937 } \
33938 \
33939 static forceinline void \
33940 store_##type##_unaligned(type v, void *p) \
33941 { \
33942 MEMCOPY(p, &v, sizeof(v)); \
33943 }
33944  
33945 DEFINE_UNALIGNED_TYPE(u16)
33946 DEFINE_UNALIGNED_TYPE(u32)
33947 DEFINE_UNALIGNED_TYPE(u64)
33948 DEFINE_UNALIGNED_TYPE(machine_word_t)
33949  
33950 #undef MEMCOPY
33951  
33952 #define load_word_unaligned load_machine_word_t_unaligned
33953 #define store_word_unaligned store_machine_word_t_unaligned
33954  
33955  
33956  
33957 static forceinline u16
33958 get_unaligned_le16(const u8 *p)
33959 {
33960 if (UNALIGNED_ACCESS_IS_FAST)
33961 return le16_bswap(load_u16_unaligned(p));
33962 else
33963 return ((u16)p[1] << 8) | p[0];
33964 }
33965  
33966 static forceinline u16
33967 get_unaligned_be16(const u8 *p)
33968 {
33969 if (UNALIGNED_ACCESS_IS_FAST)
33970 return be16_bswap(load_u16_unaligned(p));
33971 else
33972 return ((u16)p[0] << 8) | p[1];
33973 }
33974  
33975 static forceinline u32
33976 get_unaligned_le32(const u8 *p)
33977 {
33978 if (UNALIGNED_ACCESS_IS_FAST)
33979 return le32_bswap(load_u32_unaligned(p));
33980 else
33981 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
33982 ((u32)p[1] << 8) | p[0];
33983 }
33984  
33985 static forceinline u32
33986 get_unaligned_be32(const u8 *p)
33987 {
33988 if (UNALIGNED_ACCESS_IS_FAST)
33989 return be32_bswap(load_u32_unaligned(p));
33990 else
33991 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
33992 ((u32)p[2] << 8) | p[3];
33993 }
33994  
33995 static forceinline u64
33996 get_unaligned_le64(const u8 *p)
33997 {
33998 if (UNALIGNED_ACCESS_IS_FAST)
33999 return le64_bswap(load_u64_unaligned(p));
34000 else
34001 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
34002 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
34003 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
34004 ((u64)p[1] << 8) | p[0];
34005 }
34006  
34007 static forceinline machine_word_t
34008 get_unaligned_leword(const u8 *p)
34009 {
34010 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
34011 if (WORDBITS == 32)
34012 return get_unaligned_le32(p);
34013 else
34014 return get_unaligned_le64(p);
34015 }
34016  
34017  
34018  
34019 static forceinline void
34020 put_unaligned_le16(u16 v, u8 *p)
34021 {
34022 if (UNALIGNED_ACCESS_IS_FAST) {
34023 store_u16_unaligned(le16_bswap(v), p);
34024 } else {
34025 p[0] = (u8)(v >> 0);
34026 p[1] = (u8)(v >> 8);
34027 }
34028 }
34029  
34030 static forceinline void
34031 put_unaligned_be16(u16 v, u8 *p)
34032 {
34033 if (UNALIGNED_ACCESS_IS_FAST) {
34034 store_u16_unaligned(be16_bswap(v), p);
34035 } else {
34036 p[0] = (u8)(v >> 8);
34037 p[1] = (u8)(v >> 0);
34038 }
34039 }
34040  
34041 static forceinline void
34042 put_unaligned_le32(u32 v, u8 *p)
34043 {
34044 if (UNALIGNED_ACCESS_IS_FAST) {
34045 store_u32_unaligned(le32_bswap(v), p);
34046 } else {
34047 p[0] = (u8)(v >> 0);
34048 p[1] = (u8)(v >> 8);
34049 p[2] = (u8)(v >> 16);
34050 p[3] = (u8)(v >> 24);
34051 }
34052 }
34053  
34054 static forceinline void
34055 put_unaligned_be32(u32 v, u8 *p)
34056 {
34057 if (UNALIGNED_ACCESS_IS_FAST) {
34058 store_u32_unaligned(be32_bswap(v), p);
34059 } else {
34060 p[0] = (u8)(v >> 24);
34061 p[1] = (u8)(v >> 16);
34062 p[2] = (u8)(v >> 8);
34063 p[3] = (u8)(v >> 0);
34064 }
34065 }
34066  
34067 static forceinline void
34068 put_unaligned_le64(u64 v, u8 *p)
34069 {
34070 if (UNALIGNED_ACCESS_IS_FAST) {
34071 store_u64_unaligned(le64_bswap(v), p);
34072 } else {
34073 p[0] = (u8)(v >> 0);
34074 p[1] = (u8)(v >> 8);
34075 p[2] = (u8)(v >> 16);
34076 p[3] = (u8)(v >> 24);
34077 p[4] = (u8)(v >> 32);
34078 p[5] = (u8)(v >> 40);
34079 p[6] = (u8)(v >> 48);
34080 p[7] = (u8)(v >> 56);
34081 }
34082 }
34083  
34084 static forceinline void
34085 put_unaligned_leword(machine_word_t v, u8 *p)
34086 {
34087 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
34088 if (WORDBITS == 32)
34089 put_unaligned_le32(v, p);
34090 else
34091 put_unaligned_le64(v, p);
34092 }
34093  
34094  
34095  
34096  
34097  
34098  
34099  
34100 static forceinline unsigned
34101 bsr32(u32 v)
34102 {
34103 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
34104 return 31 - __builtin_clz(v);
34105 #elif defined(_MSC_VER)
34106 unsigned long i;
34107  
34108 _BitScanReverse(&i, v);
34109 return i;
34110 #else
34111 unsigned i = 0;
34112  
34113 while ((v >>= 1) != 0)
34114 i++;
34115 return i;
34116 #endif
34117 }
34118  
34119 static forceinline unsigned
34120 bsr64(u64 v)
34121 {
34122 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
34123 return 63 - __builtin_clzll(v);
34124 #elif defined(_MSC_VER) && defined(_WIN64)
34125 unsigned long i;
34126  
34127 _BitScanReverse64(&i, v);
34128 return i;
34129 #else
34130 unsigned i = 0;
34131  
34132 while ((v >>= 1) != 0)
34133 i++;
34134 return i;
34135 #endif
34136 }
34137  
34138 static forceinline unsigned
34139 bsrw(machine_word_t v)
34140 {
34141 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
34142 if (WORDBITS == 32)
34143 return bsr32(v);
34144 else
34145 return bsr64(v);
34146 }
34147  
34148  
34149  
34150 static forceinline unsigned
34151 bsf32(u32 v)
34152 {
34153 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
34154 return __builtin_ctz(v);
34155 #elif defined(_MSC_VER)
34156 unsigned long i;
34157  
34158 _BitScanForward(&i, v);
34159 return i;
34160 #else
34161 unsigned i = 0;
34162  
34163 for (; (v & 1) == 0; v >>= 1)
34164 i++;
34165 return i;
34166 #endif
34167 }
34168  
34169 static forceinline unsigned
34170 bsf64(u64 v)
34171 {
34172 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
34173 return __builtin_ctzll(v);
34174 #elif defined(_MSC_VER) && defined(_WIN64)
34175 unsigned long i;
34176  
34177 _BitScanForward64(&i, v);
34178 return i;
34179 #else
34180 unsigned i = 0;
34181  
34182 for (; (v & 1) == 0; v >>= 1)
34183 i++;
34184 return i;
34185 #endif
34186 }
34187  
34188 static forceinline unsigned
34189 bsfw(machine_word_t v)
34190 {
34191 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
34192 if (WORDBITS == 32)
34193 return bsf32(v);
34194 else
34195 return bsf64(v);
34196 }
34197  
34198  
34199 #undef rbit32
34200 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
34201 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
34202 static forceinline u32
34203 rbit32(u32 v)
34204 {
34205 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
34206 return v;
34207 }
34208 #define rbit32 rbit32
34209 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
34210 static forceinline u32
34211 rbit32(u32 v)
34212 {
34213 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
34214 return v;
34215 }
34216 #define rbit32 rbit32
34217 #endif
34218  
34219 #endif
34220  
34221  
34222 typedef void *(*malloc_func_t)(size_t);
34223 typedef void (*free_func_t)(void *);
34224  
34225 extern malloc_func_t libdeflate_default_malloc_func;
34226 extern free_func_t libdeflate_default_free_func;
34227  
34228 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
34229 size_t alignment, size_t size);
34230 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
34231  
34232 #ifdef FREESTANDING
34233  
34234 void *memset(void *s, int c, size_t n);
34235 #define memset(s, c, n) __builtin_memset((s), (c), (n))
34236  
34237 void *memcpy(void *dest, const void *src, size_t n);
34238 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
34239  
34240 void *memmove(void *dest, const void *src, size_t n);
34241 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
34242  
34243 int memcmp(const void *s1, const void *s2, size_t n);
34244 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
34245  
34246 #undef LIBDEFLATE_ENABLE_ASSERTIONS
34247 #else
34248 # include <string.h>
34249
34250 # ifdef __clang_analyzer__
34251 # define LIBDEFLATE_ENABLE_ASSERTIONS
34252 # endif
34253 #endif
34254  
34255  
34256 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
34257 NORETURN void
34258 libdeflate_assertion_failed(const char *expr, const char *file, int line);
34259 #define ASSERT(expr) { if (unlikely(!(expr))) \
34260 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
34261 #else
34262 #define ASSERT(expr) (void)(expr)
34263 #endif
34264  
34265 #define CONCAT_IMPL(a, b) a##b
34266 #define CONCAT(a, b) CONCAT_IMPL(a, b)
34267 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
34268  
34269 #endif
34270  
34271  
34272 #ifdef FREESTANDING
34273 # define malloc NULL
34274 # define free NULL
34275 #else
34276 # include <stdlib.h>
34277 #endif
34278  
34279 malloc_func_t libdeflate_default_malloc_func = malloc;
34280 free_func_t libdeflate_default_free_func = free;
34281  
34282 void *
34283 37 libdeflate_aligned_malloc(malloc_func_t malloc_func,
34284 size_t alignment, size_t size)
34285 {
34286 37 void *ptr = (*malloc_func)(sizeof(void *) + alignment - 1 + size);
34287  
34288 37 ▶ if (ptr) {
Branch: 1 branch
BranchTrueFalseTotal
if (ptr) { 37 0 37
34289 37 void *orig_ptr = ptr;
34290  
34291 37 ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
34292 37 ((void **)ptr)[-1] = orig_ptr;
34293 }
34294 37 return ptr;
34295 }
34296  
34297 void
34298 37 libdeflate_aligned_free(free_func_t free_func, void *ptr)
34299 {
34300 37 (*free_func)(((void **)ptr)[-1]);
34301 37 }
34302  
34303 LIBDEFLATEAPI void
34304 0 libdeflate_set_memory_allocator(malloc_func_t malloc_func,
34305 free_func_t free_func)
34306 {
34307 0 libdeflate_default_malloc_func = malloc_func;
34308 0 libdeflate_default_free_func = free_func;
34309 0 }
34310  
34311  
34312 #ifdef FREESTANDING
34313 #undef memset
34314 void * __attribute__((weak))
34315 memset(void *s, int c, size_t n)
34316 {
34317 u8 *p = s;
34318 size_t i;
34319  
34320 for (i = 0; i < n; i++)
34321 p[i] = c;
34322 return s;
34323 }
34324  
34325 #undef memcpy
34326 void * __attribute__((weak))
34327 memcpy(void *dest, const void *src, size_t n)
34328 {
34329 u8 *d = dest;
34330 const u8 *s = src;
34331 size_t i;
34332  
34333 for (i = 0; i < n; i++)
34334 d[i] = s[i];
34335 return dest;
34336 }
34337  
34338 #undef memmove
34339 void * __attribute__((weak))
34340 memmove(void *dest, const void *src, size_t n)
34341 {
34342 u8 *d = dest;
34343 const u8 *s = src;
34344 size_t i;
34345  
34346 if (d <= s)
34347 return memcpy(d, s, n);
34348  
34349 for (i = n; i > 0; i--)
34350 d[i - 1] = s[i - 1];
34351 return dest;
34352 }
34353  
34354 #undef memcmp
34355 int __attribute__((weak))
34356 memcmp(const void *s1, const void *s2, size_t n)
34357 {
34358 const u8 *p1 = s1;
34359 const u8 *p2 = s2;
34360 size_t i;
34361  
34362 for (i = 0; i < n; i++) {
34363 if (p1[i] != p2[i])
34364 return (int)p1[i] - (int)p2[i];
34365 }
34366 return 0;
34367 }
34368 #endif
34369  
34370 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
34371 #include <stdio.h>
34372 #include <stdlib.h>
34373 NORETURN void
34374 libdeflate_assertion_failed(const char *expr, const char *file, int line)
34375 {
34376 fprintf(stderr, "Assertion failed: %s at %s:%d\n", expr, file, line);
34377 abort();
34378 }
34379 #endif
34380 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/zlib_compress.c */
34381  
34382  
34383 /* #include "deflate_compress.h" */
34384 #ifndef LIB_DEFLATE_COMPRESS_H
34385 #define LIB_DEFLATE_COMPRESS_H
34386  
34387 /* #include "lib_common.h" */
34388  
34389  
34390 #ifndef LIB_LIB_COMMON_H
34391 #define LIB_LIB_COMMON_H
34392  
34393 #ifdef LIBDEFLATE_H
34394
34395 # error "lib_common.h must always be included before libdeflate.h"
34396 #endif
34397  
34398 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
34399 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
34400 #elif defined(__GNUC__)
34401 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
34402 #else
34403 # define LIBDEFLATE_EXPORT_SYM
34404 #endif
34405  
34406  
34407 #if defined(__GNUC__) && defined(__i386__)
34408 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
34409 #else
34410 # define LIBDEFLATE_ALIGN_STACK
34411 #endif
34412  
34413 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
34414  
34415 /* #include "../common_defs.h" */
34416  
34417  
34418 #ifndef COMMON_DEFS_H
34419 #define COMMON_DEFS_H
34420  
34421 /* #include "libdeflate.h" */
34422  
34423  
34424 #ifndef LIBDEFLATE_H
34425 #define LIBDEFLATE_H
34426  
34427 #include <stddef.h>
34428 #include <stdint.h>
34429  
34430 #ifdef __cplusplus
34431 extern "C" {
34432 #endif
34433  
34434 #define LIBDEFLATE_VERSION_MAJOR 1
34435 #define LIBDEFLATE_VERSION_MINOR 26
34436 #define LIBDEFLATE_VERSION_STRING "1.26"
34437  
34438  
34439 #ifndef LIBDEFLATEAPI
34440 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
34441 # define LIBDEFLATEAPI __declspec(dllimport)
34442 # else
34443 # define LIBDEFLATEAPI
34444 # endif
34445 #endif
34446  
34447  
34448  
34449  
34450  
34451 struct libdeflate_compressor;
34452 struct libdeflate_options;
34453  
34454  
34455 LIBDEFLATEAPI struct libdeflate_compressor *
34456 libdeflate_alloc_compressor(int compression_level);
34457  
34458  
34459 LIBDEFLATEAPI struct libdeflate_compressor *
34460 libdeflate_alloc_compressor_ex(int compression_level,
34461 const struct libdeflate_options *options);
34462  
34463  
34464 LIBDEFLATEAPI size_t
34465 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
34466 const void *in, size_t in_nbytes,
34467 void *out, size_t out_nbytes_avail);
34468  
34469  
34470 LIBDEFLATEAPI size_t
34471 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
34472 size_t in_nbytes);
34473  
34474  
34475 LIBDEFLATEAPI size_t
34476 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
34477 const void *in, size_t in_nbytes,
34478 void *out, size_t out_nbytes_avail);
34479  
34480  
34481 LIBDEFLATEAPI size_t
34482 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
34483 size_t in_nbytes);
34484  
34485  
34486 LIBDEFLATEAPI size_t
34487 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
34488 const void *in, size_t in_nbytes,
34489 void *out, size_t out_nbytes_avail);
34490  
34491  
34492 LIBDEFLATEAPI size_t
34493 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
34494 size_t in_nbytes);
34495  
34496  
34497 LIBDEFLATEAPI void
34498 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
34499  
34500  
34501  
34502  
34503  
34504 struct libdeflate_decompressor;
34505 struct libdeflate_options;
34506  
34507  
34508 LIBDEFLATEAPI struct libdeflate_decompressor *
34509 libdeflate_alloc_decompressor(void);
34510  
34511  
34512 LIBDEFLATEAPI struct libdeflate_decompressor *
34513 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
34514  
34515  
34516 enum libdeflate_result {
34517
34518 LIBDEFLATE_SUCCESS = 0,
34519  
34520
34521 LIBDEFLATE_BAD_DATA = 1,
34522  
34523
34524 LIBDEFLATE_SHORT_OUTPUT = 2,
34525  
34526
34527 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
34528 };
34529  
34530  
34531 LIBDEFLATEAPI enum libdeflate_result
34532 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
34533 const void *in, size_t in_nbytes,
34534 void *out, size_t out_nbytes_avail,
34535 size_t *actual_out_nbytes_ret);
34536  
34537  
34538 LIBDEFLATEAPI enum libdeflate_result
34539 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
34540 const void *in, size_t in_nbytes,
34541 void *out, size_t out_nbytes_avail,
34542 size_t *actual_in_nbytes_ret,
34543 size_t *actual_out_nbytes_ret);
34544  
34545  
34546 LIBDEFLATEAPI enum libdeflate_result
34547 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
34548 const void *in, size_t in_nbytes,
34549 void *out, size_t out_nbytes_avail,
34550 size_t *actual_out_nbytes_ret);
34551  
34552  
34553 LIBDEFLATEAPI enum libdeflate_result
34554 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
34555 const void *in, size_t in_nbytes,
34556 void *out, size_t out_nbytes_avail,
34557 size_t *actual_in_nbytes_ret,
34558 size_t *actual_out_nbytes_ret);
34559  
34560  
34561 LIBDEFLATEAPI enum libdeflate_result
34562 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
34563 const void *in, size_t in_nbytes,
34564 void *out, size_t out_nbytes_avail,
34565 size_t *actual_out_nbytes_ret);
34566  
34567  
34568 LIBDEFLATEAPI enum libdeflate_result
34569 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
34570 const void *in, size_t in_nbytes,
34571 void *out, size_t out_nbytes_avail,
34572 size_t *actual_in_nbytes_ret,
34573 size_t *actual_out_nbytes_ret);
34574  
34575  
34576 LIBDEFLATEAPI void
34577 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
34578  
34579  
34580  
34581  
34582  
34583  
34584 LIBDEFLATEAPI uint32_t
34585 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
34586  
34587  
34588  
34589 LIBDEFLATEAPI uint32_t
34590 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
34591  
34592  
34593  
34594  
34595  
34596  
34597 LIBDEFLATEAPI void
34598 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
34599 void (*free_func)(void *));
34600  
34601  
34602 struct libdeflate_options {
34603  
34604
34605 size_t sizeof_options;
34606  
34607
34608 void *(*malloc_func)(size_t);
34609 void (*free_func)(void *);
34610 };
34611  
34612 #ifdef __cplusplus
34613 }
34614 #endif
34615  
34616 #endif
34617  
34618  
34619 #include <stdbool.h>
34620 #include <stddef.h>
34621 #include <stdint.h>
34622 #ifdef _MSC_VER
34623 # include <intrin.h>
34624 # include <stdlib.h>
34625
34626
34627 # pragma warning(disable : 4146)
34628
34629 # pragma warning(disable : 4018)
34630 # pragma warning(disable : 4244)
34631 # pragma warning(disable : 4267)
34632 # pragma warning(disable : 4310)
34633
34634 # pragma warning(disable : 4100)
34635 # pragma warning(disable : 4127)
34636 # pragma warning(disable : 4189)
34637 # pragma warning(disable : 4232)
34638 # pragma warning(disable : 4245)
34639 # pragma warning(disable : 4295)
34640 #endif
34641 #ifndef FREESTANDING
34642 # include <string.h>
34643 #endif
34644  
34645  
34646  
34647  
34648  
34649  
34650 #undef ARCH_X86_64
34651 #undef ARCH_X86_32
34652 #undef ARCH_ARM64
34653 #undef ARCH_ARM32
34654 #undef ARCH_RISCV
34655 #ifdef _MSC_VER
34656
34657 # if defined(_M_X64) && !defined(_M_ARM64EC)
34658 # define ARCH_X86_64
34659 # elif defined(_M_IX86)
34660 # define ARCH_X86_32
34661 # elif defined(_M_ARM64)
34662 # define ARCH_ARM64
34663 # elif defined(_M_ARM)
34664 # define ARCH_ARM32
34665 # endif
34666 #else
34667 # if defined(__x86_64__)
34668 # define ARCH_X86_64
34669 # elif defined(__i386__)
34670 # define ARCH_X86_32
34671 # elif defined(__aarch64__)
34672 # define ARCH_ARM64
34673 # elif defined(__arm__)
34674 # define ARCH_ARM32
34675 # elif defined(__riscv)
34676 # define ARCH_RISCV
34677 # endif
34678 #endif
34679  
34680  
34681  
34682  
34683  
34684  
34685 typedef uint8_t u8;
34686 typedef uint16_t u16;
34687 typedef uint32_t u32;
34688 typedef uint64_t u64;
34689 typedef int8_t s8;
34690 typedef int16_t s16;
34691 typedef int32_t s32;
34692 typedef int64_t s64;
34693  
34694  
34695 #ifdef _MSC_VER
34696 # ifdef _WIN64
34697 typedef long long ssize_t;
34698 # else
34699 typedef long ssize_t;
34700 # endif
34701 #endif
34702  
34703  
34704 typedef size_t machine_word_t;
34705  
34706  
34707 #define WORDBYTES ((int)sizeof(machine_word_t))
34708  
34709  
34710 #define WORDBITS (8 * WORDBYTES)
34711  
34712  
34713  
34714  
34715  
34716  
34717 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
34718 # define GCC_PREREQ(major, minor) \
34719 (__GNUC__ > (major) || \
34720 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
34721 # if !GCC_PREREQ(4, 9)
34722 # error "gcc versions older than 4.9 are no longer supported"
34723 # endif
34724 #else
34725 # define GCC_PREREQ(major, minor) 0
34726 #endif
34727 #ifdef __clang__
34728 # ifdef __apple_build_version__
34729 # define CLANG_PREREQ(major, minor, apple_version) \
34730 (__apple_build_version__ >= (apple_version))
34731 # else
34732 # define CLANG_PREREQ(major, minor, apple_version) \
34733 (__clang_major__ > (major) || \
34734 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
34735 # endif
34736 # if !CLANG_PREREQ(3, 9, 8000000)
34737 # error "clang versions older than 3.9 are no longer supported"
34738 # endif
34739 #else
34740 # define CLANG_PREREQ(major, minor, apple_version) 0
34741 #endif
34742 #ifdef _MSC_VER
34743 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
34744 # if !MSVC_PREREQ(1928)
34745 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
34746 # endif
34747 #else
34748 # define MSVC_PREREQ(version) 0
34749 #endif
34750  
34751  
34752 #ifndef __has_attribute
34753 # define __has_attribute(attribute) 0
34754 #endif
34755  
34756  
34757 #ifndef __has_builtin
34758 # define __has_builtin(builtin) 0
34759 #endif
34760  
34761  
34762 #ifdef _MSC_VER
34763 # define inline __inline
34764 #endif
34765  
34766  
34767 #if defined(__GNUC__) || __has_attribute(always_inline)
34768 # define forceinline inline __attribute__((always_inline))
34769 #elif defined(_MSC_VER)
34770 # define forceinline __forceinline
34771 #else
34772 # define forceinline inline
34773 #endif
34774  
34775  
34776 #if defined(__GNUC__) || __has_attribute(unused)
34777 # define MAYBE_UNUSED __attribute__((unused))
34778 #else
34779 # define MAYBE_UNUSED
34780 #endif
34781  
34782  
34783 #if defined(__GNUC__) || __has_attribute(noreturn)
34784 # define NORETURN __attribute__((noreturn))
34785 #else
34786 # define NORETURN
34787 #endif
34788  
34789  
34790 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
34791 # define likely(expr) __builtin_expect(!!(expr), 1)
34792 #else
34793 # define likely(expr) (expr)
34794 #endif
34795  
34796  
34797 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
34798 # define unlikely(expr) __builtin_expect(!!(expr), 0)
34799 #else
34800 # define unlikely(expr) (expr)
34801 #endif
34802  
34803  
34804 #undef prefetchr
34805 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
34806 # define prefetchr(addr) __builtin_prefetch((addr), 0)
34807 #elif defined(_MSC_VER)
34808 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
34809 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
34810 # elif defined(ARCH_ARM64)
34811 # define prefetchr(addr) __prefetch2((addr), 0x00 )
34812 # elif defined(ARCH_ARM32)
34813 # define prefetchr(addr) __prefetch(addr)
34814 # endif
34815 #endif
34816 #ifndef prefetchr
34817 # define prefetchr(addr)
34818 #endif
34819  
34820  
34821 #undef prefetchw
34822 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
34823 # define prefetchw(addr) __builtin_prefetch((addr), 1)
34824 #elif defined(_MSC_VER)
34825 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
34826 # define prefetchw(addr) _m_prefetchw(addr)
34827 # elif defined(ARCH_ARM64)
34828 # define prefetchw(addr) __prefetch2((addr), 0x10 )
34829 # elif defined(ARCH_ARM32)
34830 # define prefetchw(addr) __prefetchw(addr)
34831 # endif
34832 #endif
34833 #ifndef prefetchw
34834 # define prefetchw(addr)
34835 #endif
34836  
34837  
34838 #undef _aligned_attribute
34839 #if defined(__GNUC__) || __has_attribute(aligned)
34840 # define _aligned_attribute(n) __attribute__((aligned(n)))
34841 #elif defined(_MSC_VER)
34842 # define _aligned_attribute(n) __declspec(align(n))
34843 #endif
34844  
34845  
34846 #if defined(__GNUC__) || __has_attribute(target)
34847 # define _target_attribute(attrs) __attribute__((target(attrs)))
34848 #else
34849 # define _target_attribute(attrs)
34850 #endif
34851  
34852  
34853  
34854  
34855  
34856 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
34857 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
34858 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
34859 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
34860 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
34861 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
34862 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
34863  
34864  
34865  
34866  
34867  
34868  
34869 #if defined(__BYTE_ORDER__)
34870 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
34871 #elif defined(_MSC_VER)
34872 # define CPU_IS_LITTLE_ENDIAN() true
34873 #else
34874 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
34875 {
34876 union {
34877 u32 w;
34878 u8 b;
34879 } u;
34880  
34881 u.w = 1;
34882 return u.b;
34883 }
34884 #endif
34885  
34886  
34887 static forceinline u16 bswap16(u16 v)
34888 {
34889 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
34890 return __builtin_bswap16(v);
34891 #elif defined(_MSC_VER)
34892 return _byteswap_ushort(v);
34893 #else
34894 return (v << 8) | (v >> 8);
34895 #endif
34896 }
34897  
34898  
34899 static forceinline u32 bswap32(u32 v)
34900 {
34901 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
34902 return __builtin_bswap32(v);
34903 #elif defined(_MSC_VER)
34904 return _byteswap_ulong(v);
34905 #else
34906 return ((v & 0x000000FF) << 24) |
34907 ((v & 0x0000FF00) << 8) |
34908 ((v & 0x00FF0000) >> 8) |
34909 ((v & 0xFF000000) >> 24);
34910 #endif
34911 }
34912  
34913  
34914 static forceinline u64 bswap64(u64 v)
34915 {
34916 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
34917 return __builtin_bswap64(v);
34918 #elif defined(_MSC_VER)
34919 return _byteswap_uint64(v);
34920 #else
34921 return ((v & 0x00000000000000FF) << 56) |
34922 ((v & 0x000000000000FF00) << 40) |
34923 ((v & 0x0000000000FF0000) << 24) |
34924 ((v & 0x00000000FF000000) << 8) |
34925 ((v & 0x000000FF00000000) >> 8) |
34926 ((v & 0x0000FF0000000000) >> 24) |
34927 ((v & 0x00FF000000000000) >> 40) |
34928 ((v & 0xFF00000000000000) >> 56);
34929 #endif
34930 }
34931  
34932 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
34933 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
34934 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
34935 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
34936 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
34937 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
34938  
34939  
34940  
34941  
34942  
34943  
34944 #if (defined(__GNUC__) || defined(__clang__)) && \
34945 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
34946 defined(__ARM_FEATURE_UNALIGNED) || \
34947 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
34948 defined(__riscv_misaligned_fast) || \
34949 defined(__wasm__))
34950 # define UNALIGNED_ACCESS_IS_FAST 1
34951 #elif defined(_MSC_VER)
34952 # define UNALIGNED_ACCESS_IS_FAST 1
34953 #else
34954 # define UNALIGNED_ACCESS_IS_FAST 0
34955 #endif
34956  
34957  
34958  
34959 #ifdef FREESTANDING
34960 # define MEMCOPY __builtin_memcpy
34961 #else
34962 # define MEMCOPY memcpy
34963 #endif
34964  
34965  
34966  
34967 #define DEFINE_UNALIGNED_TYPE(type) \
34968 static forceinline type \
34969 load_##type##_unaligned(const void *p) \
34970 { \
34971 type v; \
34972 \
34973 MEMCOPY(&v, p, sizeof(v)); \
34974 return v; \
34975 } \
34976 \
34977 static forceinline void \
34978 store_##type##_unaligned(type v, void *p) \
34979 { \
34980 MEMCOPY(p, &v, sizeof(v)); \
34981 }
34982  
34983 DEFINE_UNALIGNED_TYPE(u16)
34984 DEFINE_UNALIGNED_TYPE(u32)
34985 DEFINE_UNALIGNED_TYPE(u64)
34986 DEFINE_UNALIGNED_TYPE(machine_word_t)
34987  
34988 #undef MEMCOPY
34989  
34990 #define load_word_unaligned load_machine_word_t_unaligned
34991 #define store_word_unaligned store_machine_word_t_unaligned
34992  
34993  
34994  
34995 static forceinline u16
34996 get_unaligned_le16(const u8 *p)
34997 {
34998 if (UNALIGNED_ACCESS_IS_FAST)
34999 return le16_bswap(load_u16_unaligned(p));
35000 else
35001 return ((u16)p[1] << 8) | p[0];
35002 }
35003  
35004 static forceinline u16
35005 get_unaligned_be16(const u8 *p)
35006 {
35007 if (UNALIGNED_ACCESS_IS_FAST)
35008 return be16_bswap(load_u16_unaligned(p));
35009 else
35010 return ((u16)p[0] << 8) | p[1];
35011 }
35012  
35013 static forceinline u32
35014 get_unaligned_le32(const u8 *p)
35015 {
35016 if (UNALIGNED_ACCESS_IS_FAST)
35017 return le32_bswap(load_u32_unaligned(p));
35018 else
35019 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
35020 ((u32)p[1] << 8) | p[0];
35021 }
35022  
35023 static forceinline u32
35024 get_unaligned_be32(const u8 *p)
35025 {
35026 if (UNALIGNED_ACCESS_IS_FAST)
35027 return be32_bswap(load_u32_unaligned(p));
35028 else
35029 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
35030 ((u32)p[2] << 8) | p[3];
35031 }
35032  
35033 static forceinline u64
35034 get_unaligned_le64(const u8 *p)
35035 {
35036 if (UNALIGNED_ACCESS_IS_FAST)
35037 return le64_bswap(load_u64_unaligned(p));
35038 else
35039 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
35040 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
35041 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
35042 ((u64)p[1] << 8) | p[0];
35043 }
35044  
35045 static forceinline machine_word_t
35046 get_unaligned_leword(const u8 *p)
35047 {
35048 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
35049 if (WORDBITS == 32)
35050 return get_unaligned_le32(p);
35051 else
35052 return get_unaligned_le64(p);
35053 }
35054  
35055  
35056  
35057 static forceinline void
35058 put_unaligned_le16(u16 v, u8 *p)
35059 {
35060 if (UNALIGNED_ACCESS_IS_FAST) {
35061 store_u16_unaligned(le16_bswap(v), p);
35062 } else {
35063 p[0] = (u8)(v >> 0);
35064 p[1] = (u8)(v >> 8);
35065 }
35066 }
35067  
35068 static forceinline void
35069 put_unaligned_be16(u16 v, u8 *p)
35070 {
35071 if (UNALIGNED_ACCESS_IS_FAST) {
35072 store_u16_unaligned(be16_bswap(v), p);
35073 } else {
35074 p[0] = (u8)(v >> 8);
35075 p[1] = (u8)(v >> 0);
35076 }
35077 }
35078  
35079 static forceinline void
35080 put_unaligned_le32(u32 v, u8 *p)
35081 {
35082 if (UNALIGNED_ACCESS_IS_FAST) {
35083 store_u32_unaligned(le32_bswap(v), p);
35084 } else {
35085 p[0] = (u8)(v >> 0);
35086 p[1] = (u8)(v >> 8);
35087 p[2] = (u8)(v >> 16);
35088 p[3] = (u8)(v >> 24);
35089 }
35090 }
35091  
35092 static forceinline void
35093 put_unaligned_be32(u32 v, u8 *p)
35094 {
35095 if (UNALIGNED_ACCESS_IS_FAST) {
35096 store_u32_unaligned(be32_bswap(v), p);
35097 } else {
35098 p[0] = (u8)(v >> 24);
35099 p[1] = (u8)(v >> 16);
35100 p[2] = (u8)(v >> 8);
35101 p[3] = (u8)(v >> 0);
35102 }
35103 }
35104  
35105 static forceinline void
35106 put_unaligned_le64(u64 v, u8 *p)
35107 {
35108 if (UNALIGNED_ACCESS_IS_FAST) {
35109 store_u64_unaligned(le64_bswap(v), p);
35110 } else {
35111 p[0] = (u8)(v >> 0);
35112 p[1] = (u8)(v >> 8);
35113 p[2] = (u8)(v >> 16);
35114 p[3] = (u8)(v >> 24);
35115 p[4] = (u8)(v >> 32);
35116 p[5] = (u8)(v >> 40);
35117 p[6] = (u8)(v >> 48);
35118 p[7] = (u8)(v >> 56);
35119 }
35120 }
35121  
35122 static forceinline void
35123 put_unaligned_leword(machine_word_t v, u8 *p)
35124 {
35125 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
35126 if (WORDBITS == 32)
35127 put_unaligned_le32(v, p);
35128 else
35129 put_unaligned_le64(v, p);
35130 }
35131  
35132  
35133  
35134  
35135  
35136  
35137  
35138 static forceinline unsigned
35139 bsr32(u32 v)
35140 {
35141 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
35142 return 31 - __builtin_clz(v);
35143 #elif defined(_MSC_VER)
35144 unsigned long i;
35145  
35146 _BitScanReverse(&i, v);
35147 return i;
35148 #else
35149 unsigned i = 0;
35150  
35151 while ((v >>= 1) != 0)
35152 i++;
35153 return i;
35154 #endif
35155 }
35156  
35157 static forceinline unsigned
35158 bsr64(u64 v)
35159 {
35160 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
35161 return 63 - __builtin_clzll(v);
35162 #elif defined(_MSC_VER) && defined(_WIN64)
35163 unsigned long i;
35164  
35165 _BitScanReverse64(&i, v);
35166 return i;
35167 #else
35168 unsigned i = 0;
35169  
35170 while ((v >>= 1) != 0)
35171 i++;
35172 return i;
35173 #endif
35174 }
35175  
35176 static forceinline unsigned
35177 bsrw(machine_word_t v)
35178 {
35179 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
35180 if (WORDBITS == 32)
35181 return bsr32(v);
35182 else
35183 return bsr64(v);
35184 }
35185  
35186  
35187  
35188 static forceinline unsigned
35189 bsf32(u32 v)
35190 {
35191 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
35192 return __builtin_ctz(v);
35193 #elif defined(_MSC_VER)
35194 unsigned long i;
35195  
35196 _BitScanForward(&i, v);
35197 return i;
35198 #else
35199 unsigned i = 0;
35200  
35201 for (; (v & 1) == 0; v >>= 1)
35202 i++;
35203 return i;
35204 #endif
35205 }
35206  
35207 static forceinline unsigned
35208 bsf64(u64 v)
35209 {
35210 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
35211 return __builtin_ctzll(v);
35212 #elif defined(_MSC_VER) && defined(_WIN64)
35213 unsigned long i;
35214  
35215 _BitScanForward64(&i, v);
35216 return i;
35217 #else
35218 unsigned i = 0;
35219  
35220 for (; (v & 1) == 0; v >>= 1)
35221 i++;
35222 return i;
35223 #endif
35224 }
35225  
35226 static forceinline unsigned
35227 bsfw(machine_word_t v)
35228 {
35229 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
35230 if (WORDBITS == 32)
35231 return bsf32(v);
35232 else
35233 return bsf64(v);
35234 }
35235  
35236  
35237 #undef rbit32
35238 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
35239 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
35240 static forceinline u32
35241 rbit32(u32 v)
35242 {
35243 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
35244 return v;
35245 }
35246 #define rbit32 rbit32
35247 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
35248 static forceinline u32
35249 rbit32(u32 v)
35250 {
35251 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
35252 return v;
35253 }
35254 #define rbit32 rbit32
35255 #endif
35256  
35257 #endif
35258  
35259  
35260 typedef void *(*malloc_func_t)(size_t);
35261 typedef void (*free_func_t)(void *);
35262  
35263 extern malloc_func_t libdeflate_default_malloc_func;
35264 extern free_func_t libdeflate_default_free_func;
35265  
35266 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
35267 size_t alignment, size_t size);
35268 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
35269  
35270 #ifdef FREESTANDING
35271  
35272 void *memset(void *s, int c, size_t n);
35273 #define memset(s, c, n) __builtin_memset((s), (c), (n))
35274  
35275 void *memcpy(void *dest, const void *src, size_t n);
35276 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
35277  
35278 void *memmove(void *dest, const void *src, size_t n);
35279 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
35280  
35281 int memcmp(const void *s1, const void *s2, size_t n);
35282 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
35283  
35284 #undef LIBDEFLATE_ENABLE_ASSERTIONS
35285 #else
35286 # include <string.h>
35287
35288 # ifdef __clang_analyzer__
35289 # define LIBDEFLATE_ENABLE_ASSERTIONS
35290 # endif
35291 #endif
35292  
35293  
35294 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
35295 NORETURN void
35296 libdeflate_assertion_failed(const char *expr, const char *file, int line);
35297 #define ASSERT(expr) { if (unlikely(!(expr))) \
35298 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
35299 #else
35300 #define ASSERT(expr) (void)(expr)
35301 #endif
35302  
35303 #define CONCAT_IMPL(a, b) a##b
35304 #define CONCAT(a, b) CONCAT_IMPL(a, b)
35305 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
35306  
35307 #endif
35308  
35309  
35310  
35311  
35312 struct libdeflate_compressor;
35313  
35314 unsigned int libdeflate_get_compression_level(struct libdeflate_compressor *c);
35315  
35316 #endif
35317  
35318 /* #include "zlib_constants.h" */
35319  
35320  
35321 #ifndef LIB_ZLIB_CONSTANTS_H
35322 #define LIB_ZLIB_CONSTANTS_H
35323  
35324 #define ZLIB_MIN_HEADER_SIZE 2
35325 #define ZLIB_FOOTER_SIZE 4
35326 #define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)
35327  
35328 #define ZLIB_CM_DEFLATE 8
35329  
35330 #define ZLIB_CINFO_32K_WINDOW 7
35331  
35332 #define ZLIB_FASTEST_COMPRESSION 0
35333 #define ZLIB_FAST_COMPRESSION 1
35334 #define ZLIB_DEFAULT_COMPRESSION 2
35335 #define ZLIB_SLOWEST_COMPRESSION 3
35336  
35337 #endif
35338  
35339  
35340 LIBDEFLATEAPI size_t
35341 12 libdeflate_zlib_compress(struct libdeflate_compressor *c,
35342 const void *in, size_t in_nbytes,
35343 void *out, size_t out_nbytes_avail)
35344 {
35345 12 u8 *out_next = out;
35346 u16 hdr;
35347 unsigned compression_level;
35348 unsigned level_hint;
35349 size_t deflate_size;
35350  
35351 12 ▶ if (out_nbytes_avail <= ZLIB_MIN_OVERHEAD)
Branch: 1 branch
BranchTrueFalseTotal
if (out_nbytes_avail <= ZLIB_MIN_OVERHEAD) 0 12 12
35352 0 return 0;
35353  
35354
35355 12 hdr = (ZLIB_CM_DEFLATE << 8) | (ZLIB_CINFO_32K_WINDOW << 12);
35356 12 compression_level = libdeflate_get_compression_level(c);
35357 12 ▶ if (compression_level < 2)
Branch: 1 branch
BranchTrueFalseTotal
if (compression_level < 2) 1 11 12
35358 1 level_hint = ZLIB_FASTEST_COMPRESSION;
35359 11 ▶ else if (compression_level < 6)
Branch: 1 branch
BranchTrueFalseTotal
else if (compression_level < 6) 4 7 11
35360 4 level_hint = ZLIB_FAST_COMPRESSION;
35361 7 ▶ else if (compression_level < 8)
Branch: 1 branch
BranchTrueFalseTotal
else if (compression_level < 8) 2 5 7
35362 2 level_hint = ZLIB_DEFAULT_COMPRESSION;
35363 else
35364 5 level_hint = ZLIB_SLOWEST_COMPRESSION;
35365 12 hdr |= level_hint << 6;
35366 12 hdr |= 31 - (hdr % 31);
35367  
35368 12 put_unaligned_be16(hdr, out_next);
35369 12 out_next += 2;
35370  
35371
35372 12 deflate_size = libdeflate_deflate_compress(c, in, in_nbytes, out_next,
35373 out_nbytes_avail - ZLIB_MIN_OVERHEAD);
35374 12 ▶ if (deflate_size == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (deflate_size == 0) 0 12 12
35375 0 return 0;
35376 12 out_next += deflate_size;
35377  
35378
35379 12 put_unaligned_be32(libdeflate_adler32(1, in, in_nbytes), out_next);
35380 12 out_next += 4;
35381  
35382 12 return out_next - (u8 *)out;
35383 }
35384  
35385 LIBDEFLATEAPI size_t
35386 12 libdeflate_zlib_compress_bound(struct libdeflate_compressor *c,
35387 size_t in_nbytes)
35388 {
35389 12 return ZLIB_MIN_OVERHEAD +
35390 12 libdeflate_deflate_compress_bound(c, in_nbytes);
35391 }
35392 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/zlib_decompress.c */
35393  
35394  
35395 /* #include "lib_common.h" */
35396  
35397  
35398 #ifndef LIB_LIB_COMMON_H
35399 #define LIB_LIB_COMMON_H
35400  
35401 #ifdef LIBDEFLATE_H
35402
35403 # error "lib_common.h must always be included before libdeflate.h"
35404 #endif
35405  
35406 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
35407 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
35408 #elif defined(__GNUC__)
35409 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
35410 #else
35411 # define LIBDEFLATE_EXPORT_SYM
35412 #endif
35413  
35414  
35415 #if defined(__GNUC__) && defined(__i386__)
35416 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
35417 #else
35418 # define LIBDEFLATE_ALIGN_STACK
35419 #endif
35420  
35421 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
35422  
35423 /* #include "../common_defs.h" */
35424  
35425  
35426 #ifndef COMMON_DEFS_H
35427 #define COMMON_DEFS_H
35428  
35429 /* #include "libdeflate.h" */
35430  
35431  
35432 #ifndef LIBDEFLATE_H
35433 #define LIBDEFLATE_H
35434  
35435 #include <stddef.h>
35436 #include <stdint.h>
35437  
35438 #ifdef __cplusplus
35439 extern "C" {
35440 #endif
35441  
35442 #define LIBDEFLATE_VERSION_MAJOR 1
35443 #define LIBDEFLATE_VERSION_MINOR 26
35444 #define LIBDEFLATE_VERSION_STRING "1.26"
35445  
35446  
35447 #ifndef LIBDEFLATEAPI
35448 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
35449 # define LIBDEFLATEAPI __declspec(dllimport)
35450 # else
35451 # define LIBDEFLATEAPI
35452 # endif
35453 #endif
35454  
35455  
35456  
35457  
35458  
35459 struct libdeflate_compressor;
35460 struct libdeflate_options;
35461  
35462  
35463 LIBDEFLATEAPI struct libdeflate_compressor *
35464 libdeflate_alloc_compressor(int compression_level);
35465  
35466  
35467 LIBDEFLATEAPI struct libdeflate_compressor *
35468 libdeflate_alloc_compressor_ex(int compression_level,
35469 const struct libdeflate_options *options);
35470  
35471  
35472 LIBDEFLATEAPI size_t
35473 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
35474 const void *in, size_t in_nbytes,
35475 void *out, size_t out_nbytes_avail);
35476  
35477  
35478 LIBDEFLATEAPI size_t
35479 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
35480 size_t in_nbytes);
35481  
35482  
35483 LIBDEFLATEAPI size_t
35484 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
35485 const void *in, size_t in_nbytes,
35486 void *out, size_t out_nbytes_avail);
35487  
35488  
35489 LIBDEFLATEAPI size_t
35490 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
35491 size_t in_nbytes);
35492  
35493  
35494 LIBDEFLATEAPI size_t
35495 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
35496 const void *in, size_t in_nbytes,
35497 void *out, size_t out_nbytes_avail);
35498  
35499  
35500 LIBDEFLATEAPI size_t
35501 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
35502 size_t in_nbytes);
35503  
35504  
35505 LIBDEFLATEAPI void
35506 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
35507  
35508  
35509  
35510  
35511  
35512 struct libdeflate_decompressor;
35513 struct libdeflate_options;
35514  
35515  
35516 LIBDEFLATEAPI struct libdeflate_decompressor *
35517 libdeflate_alloc_decompressor(void);
35518  
35519  
35520 LIBDEFLATEAPI struct libdeflate_decompressor *
35521 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
35522  
35523  
35524 enum libdeflate_result {
35525
35526 LIBDEFLATE_SUCCESS = 0,
35527  
35528
35529 LIBDEFLATE_BAD_DATA = 1,
35530  
35531
35532 LIBDEFLATE_SHORT_OUTPUT = 2,
35533  
35534
35535 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
35536 };
35537  
35538  
35539 LIBDEFLATEAPI enum libdeflate_result
35540 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
35541 const void *in, size_t in_nbytes,
35542 void *out, size_t out_nbytes_avail,
35543 size_t *actual_out_nbytes_ret);
35544  
35545  
35546 LIBDEFLATEAPI enum libdeflate_result
35547 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
35548 const void *in, size_t in_nbytes,
35549 void *out, size_t out_nbytes_avail,
35550 size_t *actual_in_nbytes_ret,
35551 size_t *actual_out_nbytes_ret);
35552  
35553  
35554 LIBDEFLATEAPI enum libdeflate_result
35555 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
35556 const void *in, size_t in_nbytes,
35557 void *out, size_t out_nbytes_avail,
35558 size_t *actual_out_nbytes_ret);
35559  
35560  
35561 LIBDEFLATEAPI enum libdeflate_result
35562 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
35563 const void *in, size_t in_nbytes,
35564 void *out, size_t out_nbytes_avail,
35565 size_t *actual_in_nbytes_ret,
35566 size_t *actual_out_nbytes_ret);
35567  
35568  
35569 LIBDEFLATEAPI enum libdeflate_result
35570 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
35571 const void *in, size_t in_nbytes,
35572 void *out, size_t out_nbytes_avail,
35573 size_t *actual_out_nbytes_ret);
35574  
35575  
35576 LIBDEFLATEAPI enum libdeflate_result
35577 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
35578 const void *in, size_t in_nbytes,
35579 void *out, size_t out_nbytes_avail,
35580 size_t *actual_in_nbytes_ret,
35581 size_t *actual_out_nbytes_ret);
35582  
35583  
35584 LIBDEFLATEAPI void
35585 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
35586  
35587  
35588  
35589  
35590  
35591  
35592 LIBDEFLATEAPI uint32_t
35593 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
35594  
35595  
35596  
35597 LIBDEFLATEAPI uint32_t
35598 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
35599  
35600  
35601  
35602  
35603  
35604  
35605 LIBDEFLATEAPI void
35606 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
35607 void (*free_func)(void *));
35608  
35609  
35610 struct libdeflate_options {
35611  
35612
35613 size_t sizeof_options;
35614  
35615
35616 void *(*malloc_func)(size_t);
35617 void (*free_func)(void *);
35618 };
35619  
35620 #ifdef __cplusplus
35621 }
35622 #endif
35623  
35624 #endif
35625  
35626  
35627 #include <stdbool.h>
35628 #include <stddef.h>
35629 #include <stdint.h>
35630 #ifdef _MSC_VER
35631 # include <intrin.h>
35632 # include <stdlib.h>
35633
35634
35635 # pragma warning(disable : 4146)
35636
35637 # pragma warning(disable : 4018)
35638 # pragma warning(disable : 4244)
35639 # pragma warning(disable : 4267)
35640 # pragma warning(disable : 4310)
35641
35642 # pragma warning(disable : 4100)
35643 # pragma warning(disable : 4127)
35644 # pragma warning(disable : 4189)
35645 # pragma warning(disable : 4232)
35646 # pragma warning(disable : 4245)
35647 # pragma warning(disable : 4295)
35648 #endif
35649 #ifndef FREESTANDING
35650 # include <string.h>
35651 #endif
35652  
35653  
35654  
35655  
35656  
35657  
35658 #undef ARCH_X86_64
35659 #undef ARCH_X86_32
35660 #undef ARCH_ARM64
35661 #undef ARCH_ARM32
35662 #undef ARCH_RISCV
35663 #ifdef _MSC_VER
35664
35665 # if defined(_M_X64) && !defined(_M_ARM64EC)
35666 # define ARCH_X86_64
35667 # elif defined(_M_IX86)
35668 # define ARCH_X86_32
35669 # elif defined(_M_ARM64)
35670 # define ARCH_ARM64
35671 # elif defined(_M_ARM)
35672 # define ARCH_ARM32
35673 # endif
35674 #else
35675 # if defined(__x86_64__)
35676 # define ARCH_X86_64
35677 # elif defined(__i386__)
35678 # define ARCH_X86_32
35679 # elif defined(__aarch64__)
35680 # define ARCH_ARM64
35681 # elif defined(__arm__)
35682 # define ARCH_ARM32
35683 # elif defined(__riscv)
35684 # define ARCH_RISCV
35685 # endif
35686 #endif
35687  
35688  
35689  
35690  
35691  
35692  
35693 typedef uint8_t u8;
35694 typedef uint16_t u16;
35695 typedef uint32_t u32;
35696 typedef uint64_t u64;
35697 typedef int8_t s8;
35698 typedef int16_t s16;
35699 typedef int32_t s32;
35700 typedef int64_t s64;
35701  
35702  
35703 #ifdef _MSC_VER
35704 # ifdef _WIN64
35705 typedef long long ssize_t;
35706 # else
35707 typedef long ssize_t;
35708 # endif
35709 #endif
35710  
35711  
35712 typedef size_t machine_word_t;
35713  
35714  
35715 #define WORDBYTES ((int)sizeof(machine_word_t))
35716  
35717  
35718 #define WORDBITS (8 * WORDBYTES)
35719  
35720  
35721  
35722  
35723  
35724  
35725 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
35726 # define GCC_PREREQ(major, minor) \
35727 (__GNUC__ > (major) || \
35728 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
35729 # if !GCC_PREREQ(4, 9)
35730 # error "gcc versions older than 4.9 are no longer supported"
35731 # endif
35732 #else
35733 # define GCC_PREREQ(major, minor) 0
35734 #endif
35735 #ifdef __clang__
35736 # ifdef __apple_build_version__
35737 # define CLANG_PREREQ(major, minor, apple_version) \
35738 (__apple_build_version__ >= (apple_version))
35739 # else
35740 # define CLANG_PREREQ(major, minor, apple_version) \
35741 (__clang_major__ > (major) || \
35742 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
35743 # endif
35744 # if !CLANG_PREREQ(3, 9, 8000000)
35745 # error "clang versions older than 3.9 are no longer supported"
35746 # endif
35747 #else
35748 # define CLANG_PREREQ(major, minor, apple_version) 0
35749 #endif
35750 #ifdef _MSC_VER
35751 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
35752 # if !MSVC_PREREQ(1928)
35753 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
35754 # endif
35755 #else
35756 # define MSVC_PREREQ(version) 0
35757 #endif
35758  
35759  
35760 #ifndef __has_attribute
35761 # define __has_attribute(attribute) 0
35762 #endif
35763  
35764  
35765 #ifndef __has_builtin
35766 # define __has_builtin(builtin) 0
35767 #endif
35768  
35769  
35770 #ifdef _MSC_VER
35771 # define inline __inline
35772 #endif
35773  
35774  
35775 #if defined(__GNUC__) || __has_attribute(always_inline)
35776 # define forceinline inline __attribute__((always_inline))
35777 #elif defined(_MSC_VER)
35778 # define forceinline __forceinline
35779 #else
35780 # define forceinline inline
35781 #endif
35782  
35783  
35784 #if defined(__GNUC__) || __has_attribute(unused)
35785 # define MAYBE_UNUSED __attribute__((unused))
35786 #else
35787 # define MAYBE_UNUSED
35788 #endif
35789  
35790  
35791 #if defined(__GNUC__) || __has_attribute(noreturn)
35792 # define NORETURN __attribute__((noreturn))
35793 #else
35794 # define NORETURN
35795 #endif
35796  
35797  
35798 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
35799 # define likely(expr) __builtin_expect(!!(expr), 1)
35800 #else
35801 # define likely(expr) (expr)
35802 #endif
35803  
35804  
35805 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
35806 # define unlikely(expr) __builtin_expect(!!(expr), 0)
35807 #else
35808 # define unlikely(expr) (expr)
35809 #endif
35810  
35811  
35812 #undef prefetchr
35813 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
35814 # define prefetchr(addr) __builtin_prefetch((addr), 0)
35815 #elif defined(_MSC_VER)
35816 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
35817 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
35818 # elif defined(ARCH_ARM64)
35819 # define prefetchr(addr) __prefetch2((addr), 0x00 )
35820 # elif defined(ARCH_ARM32)
35821 # define prefetchr(addr) __prefetch(addr)
35822 # endif
35823 #endif
35824 #ifndef prefetchr
35825 # define prefetchr(addr)
35826 #endif
35827  
35828  
35829 #undef prefetchw
35830 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
35831 # define prefetchw(addr) __builtin_prefetch((addr), 1)
35832 #elif defined(_MSC_VER)
35833 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
35834 # define prefetchw(addr) _m_prefetchw(addr)
35835 # elif defined(ARCH_ARM64)
35836 # define prefetchw(addr) __prefetch2((addr), 0x10 )
35837 # elif defined(ARCH_ARM32)
35838 # define prefetchw(addr) __prefetchw(addr)
35839 # endif
35840 #endif
35841 #ifndef prefetchw
35842 # define prefetchw(addr)
35843 #endif
35844  
35845  
35846 #undef _aligned_attribute
35847 #if defined(__GNUC__) || __has_attribute(aligned)
35848 # define _aligned_attribute(n) __attribute__((aligned(n)))
35849 #elif defined(_MSC_VER)
35850 # define _aligned_attribute(n) __declspec(align(n))
35851 #endif
35852  
35853  
35854 #if defined(__GNUC__) || __has_attribute(target)
35855 # define _target_attribute(attrs) __attribute__((target(attrs)))
35856 #else
35857 # define _target_attribute(attrs)
35858 #endif
35859  
35860  
35861  
35862  
35863  
35864 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
35865 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
35866 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
35867 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
35868 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
35869 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
35870 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
35871  
35872  
35873  
35874  
35875  
35876  
35877 #if defined(__BYTE_ORDER__)
35878 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
35879 #elif defined(_MSC_VER)
35880 # define CPU_IS_LITTLE_ENDIAN() true
35881 #else
35882 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
35883 {
35884 union {
35885 u32 w;
35886 u8 b;
35887 } u;
35888  
35889 u.w = 1;
35890 return u.b;
35891 }
35892 #endif
35893  
35894  
35895 static forceinline u16 bswap16(u16 v)
35896 {
35897 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
35898 return __builtin_bswap16(v);
35899 #elif defined(_MSC_VER)
35900 return _byteswap_ushort(v);
35901 #else
35902 return (v << 8) | (v >> 8);
35903 #endif
35904 }
35905  
35906  
35907 static forceinline u32 bswap32(u32 v)
35908 {
35909 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
35910 return __builtin_bswap32(v);
35911 #elif defined(_MSC_VER)
35912 return _byteswap_ulong(v);
35913 #else
35914 return ((v & 0x000000FF) << 24) |
35915 ((v & 0x0000FF00) << 8) |
35916 ((v & 0x00FF0000) >> 8) |
35917 ((v & 0xFF000000) >> 24);
35918 #endif
35919 }
35920  
35921  
35922 static forceinline u64 bswap64(u64 v)
35923 {
35924 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
35925 return __builtin_bswap64(v);
35926 #elif defined(_MSC_VER)
35927 return _byteswap_uint64(v);
35928 #else
35929 return ((v & 0x00000000000000FF) << 56) |
35930 ((v & 0x000000000000FF00) << 40) |
35931 ((v & 0x0000000000FF0000) << 24) |
35932 ((v & 0x00000000FF000000) << 8) |
35933 ((v & 0x000000FF00000000) >> 8) |
35934 ((v & 0x0000FF0000000000) >> 24) |
35935 ((v & 0x00FF000000000000) >> 40) |
35936 ((v & 0xFF00000000000000) >> 56);
35937 #endif
35938 }
35939  
35940 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
35941 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
35942 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
35943 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
35944 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
35945 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
35946  
35947  
35948  
35949  
35950  
35951  
35952 #if (defined(__GNUC__) || defined(__clang__)) && \
35953 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
35954 defined(__ARM_FEATURE_UNALIGNED) || \
35955 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
35956 defined(__riscv_misaligned_fast) || \
35957 defined(__wasm__))
35958 # define UNALIGNED_ACCESS_IS_FAST 1
35959 #elif defined(_MSC_VER)
35960 # define UNALIGNED_ACCESS_IS_FAST 1
35961 #else
35962 # define UNALIGNED_ACCESS_IS_FAST 0
35963 #endif
35964  
35965  
35966  
35967 #ifdef FREESTANDING
35968 # define MEMCOPY __builtin_memcpy
35969 #else
35970 # define MEMCOPY memcpy
35971 #endif
35972  
35973  
35974  
35975 #define DEFINE_UNALIGNED_TYPE(type) \
35976 static forceinline type \
35977 load_##type##_unaligned(const void *p) \
35978 { \
35979 type v; \
35980 \
35981 MEMCOPY(&v, p, sizeof(v)); \
35982 return v; \
35983 } \
35984 \
35985 static forceinline void \
35986 store_##type##_unaligned(type v, void *p) \
35987 { \
35988 MEMCOPY(p, &v, sizeof(v)); \
35989 }
35990  
35991 DEFINE_UNALIGNED_TYPE(u16)
35992 DEFINE_UNALIGNED_TYPE(u32)
35993 DEFINE_UNALIGNED_TYPE(u64)
35994 DEFINE_UNALIGNED_TYPE(machine_word_t)
35995  
35996 #undef MEMCOPY
35997  
35998 #define load_word_unaligned load_machine_word_t_unaligned
35999 #define store_word_unaligned store_machine_word_t_unaligned
36000  
36001  
36002  
36003 static forceinline u16
36004 get_unaligned_le16(const u8 *p)
36005 {
36006 if (UNALIGNED_ACCESS_IS_FAST)
36007 return le16_bswap(load_u16_unaligned(p));
36008 else
36009 return ((u16)p[1] << 8) | p[0];
36010 }
36011  
36012 static forceinline u16
36013 get_unaligned_be16(const u8 *p)
36014 {
36015 if (UNALIGNED_ACCESS_IS_FAST)
36016 return be16_bswap(load_u16_unaligned(p));
36017 else
36018 return ((u16)p[0] << 8) | p[1];
36019 }
36020  
36021 static forceinline u32
36022 get_unaligned_le32(const u8 *p)
36023 {
36024 if (UNALIGNED_ACCESS_IS_FAST)
36025 return le32_bswap(load_u32_unaligned(p));
36026 else
36027 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
36028 ((u32)p[1] << 8) | p[0];
36029 }
36030  
36031 static forceinline u32
36032 get_unaligned_be32(const u8 *p)
36033 {
36034 if (UNALIGNED_ACCESS_IS_FAST)
36035 return be32_bswap(load_u32_unaligned(p));
36036 else
36037 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
36038 ((u32)p[2] << 8) | p[3];
36039 }
36040  
36041 static forceinline u64
36042 get_unaligned_le64(const u8 *p)
36043 {
36044 if (UNALIGNED_ACCESS_IS_FAST)
36045 return le64_bswap(load_u64_unaligned(p));
36046 else
36047 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
36048 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
36049 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
36050 ((u64)p[1] << 8) | p[0];
36051 }
36052  
36053 static forceinline machine_word_t
36054 get_unaligned_leword(const u8 *p)
36055 {
36056 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
36057 if (WORDBITS == 32)
36058 return get_unaligned_le32(p);
36059 else
36060 return get_unaligned_le64(p);
36061 }
36062  
36063  
36064  
36065 static forceinline void
36066 put_unaligned_le16(u16 v, u8 *p)
36067 {
36068 if (UNALIGNED_ACCESS_IS_FAST) {
36069 store_u16_unaligned(le16_bswap(v), p);
36070 } else {
36071 p[0] = (u8)(v >> 0);
36072 p[1] = (u8)(v >> 8);
36073 }
36074 }
36075  
36076 static forceinline void
36077 put_unaligned_be16(u16 v, u8 *p)
36078 {
36079 if (UNALIGNED_ACCESS_IS_FAST) {
36080 store_u16_unaligned(be16_bswap(v), p);
36081 } else {
36082 p[0] = (u8)(v >> 8);
36083 p[1] = (u8)(v >> 0);
36084 }
36085 }
36086  
36087 static forceinline void
36088 put_unaligned_le32(u32 v, u8 *p)
36089 {
36090 if (UNALIGNED_ACCESS_IS_FAST) {
36091 store_u32_unaligned(le32_bswap(v), p);
36092 } else {
36093 p[0] = (u8)(v >> 0);
36094 p[1] = (u8)(v >> 8);
36095 p[2] = (u8)(v >> 16);
36096 p[3] = (u8)(v >> 24);
36097 }
36098 }
36099  
36100 static forceinline void
36101 put_unaligned_be32(u32 v, u8 *p)
36102 {
36103 if (UNALIGNED_ACCESS_IS_FAST) {
36104 store_u32_unaligned(be32_bswap(v), p);
36105 } else {
36106 p[0] = (u8)(v >> 24);
36107 p[1] = (u8)(v >> 16);
36108 p[2] = (u8)(v >> 8);
36109 p[3] = (u8)(v >> 0);
36110 }
36111 }
36112  
36113 static forceinline void
36114 put_unaligned_le64(u64 v, u8 *p)
36115 {
36116 if (UNALIGNED_ACCESS_IS_FAST) {
36117 store_u64_unaligned(le64_bswap(v), p);
36118 } else {
36119 p[0] = (u8)(v >> 0);
36120 p[1] = (u8)(v >> 8);
36121 p[2] = (u8)(v >> 16);
36122 p[3] = (u8)(v >> 24);
36123 p[4] = (u8)(v >> 32);
36124 p[5] = (u8)(v >> 40);
36125 p[6] = (u8)(v >> 48);
36126 p[7] = (u8)(v >> 56);
36127 }
36128 }
36129  
36130 static forceinline void
36131 put_unaligned_leword(machine_word_t v, u8 *p)
36132 {
36133 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
36134 if (WORDBITS == 32)
36135 put_unaligned_le32(v, p);
36136 else
36137 put_unaligned_le64(v, p);
36138 }
36139  
36140  
36141  
36142  
36143  
36144  
36145  
36146 static forceinline unsigned
36147 bsr32(u32 v)
36148 {
36149 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
36150 return 31 - __builtin_clz(v);
36151 #elif defined(_MSC_VER)
36152 unsigned long i;
36153  
36154 _BitScanReverse(&i, v);
36155 return i;
36156 #else
36157 unsigned i = 0;
36158  
36159 while ((v >>= 1) != 0)
36160 i++;
36161 return i;
36162 #endif
36163 }
36164  
36165 static forceinline unsigned
36166 bsr64(u64 v)
36167 {
36168 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
36169 return 63 - __builtin_clzll(v);
36170 #elif defined(_MSC_VER) && defined(_WIN64)
36171 unsigned long i;
36172  
36173 _BitScanReverse64(&i, v);
36174 return i;
36175 #else
36176 unsigned i = 0;
36177  
36178 while ((v >>= 1) != 0)
36179 i++;
36180 return i;
36181 #endif
36182 }
36183  
36184 static forceinline unsigned
36185 bsrw(machine_word_t v)
36186 {
36187 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
36188 if (WORDBITS == 32)
36189 return bsr32(v);
36190 else
36191 return bsr64(v);
36192 }
36193  
36194  
36195  
36196 static forceinline unsigned
36197 bsf32(u32 v)
36198 {
36199 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
36200 return __builtin_ctz(v);
36201 #elif defined(_MSC_VER)
36202 unsigned long i;
36203  
36204 _BitScanForward(&i, v);
36205 return i;
36206 #else
36207 unsigned i = 0;
36208  
36209 for (; (v & 1) == 0; v >>= 1)
36210 i++;
36211 return i;
36212 #endif
36213 }
36214  
36215 static forceinline unsigned
36216 bsf64(u64 v)
36217 {
36218 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
36219 return __builtin_ctzll(v);
36220 #elif defined(_MSC_VER) && defined(_WIN64)
36221 unsigned long i;
36222  
36223 _BitScanForward64(&i, v);
36224 return i;
36225 #else
36226 unsigned i = 0;
36227  
36228 for (; (v & 1) == 0; v >>= 1)
36229 i++;
36230 return i;
36231 #endif
36232 }
36233  
36234 static forceinline unsigned
36235 bsfw(machine_word_t v)
36236 {
36237 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
36238 if (WORDBITS == 32)
36239 return bsf32(v);
36240 else
36241 return bsf64(v);
36242 }
36243  
36244  
36245 #undef rbit32
36246 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
36247 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
36248 static forceinline u32
36249 rbit32(u32 v)
36250 {
36251 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
36252 return v;
36253 }
36254 #define rbit32 rbit32
36255 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
36256 static forceinline u32
36257 rbit32(u32 v)
36258 {
36259 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
36260 return v;
36261 }
36262 #define rbit32 rbit32
36263 #endif
36264  
36265 #endif
36266  
36267  
36268 typedef void *(*malloc_func_t)(size_t);
36269 typedef void (*free_func_t)(void *);
36270  
36271 extern malloc_func_t libdeflate_default_malloc_func;
36272 extern free_func_t libdeflate_default_free_func;
36273  
36274 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
36275 size_t alignment, size_t size);
36276 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
36277  
36278 #ifdef FREESTANDING
36279  
36280 void *memset(void *s, int c, size_t n);
36281 #define memset(s, c, n) __builtin_memset((s), (c), (n))
36282  
36283 void *memcpy(void *dest, const void *src, size_t n);
36284 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
36285  
36286 void *memmove(void *dest, const void *src, size_t n);
36287 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
36288  
36289 int memcmp(const void *s1, const void *s2, size_t n);
36290 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
36291  
36292 #undef LIBDEFLATE_ENABLE_ASSERTIONS
36293 #else
36294 # include <string.h>
36295
36296 # ifdef __clang_analyzer__
36297 # define LIBDEFLATE_ENABLE_ASSERTIONS
36298 # endif
36299 #endif
36300  
36301  
36302 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
36303 NORETURN void
36304 libdeflate_assertion_failed(const char *expr, const char *file, int line);
36305 #define ASSERT(expr) { if (unlikely(!(expr))) \
36306 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
36307 #else
36308 #define ASSERT(expr) (void)(expr)
36309 #endif
36310  
36311 #define CONCAT_IMPL(a, b) a##b
36312 #define CONCAT(a, b) CONCAT_IMPL(a, b)
36313 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
36314  
36315 #endif
36316  
36317 /* #include "zlib_constants.h" */
36318  
36319  
36320 #ifndef LIB_ZLIB_CONSTANTS_H
36321 #define LIB_ZLIB_CONSTANTS_H
36322  
36323 #define ZLIB_MIN_HEADER_SIZE 2
36324 #define ZLIB_FOOTER_SIZE 4
36325 #define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)
36326  
36327 #define ZLIB_CM_DEFLATE 8
36328  
36329 #define ZLIB_CINFO_32K_WINDOW 7
36330  
36331 #define ZLIB_FASTEST_COMPRESSION 0
36332 #define ZLIB_FAST_COMPRESSION 1
36333 #define ZLIB_DEFAULT_COMPRESSION 2
36334 #define ZLIB_SLOWEST_COMPRESSION 3
36335  
36336 #endif
36337  
36338  
36339 LIBDEFLATEAPI enum libdeflate_result
36340 12 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
36341 const void *in, size_t in_nbytes,
36342 void *out, size_t out_nbytes_avail,
36343 size_t *actual_in_nbytes_ret,
36344 size_t *actual_out_nbytes_ret)
36345 {
36346 12 const u8 *in_next = in;
36347 12 const u8 * const in_end = in_next + in_nbytes;
36348 u16 hdr;
36349 size_t actual_in_nbytes;
36350 size_t actual_out_nbytes;
36351 enum libdeflate_result result;
36352  
36353 12 ▶ if (in_nbytes < ZLIB_MIN_OVERHEAD)
Branch: 1 branch
BranchTrueFalseTotal
if (in_nbytes < ZLIB_MIN_OVERHEAD) 0 12 12
36354 0 return LIBDEFLATE_BAD_DATA;
36355  
36356
36357 12 hdr = get_unaligned_be16(in_next);
36358 12 in_next += 2;
36359  
36360
36361 12 ▶ if ((hdr % 31) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if ((hdr % 31) != 0) 0 12 12
36362 0 return LIBDEFLATE_BAD_DATA;
36363  
36364
36365 12 ▶ if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
Branch: 1 branch
BranchTrueFalseTotal
if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE) 0 12 12
36366 0 return LIBDEFLATE_BAD_DATA;
36367  
36368
36369 12 ▶ if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
Branch: 1 branch
BranchTrueFalseTotal
if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW) 0 12 12
36370 0 return LIBDEFLATE_BAD_DATA;
36371  
36372
36373 12 ▶ if ((hdr >> 5) & 1)
Branch: 1 branch
BranchTrueFalseTotal
if ((hdr >> 5) & 1) 0 12 12
36374 0 return LIBDEFLATE_BAD_DATA;
36375  
36376
36377 12 result = libdeflate_deflate_decompress_ex(d, in_next,
36378 12 in_end - ZLIB_FOOTER_SIZE - in_next,
36379 out, out_nbytes_avail,
36380 &actual_in_nbytes, actual_out_nbytes_ret);
36381 12 ▶ if (result != LIBDEFLATE_SUCCESS)
Branch: 1 branch
BranchTrueFalseTotal
if (result != LIBDEFLATE_SUCCESS) 0 12 12
36382 0 return result;
36383  
36384 12 ▶ if (actual_out_nbytes_ret)
Branch: 1 branch
BranchTrueFalseTotal
if (actual_out_nbytes_ret) 12 0 12
36385 12 actual_out_nbytes = *actual_out_nbytes_ret;
36386 else
36387 0 actual_out_nbytes = out_nbytes_avail;
36388  
36389 12 in_next += actual_in_nbytes;
36390  
36391
36392 12 if (libdeflate_adler32(1, out, actual_out_nbytes) !=
36393 12 ▶ get_unaligned_be32(in_next))
Branch: 1 branch
BranchTrueFalseTotal
get_unaligned_be32(in_next)) 0 12 12
36394 0 return LIBDEFLATE_BAD_DATA;
36395 12 in_next += 4;
36396  
36397 12 ▶ if (actual_in_nbytes_ret)
Branch: 1 branch
BranchTrueFalseTotal
if (actual_in_nbytes_ret) 12 0 12
36398 12 *actual_in_nbytes_ret = in_next - (u8 *)in;
36399  
36400 12 return LIBDEFLATE_SUCCESS;
36401 }
36402  
36403 LIBDEFLATEAPI enum libdeflate_result
36404 0 libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
36405 const void *in, size_t in_nbytes,
36406 void *out, size_t out_nbytes_avail,
36407 size_t *actual_out_nbytes_ret)
36408 {
36409 0 return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
36410 out, out_nbytes_avail,
36411 NULL, actual_out_nbytes_ret);
36412 }
36413 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/arm/cpu_features.c */
36414  
36415  
36416  
36417  
36418 #ifdef __APPLE__
36419 # undef _ANSI_SOURCE
36420 # undef _DARWIN_C_SOURCE
36421 # define _DARWIN_C_SOURCE
36422 #endif
36423  
36424 /* #include "cpu_features_common.h" */
36425  
36426  
36427 #ifndef LIB_CPU_FEATURES_COMMON_H
36428 #define LIB_CPU_FEATURES_COMMON_H
36429  
36430 #if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
36431
36432 # undef _ANSI_SOURCE
36433 # ifndef __APPLE__
36434 # undef _GNU_SOURCE
36435 # define _GNU_SOURCE
36436 # endif
36437 # include <stdio.h>
36438 # include <stdlib.h>
36439 # include <string.h>
36440 #endif
36441  
36442 /* #include "lib_common.h" */
36443  
36444  
36445 #ifndef LIB_LIB_COMMON_H
36446 #define LIB_LIB_COMMON_H
36447  
36448 #ifdef LIBDEFLATE_H
36449
36450 # error "lib_common.h must always be included before libdeflate.h"
36451 #endif
36452  
36453 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
36454 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
36455 #elif defined(__GNUC__)
36456 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
36457 #else
36458 # define LIBDEFLATE_EXPORT_SYM
36459 #endif
36460  
36461  
36462 #if defined(__GNUC__) && defined(__i386__)
36463 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
36464 #else
36465 # define LIBDEFLATE_ALIGN_STACK
36466 #endif
36467  
36468 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
36469  
36470 /* #include "../common_defs.h" */
36471  
36472  
36473 #ifndef COMMON_DEFS_H
36474 #define COMMON_DEFS_H
36475  
36476 /* #include "libdeflate.h" */
36477  
36478  
36479 #ifndef LIBDEFLATE_H
36480 #define LIBDEFLATE_H
36481  
36482 #include <stddef.h>
36483 #include <stdint.h>
36484  
36485 #ifdef __cplusplus
36486 extern "C" {
36487 #endif
36488  
36489 #define LIBDEFLATE_VERSION_MAJOR 1
36490 #define LIBDEFLATE_VERSION_MINOR 26
36491 #define LIBDEFLATE_VERSION_STRING "1.26"
36492  
36493  
36494 #ifndef LIBDEFLATEAPI
36495 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
36496 # define LIBDEFLATEAPI __declspec(dllimport)
36497 # else
36498 # define LIBDEFLATEAPI
36499 # endif
36500 #endif
36501  
36502  
36503  
36504  
36505  
36506 struct libdeflate_compressor;
36507 struct libdeflate_options;
36508  
36509  
36510 LIBDEFLATEAPI struct libdeflate_compressor *
36511 libdeflate_alloc_compressor(int compression_level);
36512  
36513  
36514 LIBDEFLATEAPI struct libdeflate_compressor *
36515 libdeflate_alloc_compressor_ex(int compression_level,
36516 const struct libdeflate_options *options);
36517  
36518  
36519 LIBDEFLATEAPI size_t
36520 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
36521 const void *in, size_t in_nbytes,
36522 void *out, size_t out_nbytes_avail);
36523  
36524  
36525 LIBDEFLATEAPI size_t
36526 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
36527 size_t in_nbytes);
36528  
36529  
36530 LIBDEFLATEAPI size_t
36531 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
36532 const void *in, size_t in_nbytes,
36533 void *out, size_t out_nbytes_avail);
36534  
36535  
36536 LIBDEFLATEAPI size_t
36537 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
36538 size_t in_nbytes);
36539  
36540  
36541 LIBDEFLATEAPI size_t
36542 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
36543 const void *in, size_t in_nbytes,
36544 void *out, size_t out_nbytes_avail);
36545  
36546  
36547 LIBDEFLATEAPI size_t
36548 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
36549 size_t in_nbytes);
36550  
36551  
36552 LIBDEFLATEAPI void
36553 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
36554  
36555  
36556  
36557  
36558  
36559 struct libdeflate_decompressor;
36560 struct libdeflate_options;
36561  
36562  
36563 LIBDEFLATEAPI struct libdeflate_decompressor *
36564 libdeflate_alloc_decompressor(void);
36565  
36566  
36567 LIBDEFLATEAPI struct libdeflate_decompressor *
36568 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
36569  
36570  
36571 enum libdeflate_result {
36572
36573 LIBDEFLATE_SUCCESS = 0,
36574  
36575
36576 LIBDEFLATE_BAD_DATA = 1,
36577  
36578
36579 LIBDEFLATE_SHORT_OUTPUT = 2,
36580  
36581
36582 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
36583 };
36584  
36585  
36586 LIBDEFLATEAPI enum libdeflate_result
36587 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
36588 const void *in, size_t in_nbytes,
36589 void *out, size_t out_nbytes_avail,
36590 size_t *actual_out_nbytes_ret);
36591  
36592  
36593 LIBDEFLATEAPI enum libdeflate_result
36594 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
36595 const void *in, size_t in_nbytes,
36596 void *out, size_t out_nbytes_avail,
36597 size_t *actual_in_nbytes_ret,
36598 size_t *actual_out_nbytes_ret);
36599  
36600  
36601 LIBDEFLATEAPI enum libdeflate_result
36602 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
36603 const void *in, size_t in_nbytes,
36604 void *out, size_t out_nbytes_avail,
36605 size_t *actual_out_nbytes_ret);
36606  
36607  
36608 LIBDEFLATEAPI enum libdeflate_result
36609 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
36610 const void *in, size_t in_nbytes,
36611 void *out, size_t out_nbytes_avail,
36612 size_t *actual_in_nbytes_ret,
36613 size_t *actual_out_nbytes_ret);
36614  
36615  
36616 LIBDEFLATEAPI enum libdeflate_result
36617 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
36618 const void *in, size_t in_nbytes,
36619 void *out, size_t out_nbytes_avail,
36620 size_t *actual_out_nbytes_ret);
36621  
36622  
36623 LIBDEFLATEAPI enum libdeflate_result
36624 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
36625 const void *in, size_t in_nbytes,
36626 void *out, size_t out_nbytes_avail,
36627 size_t *actual_in_nbytes_ret,
36628 size_t *actual_out_nbytes_ret);
36629  
36630  
36631 LIBDEFLATEAPI void
36632 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
36633  
36634  
36635  
36636  
36637  
36638  
36639 LIBDEFLATEAPI uint32_t
36640 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
36641  
36642  
36643  
36644 LIBDEFLATEAPI uint32_t
36645 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
36646  
36647  
36648  
36649  
36650  
36651  
36652 LIBDEFLATEAPI void
36653 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
36654 void (*free_func)(void *));
36655  
36656  
36657 struct libdeflate_options {
36658  
36659
36660 size_t sizeof_options;
36661  
36662
36663 void *(*malloc_func)(size_t);
36664 void (*free_func)(void *);
36665 };
36666  
36667 #ifdef __cplusplus
36668 }
36669 #endif
36670  
36671 #endif
36672  
36673  
36674 #include <stdbool.h>
36675 #include <stddef.h>
36676 #include <stdint.h>
36677 #ifdef _MSC_VER
36678 # include <intrin.h>
36679 # include <stdlib.h>
36680
36681
36682 # pragma warning(disable : 4146)
36683
36684 # pragma warning(disable : 4018)
36685 # pragma warning(disable : 4244)
36686 # pragma warning(disable : 4267)
36687 # pragma warning(disable : 4310)
36688
36689 # pragma warning(disable : 4100)
36690 # pragma warning(disable : 4127)
36691 # pragma warning(disable : 4189)
36692 # pragma warning(disable : 4232)
36693 # pragma warning(disable : 4245)
36694 # pragma warning(disable : 4295)
36695 #endif
36696 #ifndef FREESTANDING
36697 # include <string.h>
36698 #endif
36699  
36700  
36701  
36702  
36703  
36704  
36705 #undef ARCH_X86_64
36706 #undef ARCH_X86_32
36707 #undef ARCH_ARM64
36708 #undef ARCH_ARM32
36709 #undef ARCH_RISCV
36710 #ifdef _MSC_VER
36711
36712 # if defined(_M_X64) && !defined(_M_ARM64EC)
36713 # define ARCH_X86_64
36714 # elif defined(_M_IX86)
36715 # define ARCH_X86_32
36716 # elif defined(_M_ARM64)
36717 # define ARCH_ARM64
36718 # elif defined(_M_ARM)
36719 # define ARCH_ARM32
36720 # endif
36721 #else
36722 # if defined(__x86_64__)
36723 # define ARCH_X86_64
36724 # elif defined(__i386__)
36725 # define ARCH_X86_32
36726 # elif defined(__aarch64__)
36727 # define ARCH_ARM64
36728 # elif defined(__arm__)
36729 # define ARCH_ARM32
36730 # elif defined(__riscv)
36731 # define ARCH_RISCV
36732 # endif
36733 #endif
36734  
36735  
36736  
36737  
36738  
36739  
36740 typedef uint8_t u8;
36741 typedef uint16_t u16;
36742 typedef uint32_t u32;
36743 typedef uint64_t u64;
36744 typedef int8_t s8;
36745 typedef int16_t s16;
36746 typedef int32_t s32;
36747 typedef int64_t s64;
36748  
36749  
36750 #ifdef _MSC_VER
36751 # ifdef _WIN64
36752 typedef long long ssize_t;
36753 # else
36754 typedef long ssize_t;
36755 # endif
36756 #endif
36757  
36758  
36759 typedef size_t machine_word_t;
36760  
36761  
36762 #define WORDBYTES ((int)sizeof(machine_word_t))
36763  
36764  
36765 #define WORDBITS (8 * WORDBYTES)
36766  
36767  
36768  
36769  
36770  
36771  
36772 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
36773 # define GCC_PREREQ(major, minor) \
36774 (__GNUC__ > (major) || \
36775 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
36776 # if !GCC_PREREQ(4, 9)
36777 # error "gcc versions older than 4.9 are no longer supported"
36778 # endif
36779 #else
36780 # define GCC_PREREQ(major, minor) 0
36781 #endif
36782 #ifdef __clang__
36783 # ifdef __apple_build_version__
36784 # define CLANG_PREREQ(major, minor, apple_version) \
36785 (__apple_build_version__ >= (apple_version))
36786 # else
36787 # define CLANG_PREREQ(major, minor, apple_version) \
36788 (__clang_major__ > (major) || \
36789 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
36790 # endif
36791 # if !CLANG_PREREQ(3, 9, 8000000)
36792 # error "clang versions older than 3.9 are no longer supported"
36793 # endif
36794 #else
36795 # define CLANG_PREREQ(major, minor, apple_version) 0
36796 #endif
36797 #ifdef _MSC_VER
36798 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
36799 # if !MSVC_PREREQ(1928)
36800 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
36801 # endif
36802 #else
36803 # define MSVC_PREREQ(version) 0
36804 #endif
36805  
36806  
36807 #ifndef __has_attribute
36808 # define __has_attribute(attribute) 0
36809 #endif
36810  
36811  
36812 #ifndef __has_builtin
36813 # define __has_builtin(builtin) 0
36814 #endif
36815  
36816  
36817 #ifdef _MSC_VER
36818 # define inline __inline
36819 #endif
36820  
36821  
36822 #if defined(__GNUC__) || __has_attribute(always_inline)
36823 # define forceinline inline __attribute__((always_inline))
36824 #elif defined(_MSC_VER)
36825 # define forceinline __forceinline
36826 #else
36827 # define forceinline inline
36828 #endif
36829  
36830  
36831 #if defined(__GNUC__) || __has_attribute(unused)
36832 # define MAYBE_UNUSED __attribute__((unused))
36833 #else
36834 # define MAYBE_UNUSED
36835 #endif
36836  
36837  
36838 #if defined(__GNUC__) || __has_attribute(noreturn)
36839 # define NORETURN __attribute__((noreturn))
36840 #else
36841 # define NORETURN
36842 #endif
36843  
36844  
36845 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
36846 # define likely(expr) __builtin_expect(!!(expr), 1)
36847 #else
36848 # define likely(expr) (expr)
36849 #endif
36850  
36851  
36852 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
36853 # define unlikely(expr) __builtin_expect(!!(expr), 0)
36854 #else
36855 # define unlikely(expr) (expr)
36856 #endif
36857  
36858  
36859 #undef prefetchr
36860 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
36861 # define prefetchr(addr) __builtin_prefetch((addr), 0)
36862 #elif defined(_MSC_VER)
36863 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
36864 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
36865 # elif defined(ARCH_ARM64)
36866 # define prefetchr(addr) __prefetch2((addr), 0x00 )
36867 # elif defined(ARCH_ARM32)
36868 # define prefetchr(addr) __prefetch(addr)
36869 # endif
36870 #endif
36871 #ifndef prefetchr
36872 # define prefetchr(addr)
36873 #endif
36874  
36875  
36876 #undef prefetchw
36877 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
36878 # define prefetchw(addr) __builtin_prefetch((addr), 1)
36879 #elif defined(_MSC_VER)
36880 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
36881 # define prefetchw(addr) _m_prefetchw(addr)
36882 # elif defined(ARCH_ARM64)
36883 # define prefetchw(addr) __prefetch2((addr), 0x10 )
36884 # elif defined(ARCH_ARM32)
36885 # define prefetchw(addr) __prefetchw(addr)
36886 # endif
36887 #endif
36888 #ifndef prefetchw
36889 # define prefetchw(addr)
36890 #endif
36891  
36892  
36893 #undef _aligned_attribute
36894 #if defined(__GNUC__) || __has_attribute(aligned)
36895 # define _aligned_attribute(n) __attribute__((aligned(n)))
36896 #elif defined(_MSC_VER)
36897 # define _aligned_attribute(n) __declspec(align(n))
36898 #endif
36899  
36900  
36901 #if defined(__GNUC__) || __has_attribute(target)
36902 # define _target_attribute(attrs) __attribute__((target(attrs)))
36903 #else
36904 # define _target_attribute(attrs)
36905 #endif
36906  
36907  
36908  
36909  
36910  
36911 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
36912 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
36913 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
36914 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
36915 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
36916 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
36917 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
36918  
36919  
36920  
36921  
36922  
36923  
36924 #if defined(__BYTE_ORDER__)
36925 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
36926 #elif defined(_MSC_VER)
36927 # define CPU_IS_LITTLE_ENDIAN() true
36928 #else
36929 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
36930 {
36931 union {
36932 u32 w;
36933 u8 b;
36934 } u;
36935  
36936 u.w = 1;
36937 return u.b;
36938 }
36939 #endif
36940  
36941  
36942 static forceinline u16 bswap16(u16 v)
36943 {
36944 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
36945 return __builtin_bswap16(v);
36946 #elif defined(_MSC_VER)
36947 return _byteswap_ushort(v);
36948 #else
36949 return (v << 8) | (v >> 8);
36950 #endif
36951 }
36952  
36953  
36954 static forceinline u32 bswap32(u32 v)
36955 {
36956 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
36957 return __builtin_bswap32(v);
36958 #elif defined(_MSC_VER)
36959 return _byteswap_ulong(v);
36960 #else
36961 return ((v & 0x000000FF) << 24) |
36962 ((v & 0x0000FF00) << 8) |
36963 ((v & 0x00FF0000) >> 8) |
36964 ((v & 0xFF000000) >> 24);
36965 #endif
36966 }
36967  
36968  
36969 static forceinline u64 bswap64(u64 v)
36970 {
36971 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
36972 return __builtin_bswap64(v);
36973 #elif defined(_MSC_VER)
36974 return _byteswap_uint64(v);
36975 #else
36976 return ((v & 0x00000000000000FF) << 56) |
36977 ((v & 0x000000000000FF00) << 40) |
36978 ((v & 0x0000000000FF0000) << 24) |
36979 ((v & 0x00000000FF000000) << 8) |
36980 ((v & 0x000000FF00000000) >> 8) |
36981 ((v & 0x0000FF0000000000) >> 24) |
36982 ((v & 0x00FF000000000000) >> 40) |
36983 ((v & 0xFF00000000000000) >> 56);
36984 #endif
36985 }
36986  
36987 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
36988 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
36989 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
36990 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
36991 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
36992 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
36993  
36994  
36995  
36996  
36997  
36998  
36999 #if (defined(__GNUC__) || defined(__clang__)) && \
37000 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
37001 defined(__ARM_FEATURE_UNALIGNED) || \
37002 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
37003 defined(__riscv_misaligned_fast) || \
37004 defined(__wasm__))
37005 # define UNALIGNED_ACCESS_IS_FAST 1
37006 #elif defined(_MSC_VER)
37007 # define UNALIGNED_ACCESS_IS_FAST 1
37008 #else
37009 # define UNALIGNED_ACCESS_IS_FAST 0
37010 #endif
37011  
37012  
37013  
37014 #ifdef FREESTANDING
37015 # define MEMCOPY __builtin_memcpy
37016 #else
37017 # define MEMCOPY memcpy
37018 #endif
37019  
37020  
37021  
37022 #define DEFINE_UNALIGNED_TYPE(type) \
37023 static forceinline type \
37024 load_##type##_unaligned(const void *p) \
37025 { \
37026 type v; \
37027 \
37028 MEMCOPY(&v, p, sizeof(v)); \
37029 return v; \
37030 } \
37031 \
37032 static forceinline void \
37033 store_##type##_unaligned(type v, void *p) \
37034 { \
37035 MEMCOPY(p, &v, sizeof(v)); \
37036 }
37037  
37038 DEFINE_UNALIGNED_TYPE(u16)
37039 DEFINE_UNALIGNED_TYPE(u32)
37040 DEFINE_UNALIGNED_TYPE(u64)
37041 DEFINE_UNALIGNED_TYPE(machine_word_t)
37042  
37043 #undef MEMCOPY
37044  
37045 #define load_word_unaligned load_machine_word_t_unaligned
37046 #define store_word_unaligned store_machine_word_t_unaligned
37047  
37048  
37049  
37050 static forceinline u16
37051 get_unaligned_le16(const u8 *p)
37052 {
37053 if (UNALIGNED_ACCESS_IS_FAST)
37054 return le16_bswap(load_u16_unaligned(p));
37055 else
37056 return ((u16)p[1] << 8) | p[0];
37057 }
37058  
37059 static forceinline u16
37060 get_unaligned_be16(const u8 *p)
37061 {
37062 if (UNALIGNED_ACCESS_IS_FAST)
37063 return be16_bswap(load_u16_unaligned(p));
37064 else
37065 return ((u16)p[0] << 8) | p[1];
37066 }
37067  
37068 static forceinline u32
37069 get_unaligned_le32(const u8 *p)
37070 {
37071 if (UNALIGNED_ACCESS_IS_FAST)
37072 return le32_bswap(load_u32_unaligned(p));
37073 else
37074 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
37075 ((u32)p[1] << 8) | p[0];
37076 }
37077  
37078 static forceinline u32
37079 get_unaligned_be32(const u8 *p)
37080 {
37081 if (UNALIGNED_ACCESS_IS_FAST)
37082 return be32_bswap(load_u32_unaligned(p));
37083 else
37084 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
37085 ((u32)p[2] << 8) | p[3];
37086 }
37087  
37088 static forceinline u64
37089 get_unaligned_le64(const u8 *p)
37090 {
37091 if (UNALIGNED_ACCESS_IS_FAST)
37092 return le64_bswap(load_u64_unaligned(p));
37093 else
37094 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
37095 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
37096 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
37097 ((u64)p[1] << 8) | p[0];
37098 }
37099  
37100 static forceinline machine_word_t
37101 get_unaligned_leword(const u8 *p)
37102 {
37103 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
37104 if (WORDBITS == 32)
37105 return get_unaligned_le32(p);
37106 else
37107 return get_unaligned_le64(p);
37108 }
37109  
37110  
37111  
37112 static forceinline void
37113 put_unaligned_le16(u16 v, u8 *p)
37114 {
37115 if (UNALIGNED_ACCESS_IS_FAST) {
37116 store_u16_unaligned(le16_bswap(v), p);
37117 } else {
37118 p[0] = (u8)(v >> 0);
37119 p[1] = (u8)(v >> 8);
37120 }
37121 }
37122  
37123 static forceinline void
37124 put_unaligned_be16(u16 v, u8 *p)
37125 {
37126 if (UNALIGNED_ACCESS_IS_FAST) {
37127 store_u16_unaligned(be16_bswap(v), p);
37128 } else {
37129 p[0] = (u8)(v >> 8);
37130 p[1] = (u8)(v >> 0);
37131 }
37132 }
37133  
37134 static forceinline void
37135 put_unaligned_le32(u32 v, u8 *p)
37136 {
37137 if (UNALIGNED_ACCESS_IS_FAST) {
37138 store_u32_unaligned(le32_bswap(v), p);
37139 } else {
37140 p[0] = (u8)(v >> 0);
37141 p[1] = (u8)(v >> 8);
37142 p[2] = (u8)(v >> 16);
37143 p[3] = (u8)(v >> 24);
37144 }
37145 }
37146  
37147 static forceinline void
37148 put_unaligned_be32(u32 v, u8 *p)
37149 {
37150 if (UNALIGNED_ACCESS_IS_FAST) {
37151 store_u32_unaligned(be32_bswap(v), p);
37152 } else {
37153 p[0] = (u8)(v >> 24);
37154 p[1] = (u8)(v >> 16);
37155 p[2] = (u8)(v >> 8);
37156 p[3] = (u8)(v >> 0);
37157 }
37158 }
37159  
37160 static forceinline void
37161 put_unaligned_le64(u64 v, u8 *p)
37162 {
37163 if (UNALIGNED_ACCESS_IS_FAST) {
37164 store_u64_unaligned(le64_bswap(v), p);
37165 } else {
37166 p[0] = (u8)(v >> 0);
37167 p[1] = (u8)(v >> 8);
37168 p[2] = (u8)(v >> 16);
37169 p[3] = (u8)(v >> 24);
37170 p[4] = (u8)(v >> 32);
37171 p[5] = (u8)(v >> 40);
37172 p[6] = (u8)(v >> 48);
37173 p[7] = (u8)(v >> 56);
37174 }
37175 }
37176  
37177 static forceinline void
37178 put_unaligned_leword(machine_word_t v, u8 *p)
37179 {
37180 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
37181 if (WORDBITS == 32)
37182 put_unaligned_le32(v, p);
37183 else
37184 put_unaligned_le64(v, p);
37185 }
37186  
37187  
37188  
37189  
37190  
37191  
37192  
37193 static forceinline unsigned
37194 bsr32(u32 v)
37195 {
37196 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
37197 return 31 - __builtin_clz(v);
37198 #elif defined(_MSC_VER)
37199 unsigned long i;
37200  
37201 _BitScanReverse(&i, v);
37202 return i;
37203 #else
37204 unsigned i = 0;
37205  
37206 while ((v >>= 1) != 0)
37207 i++;
37208 return i;
37209 #endif
37210 }
37211  
37212 static forceinline unsigned
37213 bsr64(u64 v)
37214 {
37215 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
37216 return 63 - __builtin_clzll(v);
37217 #elif defined(_MSC_VER) && defined(_WIN64)
37218 unsigned long i;
37219  
37220 _BitScanReverse64(&i, v);
37221 return i;
37222 #else
37223 unsigned i = 0;
37224  
37225 while ((v >>= 1) != 0)
37226 i++;
37227 return i;
37228 #endif
37229 }
37230  
37231 static forceinline unsigned
37232 bsrw(machine_word_t v)
37233 {
37234 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
37235 if (WORDBITS == 32)
37236 return bsr32(v);
37237 else
37238 return bsr64(v);
37239 }
37240  
37241  
37242  
37243 static forceinline unsigned
37244 bsf32(u32 v)
37245 {
37246 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
37247 return __builtin_ctz(v);
37248 #elif defined(_MSC_VER)
37249 unsigned long i;
37250  
37251 _BitScanForward(&i, v);
37252 return i;
37253 #else
37254 unsigned i = 0;
37255  
37256 for (; (v & 1) == 0; v >>= 1)
37257 i++;
37258 return i;
37259 #endif
37260 }
37261  
37262 static forceinline unsigned
37263 bsf64(u64 v)
37264 {
37265 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
37266 return __builtin_ctzll(v);
37267 #elif defined(_MSC_VER) && defined(_WIN64)
37268 unsigned long i;
37269  
37270 _BitScanForward64(&i, v);
37271 return i;
37272 #else
37273 unsigned i = 0;
37274  
37275 for (; (v & 1) == 0; v >>= 1)
37276 i++;
37277 return i;
37278 #endif
37279 }
37280  
37281 static forceinline unsigned
37282 bsfw(machine_word_t v)
37283 {
37284 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
37285 if (WORDBITS == 32)
37286 return bsf32(v);
37287 else
37288 return bsf64(v);
37289 }
37290  
37291  
37292 #undef rbit32
37293 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
37294 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
37295 static forceinline u32
37296 rbit32(u32 v)
37297 {
37298 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
37299 return v;
37300 }
37301 #define rbit32 rbit32
37302 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
37303 static forceinline u32
37304 rbit32(u32 v)
37305 {
37306 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
37307 return v;
37308 }
37309 #define rbit32 rbit32
37310 #endif
37311  
37312 #endif
37313  
37314  
37315 typedef void *(*malloc_func_t)(size_t);
37316 typedef void (*free_func_t)(void *);
37317  
37318 extern malloc_func_t libdeflate_default_malloc_func;
37319 extern free_func_t libdeflate_default_free_func;
37320  
37321 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
37322 size_t alignment, size_t size);
37323 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
37324  
37325 #ifdef FREESTANDING
37326  
37327 void *memset(void *s, int c, size_t n);
37328 #define memset(s, c, n) __builtin_memset((s), (c), (n))
37329  
37330 void *memcpy(void *dest, const void *src, size_t n);
37331 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
37332  
37333 void *memmove(void *dest, const void *src, size_t n);
37334 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
37335  
37336 int memcmp(const void *s1, const void *s2, size_t n);
37337 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
37338  
37339 #undef LIBDEFLATE_ENABLE_ASSERTIONS
37340 #else
37341 # include <string.h>
37342
37343 # ifdef __clang_analyzer__
37344 # define LIBDEFLATE_ENABLE_ASSERTIONS
37345 # endif
37346 #endif
37347  
37348  
37349 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
37350 NORETURN void
37351 libdeflate_assertion_failed(const char *expr, const char *file, int line);
37352 #define ASSERT(expr) { if (unlikely(!(expr))) \
37353 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
37354 #else
37355 #define ASSERT(expr) (void)(expr)
37356 #endif
37357  
37358 #define CONCAT_IMPL(a, b) a##b
37359 #define CONCAT(a, b) CONCAT_IMPL(a, b)
37360 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
37361  
37362 #endif
37363  
37364  
37365 struct cpu_feature {
37366 u32 bit;
37367 const char *name;
37368 };
37369  
37370 #if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
37371  
37372 static inline void
37373 disable_cpu_features_for_testing(u32 *features,
37374 const struct cpu_feature *feature_table,
37375 size_t feature_table_length)
37376 {
37377 char *env_value, *strbuf, *p, *saveptr = NULL;
37378 size_t i;
37379  
37380 env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
37381 if (!env_value)
37382 return;
37383 strbuf = strdup(env_value);
37384 if (!strbuf)
37385 abort();
37386 p = strtok_r(strbuf, ",", &saveptr);
37387 while (p) {
37388 for (i = 0; i < feature_table_length; i++) {
37389 if (strcmp(p, feature_table[i].name) == 0) {
37390 *features &= ~feature_table[i].bit;
37391 break;
37392 }
37393 }
37394 if (i == feature_table_length) {
37395 fprintf(stderr,
37396 "unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
37397 p);
37398 abort();
37399 }
37400 p = strtok_r(NULL, ",", &saveptr);
37401 }
37402 free(strbuf);
37403 }
37404 #else
37405 static inline void
37406 2 disable_cpu_features_for_testing(u32 *features,
37407 const struct cpu_feature *feature_table,
37408 size_t feature_table_length)
37409 {
37410 2 }
37411 #endif
37412  
37413 #endif
37414
37415 /* #include "arm-cpu_features.h" */
37416  
37417  
37418 #ifndef LIB_ARM_CPU_FEATURES_H
37419 #define LIB_ARM_CPU_FEATURES_H
37420  
37421 /* #include "lib_common.h" */
37422  
37423  
37424 #ifndef LIB_LIB_COMMON_H
37425 #define LIB_LIB_COMMON_H
37426  
37427 #ifdef LIBDEFLATE_H
37428
37429 # error "lib_common.h must always be included before libdeflate.h"
37430 #endif
37431  
37432 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
37433 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
37434 #elif defined(__GNUC__)
37435 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
37436 #else
37437 # define LIBDEFLATE_EXPORT_SYM
37438 #endif
37439  
37440  
37441 #if defined(__GNUC__) && defined(__i386__)
37442 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
37443 #else
37444 # define LIBDEFLATE_ALIGN_STACK
37445 #endif
37446  
37447 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
37448  
37449 /* #include "../common_defs.h" */
37450  
37451  
37452 #ifndef COMMON_DEFS_H
37453 #define COMMON_DEFS_H
37454  
37455 /* #include "libdeflate.h" */
37456  
37457  
37458 #ifndef LIBDEFLATE_H
37459 #define LIBDEFLATE_H
37460  
37461 #include <stddef.h>
37462 #include <stdint.h>
37463  
37464 #ifdef __cplusplus
37465 extern "C" {
37466 #endif
37467  
37468 #define LIBDEFLATE_VERSION_MAJOR 1
37469 #define LIBDEFLATE_VERSION_MINOR 26
37470 #define LIBDEFLATE_VERSION_STRING "1.26"
37471  
37472  
37473 #ifndef LIBDEFLATEAPI
37474 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
37475 # define LIBDEFLATEAPI __declspec(dllimport)
37476 # else
37477 # define LIBDEFLATEAPI
37478 # endif
37479 #endif
37480  
37481  
37482  
37483  
37484  
37485 struct libdeflate_compressor;
37486 struct libdeflate_options;
37487  
37488  
37489 LIBDEFLATEAPI struct libdeflate_compressor *
37490 libdeflate_alloc_compressor(int compression_level);
37491  
37492  
37493 LIBDEFLATEAPI struct libdeflate_compressor *
37494 libdeflate_alloc_compressor_ex(int compression_level,
37495 const struct libdeflate_options *options);
37496  
37497  
37498 LIBDEFLATEAPI size_t
37499 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
37500 const void *in, size_t in_nbytes,
37501 void *out, size_t out_nbytes_avail);
37502  
37503  
37504 LIBDEFLATEAPI size_t
37505 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
37506 size_t in_nbytes);
37507  
37508  
37509 LIBDEFLATEAPI size_t
37510 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
37511 const void *in, size_t in_nbytes,
37512 void *out, size_t out_nbytes_avail);
37513  
37514  
37515 LIBDEFLATEAPI size_t
37516 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
37517 size_t in_nbytes);
37518  
37519  
37520 LIBDEFLATEAPI size_t
37521 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
37522 const void *in, size_t in_nbytes,
37523 void *out, size_t out_nbytes_avail);
37524  
37525  
37526 LIBDEFLATEAPI size_t
37527 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
37528 size_t in_nbytes);
37529  
37530  
37531 LIBDEFLATEAPI void
37532 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
37533  
37534  
37535  
37536  
37537  
37538 struct libdeflate_decompressor;
37539 struct libdeflate_options;
37540  
37541  
37542 LIBDEFLATEAPI struct libdeflate_decompressor *
37543 libdeflate_alloc_decompressor(void);
37544  
37545  
37546 LIBDEFLATEAPI struct libdeflate_decompressor *
37547 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
37548  
37549  
37550 enum libdeflate_result {
37551
37552 LIBDEFLATE_SUCCESS = 0,
37553  
37554
37555 LIBDEFLATE_BAD_DATA = 1,
37556  
37557
37558 LIBDEFLATE_SHORT_OUTPUT = 2,
37559  
37560
37561 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
37562 };
37563  
37564  
37565 LIBDEFLATEAPI enum libdeflate_result
37566 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
37567 const void *in, size_t in_nbytes,
37568 void *out, size_t out_nbytes_avail,
37569 size_t *actual_out_nbytes_ret);
37570  
37571  
37572 LIBDEFLATEAPI enum libdeflate_result
37573 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
37574 const void *in, size_t in_nbytes,
37575 void *out, size_t out_nbytes_avail,
37576 size_t *actual_in_nbytes_ret,
37577 size_t *actual_out_nbytes_ret);
37578  
37579  
37580 LIBDEFLATEAPI enum libdeflate_result
37581 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
37582 const void *in, size_t in_nbytes,
37583 void *out, size_t out_nbytes_avail,
37584 size_t *actual_out_nbytes_ret);
37585  
37586  
37587 LIBDEFLATEAPI enum libdeflate_result
37588 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
37589 const void *in, size_t in_nbytes,
37590 void *out, size_t out_nbytes_avail,
37591 size_t *actual_in_nbytes_ret,
37592 size_t *actual_out_nbytes_ret);
37593  
37594  
37595 LIBDEFLATEAPI enum libdeflate_result
37596 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
37597 const void *in, size_t in_nbytes,
37598 void *out, size_t out_nbytes_avail,
37599 size_t *actual_out_nbytes_ret);
37600  
37601  
37602 LIBDEFLATEAPI enum libdeflate_result
37603 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
37604 const void *in, size_t in_nbytes,
37605 void *out, size_t out_nbytes_avail,
37606 size_t *actual_in_nbytes_ret,
37607 size_t *actual_out_nbytes_ret);
37608  
37609  
37610 LIBDEFLATEAPI void
37611 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
37612  
37613  
37614  
37615  
37616  
37617  
37618 LIBDEFLATEAPI uint32_t
37619 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
37620  
37621  
37622  
37623 LIBDEFLATEAPI uint32_t
37624 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
37625  
37626  
37627  
37628  
37629  
37630  
37631 LIBDEFLATEAPI void
37632 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
37633 void (*free_func)(void *));
37634  
37635  
37636 struct libdeflate_options {
37637  
37638
37639 size_t sizeof_options;
37640  
37641
37642 void *(*malloc_func)(size_t);
37643 void (*free_func)(void *);
37644 };
37645  
37646 #ifdef __cplusplus
37647 }
37648 #endif
37649  
37650 #endif
37651  
37652  
37653 #include <stdbool.h>
37654 #include <stddef.h>
37655 #include <stdint.h>
37656 #ifdef _MSC_VER
37657 # include <intrin.h>
37658 # include <stdlib.h>
37659
37660
37661 # pragma warning(disable : 4146)
37662
37663 # pragma warning(disable : 4018)
37664 # pragma warning(disable : 4244)
37665 # pragma warning(disable : 4267)
37666 # pragma warning(disable : 4310)
37667
37668 # pragma warning(disable : 4100)
37669 # pragma warning(disable : 4127)
37670 # pragma warning(disable : 4189)
37671 # pragma warning(disable : 4232)
37672 # pragma warning(disable : 4245)
37673 # pragma warning(disable : 4295)
37674 #endif
37675 #ifndef FREESTANDING
37676 # include <string.h>
37677 #endif
37678  
37679  
37680  
37681  
37682  
37683  
37684 #undef ARCH_X86_64
37685 #undef ARCH_X86_32
37686 #undef ARCH_ARM64
37687 #undef ARCH_ARM32
37688 #undef ARCH_RISCV
37689 #ifdef _MSC_VER
37690
37691 # if defined(_M_X64) && !defined(_M_ARM64EC)
37692 # define ARCH_X86_64
37693 # elif defined(_M_IX86)
37694 # define ARCH_X86_32
37695 # elif defined(_M_ARM64)
37696 # define ARCH_ARM64
37697 # elif defined(_M_ARM)
37698 # define ARCH_ARM32
37699 # endif
37700 #else
37701 # if defined(__x86_64__)
37702 # define ARCH_X86_64
37703 # elif defined(__i386__)
37704 # define ARCH_X86_32
37705 # elif defined(__aarch64__)
37706 # define ARCH_ARM64
37707 # elif defined(__arm__)
37708 # define ARCH_ARM32
37709 # elif defined(__riscv)
37710 # define ARCH_RISCV
37711 # endif
37712 #endif
37713  
37714  
37715  
37716  
37717  
37718  
37719 typedef uint8_t u8;
37720 typedef uint16_t u16;
37721 typedef uint32_t u32;
37722 typedef uint64_t u64;
37723 typedef int8_t s8;
37724 typedef int16_t s16;
37725 typedef int32_t s32;
37726 typedef int64_t s64;
37727  
37728  
37729 #ifdef _MSC_VER
37730 # ifdef _WIN64
37731 typedef long long ssize_t;
37732 # else
37733 typedef long ssize_t;
37734 # endif
37735 #endif
37736  
37737  
37738 typedef size_t machine_word_t;
37739  
37740  
37741 #define WORDBYTES ((int)sizeof(machine_word_t))
37742  
37743  
37744 #define WORDBITS (8 * WORDBYTES)
37745  
37746  
37747  
37748  
37749  
37750  
37751 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
37752 # define GCC_PREREQ(major, minor) \
37753 (__GNUC__ > (major) || \
37754 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
37755 # if !GCC_PREREQ(4, 9)
37756 # error "gcc versions older than 4.9 are no longer supported"
37757 # endif
37758 #else
37759 # define GCC_PREREQ(major, minor) 0
37760 #endif
37761 #ifdef __clang__
37762 # ifdef __apple_build_version__
37763 # define CLANG_PREREQ(major, minor, apple_version) \
37764 (__apple_build_version__ >= (apple_version))
37765 # else
37766 # define CLANG_PREREQ(major, minor, apple_version) \
37767 (__clang_major__ > (major) || \
37768 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
37769 # endif
37770 # if !CLANG_PREREQ(3, 9, 8000000)
37771 # error "clang versions older than 3.9 are no longer supported"
37772 # endif
37773 #else
37774 # define CLANG_PREREQ(major, minor, apple_version) 0
37775 #endif
37776 #ifdef _MSC_VER
37777 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
37778 # if !MSVC_PREREQ(1928)
37779 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
37780 # endif
37781 #else
37782 # define MSVC_PREREQ(version) 0
37783 #endif
37784  
37785  
37786 #ifndef __has_attribute
37787 # define __has_attribute(attribute) 0
37788 #endif
37789  
37790  
37791 #ifndef __has_builtin
37792 # define __has_builtin(builtin) 0
37793 #endif
37794  
37795  
37796 #ifdef _MSC_VER
37797 # define inline __inline
37798 #endif
37799  
37800  
37801 #if defined(__GNUC__) || __has_attribute(always_inline)
37802 # define forceinline inline __attribute__((always_inline))
37803 #elif defined(_MSC_VER)
37804 # define forceinline __forceinline
37805 #else
37806 # define forceinline inline
37807 #endif
37808  
37809  
37810 #if defined(__GNUC__) || __has_attribute(unused)
37811 # define MAYBE_UNUSED __attribute__((unused))
37812 #else
37813 # define MAYBE_UNUSED
37814 #endif
37815  
37816  
37817 #if defined(__GNUC__) || __has_attribute(noreturn)
37818 # define NORETURN __attribute__((noreturn))
37819 #else
37820 # define NORETURN
37821 #endif
37822  
37823  
37824 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
37825 # define likely(expr) __builtin_expect(!!(expr), 1)
37826 #else
37827 # define likely(expr) (expr)
37828 #endif
37829  
37830  
37831 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
37832 # define unlikely(expr) __builtin_expect(!!(expr), 0)
37833 #else
37834 # define unlikely(expr) (expr)
37835 #endif
37836  
37837  
37838 #undef prefetchr
37839 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
37840 # define prefetchr(addr) __builtin_prefetch((addr), 0)
37841 #elif defined(_MSC_VER)
37842 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
37843 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
37844 # elif defined(ARCH_ARM64)
37845 # define prefetchr(addr) __prefetch2((addr), 0x00 )
37846 # elif defined(ARCH_ARM32)
37847 # define prefetchr(addr) __prefetch(addr)
37848 # endif
37849 #endif
37850 #ifndef prefetchr
37851 # define prefetchr(addr)
37852 #endif
37853  
37854  
37855 #undef prefetchw
37856 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
37857 # define prefetchw(addr) __builtin_prefetch((addr), 1)
37858 #elif defined(_MSC_VER)
37859 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
37860 # define prefetchw(addr) _m_prefetchw(addr)
37861 # elif defined(ARCH_ARM64)
37862 # define prefetchw(addr) __prefetch2((addr), 0x10 )
37863 # elif defined(ARCH_ARM32)
37864 # define prefetchw(addr) __prefetchw(addr)
37865 # endif
37866 #endif
37867 #ifndef prefetchw
37868 # define prefetchw(addr)
37869 #endif
37870  
37871  
37872 #undef _aligned_attribute
37873 #if defined(__GNUC__) || __has_attribute(aligned)
37874 # define _aligned_attribute(n) __attribute__((aligned(n)))
37875 #elif defined(_MSC_VER)
37876 # define _aligned_attribute(n) __declspec(align(n))
37877 #endif
37878  
37879  
37880 #if defined(__GNUC__) || __has_attribute(target)
37881 # define _target_attribute(attrs) __attribute__((target(attrs)))
37882 #else
37883 # define _target_attribute(attrs)
37884 #endif
37885  
37886  
37887  
37888  
37889  
37890 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
37891 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
37892 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
37893 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
37894 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
37895 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
37896 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
37897  
37898  
37899  
37900  
37901  
37902  
37903 #if defined(__BYTE_ORDER__)
37904 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
37905 #elif defined(_MSC_VER)
37906 # define CPU_IS_LITTLE_ENDIAN() true
37907 #else
37908 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
37909 {
37910 union {
37911 u32 w;
37912 u8 b;
37913 } u;
37914  
37915 u.w = 1;
37916 return u.b;
37917 }
37918 #endif
37919  
37920  
37921 static forceinline u16 bswap16(u16 v)
37922 {
37923 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
37924 return __builtin_bswap16(v);
37925 #elif defined(_MSC_VER)
37926 return _byteswap_ushort(v);
37927 #else
37928 return (v << 8) | (v >> 8);
37929 #endif
37930 }
37931  
37932  
37933 static forceinline u32 bswap32(u32 v)
37934 {
37935 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
37936 return __builtin_bswap32(v);
37937 #elif defined(_MSC_VER)
37938 return _byteswap_ulong(v);
37939 #else
37940 return ((v & 0x000000FF) << 24) |
37941 ((v & 0x0000FF00) << 8) |
37942 ((v & 0x00FF0000) >> 8) |
37943 ((v & 0xFF000000) >> 24);
37944 #endif
37945 }
37946  
37947  
37948 static forceinline u64 bswap64(u64 v)
37949 {
37950 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
37951 return __builtin_bswap64(v);
37952 #elif defined(_MSC_VER)
37953 return _byteswap_uint64(v);
37954 #else
37955 return ((v & 0x00000000000000FF) << 56) |
37956 ((v & 0x000000000000FF00) << 40) |
37957 ((v & 0x0000000000FF0000) << 24) |
37958 ((v & 0x00000000FF000000) << 8) |
37959 ((v & 0x000000FF00000000) >> 8) |
37960 ((v & 0x0000FF0000000000) >> 24) |
37961 ((v & 0x00FF000000000000) >> 40) |
37962 ((v & 0xFF00000000000000) >> 56);
37963 #endif
37964 }
37965  
37966 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
37967 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
37968 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
37969 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
37970 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
37971 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
37972  
37973  
37974  
37975  
37976  
37977  
37978 #if (defined(__GNUC__) || defined(__clang__)) && \
37979 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
37980 defined(__ARM_FEATURE_UNALIGNED) || \
37981 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
37982 defined(__riscv_misaligned_fast) || \
37983 defined(__wasm__))
37984 # define UNALIGNED_ACCESS_IS_FAST 1
37985 #elif defined(_MSC_VER)
37986 # define UNALIGNED_ACCESS_IS_FAST 1
37987 #else
37988 # define UNALIGNED_ACCESS_IS_FAST 0
37989 #endif
37990  
37991  
37992  
37993 #ifdef FREESTANDING
37994 # define MEMCOPY __builtin_memcpy
37995 #else
37996 # define MEMCOPY memcpy
37997 #endif
37998  
37999  
38000  
38001 #define DEFINE_UNALIGNED_TYPE(type) \
38002 static forceinline type \
38003 load_##type##_unaligned(const void *p) \
38004 { \
38005 type v; \
38006 \
38007 MEMCOPY(&v, p, sizeof(v)); \
38008 return v; \
38009 } \
38010 \
38011 static forceinline void \
38012 store_##type##_unaligned(type v, void *p) \
38013 { \
38014 MEMCOPY(p, &v, sizeof(v)); \
38015 }
38016  
38017 DEFINE_UNALIGNED_TYPE(u16)
38018 DEFINE_UNALIGNED_TYPE(u32)
38019 DEFINE_UNALIGNED_TYPE(u64)
38020 DEFINE_UNALIGNED_TYPE(machine_word_t)
38021  
38022 #undef MEMCOPY
38023  
38024 #define load_word_unaligned load_machine_word_t_unaligned
38025 #define store_word_unaligned store_machine_word_t_unaligned
38026  
38027  
38028  
38029 static forceinline u16
38030 get_unaligned_le16(const u8 *p)
38031 {
38032 if (UNALIGNED_ACCESS_IS_FAST)
38033 return le16_bswap(load_u16_unaligned(p));
38034 else
38035 return ((u16)p[1] << 8) | p[0];
38036 }
38037  
38038 static forceinline u16
38039 get_unaligned_be16(const u8 *p)
38040 {
38041 if (UNALIGNED_ACCESS_IS_FAST)
38042 return be16_bswap(load_u16_unaligned(p));
38043 else
38044 return ((u16)p[0] << 8) | p[1];
38045 }
38046  
38047 static forceinline u32
38048 get_unaligned_le32(const u8 *p)
38049 {
38050 if (UNALIGNED_ACCESS_IS_FAST)
38051 return le32_bswap(load_u32_unaligned(p));
38052 else
38053 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
38054 ((u32)p[1] << 8) | p[0];
38055 }
38056  
38057 static forceinline u32
38058 get_unaligned_be32(const u8 *p)
38059 {
38060 if (UNALIGNED_ACCESS_IS_FAST)
38061 return be32_bswap(load_u32_unaligned(p));
38062 else
38063 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
38064 ((u32)p[2] << 8) | p[3];
38065 }
38066  
38067 static forceinline u64
38068 get_unaligned_le64(const u8 *p)
38069 {
38070 if (UNALIGNED_ACCESS_IS_FAST)
38071 return le64_bswap(load_u64_unaligned(p));
38072 else
38073 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
38074 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
38075 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
38076 ((u64)p[1] << 8) | p[0];
38077 }
38078  
38079 static forceinline machine_word_t
38080 get_unaligned_leword(const u8 *p)
38081 {
38082 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
38083 if (WORDBITS == 32)
38084 return get_unaligned_le32(p);
38085 else
38086 return get_unaligned_le64(p);
38087 }
38088  
38089  
38090  
38091 static forceinline void
38092 put_unaligned_le16(u16 v, u8 *p)
38093 {
38094 if (UNALIGNED_ACCESS_IS_FAST) {
38095 store_u16_unaligned(le16_bswap(v), p);
38096 } else {
38097 p[0] = (u8)(v >> 0);
38098 p[1] = (u8)(v >> 8);
38099 }
38100 }
38101  
38102 static forceinline void
38103 put_unaligned_be16(u16 v, u8 *p)
38104 {
38105 if (UNALIGNED_ACCESS_IS_FAST) {
38106 store_u16_unaligned(be16_bswap(v), p);
38107 } else {
38108 p[0] = (u8)(v >> 8);
38109 p[1] = (u8)(v >> 0);
38110 }
38111 }
38112  
38113 static forceinline void
38114 put_unaligned_le32(u32 v, u8 *p)
38115 {
38116 if (UNALIGNED_ACCESS_IS_FAST) {
38117 store_u32_unaligned(le32_bswap(v), p);
38118 } else {
38119 p[0] = (u8)(v >> 0);
38120 p[1] = (u8)(v >> 8);
38121 p[2] = (u8)(v >> 16);
38122 p[3] = (u8)(v >> 24);
38123 }
38124 }
38125  
38126 static forceinline void
38127 put_unaligned_be32(u32 v, u8 *p)
38128 {
38129 if (UNALIGNED_ACCESS_IS_FAST) {
38130 store_u32_unaligned(be32_bswap(v), p);
38131 } else {
38132 p[0] = (u8)(v >> 24);
38133 p[1] = (u8)(v >> 16);
38134 p[2] = (u8)(v >> 8);
38135 p[3] = (u8)(v >> 0);
38136 }
38137 }
38138  
38139 static forceinline void
38140 put_unaligned_le64(u64 v, u8 *p)
38141 {
38142 if (UNALIGNED_ACCESS_IS_FAST) {
38143 store_u64_unaligned(le64_bswap(v), p);
38144 } else {
38145 p[0] = (u8)(v >> 0);
38146 p[1] = (u8)(v >> 8);
38147 p[2] = (u8)(v >> 16);
38148 p[3] = (u8)(v >> 24);
38149 p[4] = (u8)(v >> 32);
38150 p[5] = (u8)(v >> 40);
38151 p[6] = (u8)(v >> 48);
38152 p[7] = (u8)(v >> 56);
38153 }
38154 }
38155  
38156 static forceinline void
38157 put_unaligned_leword(machine_word_t v, u8 *p)
38158 {
38159 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
38160 if (WORDBITS == 32)
38161 put_unaligned_le32(v, p);
38162 else
38163 put_unaligned_le64(v, p);
38164 }
38165  
38166  
38167  
38168  
38169  
38170  
38171  
38172 static forceinline unsigned
38173 bsr32(u32 v)
38174 {
38175 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
38176 return 31 - __builtin_clz(v);
38177 #elif defined(_MSC_VER)
38178 unsigned long i;
38179  
38180 _BitScanReverse(&i, v);
38181 return i;
38182 #else
38183 unsigned i = 0;
38184  
38185 while ((v >>= 1) != 0)
38186 i++;
38187 return i;
38188 #endif
38189 }
38190  
38191 static forceinline unsigned
38192 bsr64(u64 v)
38193 {
38194 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
38195 return 63 - __builtin_clzll(v);
38196 #elif defined(_MSC_VER) && defined(_WIN64)
38197 unsigned long i;
38198  
38199 _BitScanReverse64(&i, v);
38200 return i;
38201 #else
38202 unsigned i = 0;
38203  
38204 while ((v >>= 1) != 0)
38205 i++;
38206 return i;
38207 #endif
38208 }
38209  
38210 static forceinline unsigned
38211 bsrw(machine_word_t v)
38212 {
38213 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
38214 if (WORDBITS == 32)
38215 return bsr32(v);
38216 else
38217 return bsr64(v);
38218 }
38219  
38220  
38221  
38222 static forceinline unsigned
38223 bsf32(u32 v)
38224 {
38225 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
38226 return __builtin_ctz(v);
38227 #elif defined(_MSC_VER)
38228 unsigned long i;
38229  
38230 _BitScanForward(&i, v);
38231 return i;
38232 #else
38233 unsigned i = 0;
38234  
38235 for (; (v & 1) == 0; v >>= 1)
38236 i++;
38237 return i;
38238 #endif
38239 }
38240  
38241 static forceinline unsigned
38242 bsf64(u64 v)
38243 {
38244 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
38245 return __builtin_ctzll(v);
38246 #elif defined(_MSC_VER) && defined(_WIN64)
38247 unsigned long i;
38248  
38249 _BitScanForward64(&i, v);
38250 return i;
38251 #else
38252 unsigned i = 0;
38253  
38254 for (; (v & 1) == 0; v >>= 1)
38255 i++;
38256 return i;
38257 #endif
38258 }
38259  
38260 static forceinline unsigned
38261 bsfw(machine_word_t v)
38262 {
38263 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
38264 if (WORDBITS == 32)
38265 return bsf32(v);
38266 else
38267 return bsf64(v);
38268 }
38269  
38270  
38271 #undef rbit32
38272 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
38273 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
38274 static forceinline u32
38275 rbit32(u32 v)
38276 {
38277 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
38278 return v;
38279 }
38280 #define rbit32 rbit32
38281 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
38282 static forceinline u32
38283 rbit32(u32 v)
38284 {
38285 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
38286 return v;
38287 }
38288 #define rbit32 rbit32
38289 #endif
38290  
38291 #endif
38292  
38293  
38294 typedef void *(*malloc_func_t)(size_t);
38295 typedef void (*free_func_t)(void *);
38296  
38297 extern malloc_func_t libdeflate_default_malloc_func;
38298 extern free_func_t libdeflate_default_free_func;
38299  
38300 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
38301 size_t alignment, size_t size);
38302 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
38303  
38304 #ifdef FREESTANDING
38305  
38306 void *memset(void *s, int c, size_t n);
38307 #define memset(s, c, n) __builtin_memset((s), (c), (n))
38308  
38309 void *memcpy(void *dest, const void *src, size_t n);
38310 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
38311  
38312 void *memmove(void *dest, const void *src, size_t n);
38313 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
38314  
38315 int memcmp(const void *s1, const void *s2, size_t n);
38316 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
38317  
38318 #undef LIBDEFLATE_ENABLE_ASSERTIONS
38319 #else
38320 # include <string.h>
38321
38322 # ifdef __clang_analyzer__
38323 # define LIBDEFLATE_ENABLE_ASSERTIONS
38324 # endif
38325 #endif
38326  
38327  
38328 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
38329 NORETURN void
38330 libdeflate_assertion_failed(const char *expr, const char *file, int line);
38331 #define ASSERT(expr) { if (unlikely(!(expr))) \
38332 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
38333 #else
38334 #define ASSERT(expr) (void)(expr)
38335 #endif
38336  
38337 #define CONCAT_IMPL(a, b) a##b
38338 #define CONCAT(a, b) CONCAT_IMPL(a, b)
38339 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
38340  
38341 #endif
38342  
38343  
38344 #if defined(ARCH_ARM32) || defined(ARCH_ARM64)
38345  
38346 #define ARM_CPU_FEATURE_NEON (1 << 0)
38347 #define ARM_CPU_FEATURE_PMULL (1 << 1)
38348  
38349 #define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
38350 #define ARM_CPU_FEATURE_CRC32 (1 << 3)
38351 #define ARM_CPU_FEATURE_SHA3 (1 << 4)
38352 #define ARM_CPU_FEATURE_DOTPROD (1 << 5)
38353  
38354 #if !defined(FREESTANDING) && \
38355 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
38356 (defined(__linux__) || \
38357 (defined(__APPLE__) && defined(ARCH_ARM64)) || \
38358 (defined(_WIN32) && defined(ARCH_ARM64)))
38359  
38360 # define ARM_CPU_FEATURES_KNOWN (1U << 31)
38361 extern volatile u32 libdeflate_arm_cpu_features;
38362  
38363 void libdeflate_init_arm_cpu_features(void);
38364  
38365 static inline u32 get_arm_cpu_features(void)
38366 {
38367 if (libdeflate_arm_cpu_features == 0)
38368 libdeflate_init_arm_cpu_features();
38369 return libdeflate_arm_cpu_features;
38370 }
38371 #else
38372 static inline u32 get_arm_cpu_features(void) { return 0; }
38373 #endif
38374  
38375  
38376 #if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
38377 # define HAVE_NEON(features) 1
38378 # define HAVE_NEON_NATIVE 1
38379 #else
38380 # define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
38381 # define HAVE_NEON_NATIVE 0
38382 #endif
38383  
38384 #if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
38385 (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
38386 # define HAVE_NEON_INTRIN 1
38387 # include <arm_neon.h>
38388 #else
38389 # define HAVE_NEON_INTRIN 0
38390 #endif
38391  
38392  
38393 #ifdef __ARM_FEATURE_CRYPTO
38394 # define HAVE_PMULL(features) 1
38395 #else
38396 # define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
38397 #endif
38398 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
38399 (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
38400 CPU_IS_LITTLE_ENDIAN()
38401 # define HAVE_PMULL_INTRIN 1
38402
38403 # ifdef _MSC_VER
38404 # define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
38405 # else
38406 # define compat_vmull_p64(a, b) vmull_p64((a), (b))
38407 # endif
38408 #else
38409 # define HAVE_PMULL_INTRIN 0
38410 #endif
38411  
38412  
38413 #ifdef __ARM_FEATURE_CRC32
38414 # define HAVE_CRC32(features) 1
38415 #else
38416 # define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
38417 #endif
38418 #if defined(ARCH_ARM64) && \
38419 (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
38420 # define HAVE_CRC32_INTRIN 1
38421 # if defined(__GNUC__) || defined(__clang__)
38422 # include <arm_acle.h>
38423 # endif
38424
38425 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
38426 !defined(__ARM_FEATURE_CRC32)
38427 # undef __crc32b
38428 # define __crc32b(a, b) \
38429 ({ uint32_t res; \
38430 __asm__("crc32b %w0, %w1, %w2" \
38431 : "=r" (res) : "r" (a), "r" (b)); \
38432 res; })
38433 # undef __crc32h
38434 # define __crc32h(a, b) \
38435 ({ uint32_t res; \
38436 __asm__("crc32h %w0, %w1, %w2" \
38437 : "=r" (res) : "r" (a), "r" (b)); \
38438 res; })
38439 # undef __crc32w
38440 # define __crc32w(a, b) \
38441 ({ uint32_t res; \
38442 __asm__("crc32w %w0, %w1, %w2" \
38443 : "=r" (res) : "r" (a), "r" (b)); \
38444 res; })
38445 # undef __crc32d
38446 # define __crc32d(a, b) \
38447 ({ uint32_t res; \
38448 __asm__("crc32x %w0, %w1, %2" \
38449 : "=r" (res) : "r" (a), "r" (b)); \
38450 res; })
38451 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
38452 # endif
38453 #else
38454 # define HAVE_CRC32_INTRIN 0
38455 #endif
38456  
38457  
38458 #ifdef __ARM_FEATURE_SHA3
38459 # define HAVE_SHA3(features) 1
38460 #else
38461 # define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
38462 #endif
38463 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
38464 (GCC_PREREQ(9, 1) || \
38465 CLANG_PREREQ(7, 0, 10010463) )
38466 # define HAVE_SHA3_INTRIN 1
38467
38468 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
38469 !defined(__ARM_FEATURE_SHA3)
38470 # undef veor3q_u8
38471 # define veor3q_u8(a, b, c) \
38472 ({ uint8x16_t res; \
38473 __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
38474 : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
38475 res; })
38476 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
38477 # endif
38478 #else
38479 # define HAVE_SHA3_INTRIN 0
38480 #endif
38481  
38482  
38483 #ifdef __ARM_FEATURE_DOTPROD
38484 # define HAVE_DOTPROD(features) 1
38485 #else
38486 # define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
38487 #endif
38488 #if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
38489 (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
38490 # define HAVE_DOTPROD_INTRIN 1
38491
38492 # if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
38493 !defined(__ARM_FEATURE_DOTPROD)
38494 # undef vdotq_u32
38495 # define vdotq_u32(a, b, c) \
38496 ({ uint32x4_t res = (a); \
38497 __asm__("udot %0.4s, %1.16b, %2.16b" \
38498 : "+w" (res) : "w" (b), "w" (c)); \
38499 res; })
38500 # pragma clang diagnostic ignored "-Wgnu-statement-expression"
38501 # endif
38502 #else
38503 # define HAVE_DOTPROD_INTRIN 0
38504 #endif
38505  
38506 #endif
38507  
38508 #endif
38509  
38510  
38511 #ifdef ARM_CPU_FEATURES_KNOWN
38512  
38513  
38514 #ifdef __linux__
38515  
38516  
38517 #include <errno.h>
38518 #include <fcntl.h>
38519 #include <stdlib.h>
38520 #include <string.h>
38521 #include <unistd.h>
38522  
38523 #define AT_HWCAP 16
38524 #define AT_HWCAP2 26
38525  
38526 static void scan_auxv(unsigned long *hwcap, unsigned long *hwcap2)
38527 {
38528 int fd;
38529 unsigned long auxbuf[32];
38530 int filled = 0;
38531 int i;
38532  
38533 fd = open("/proc/self/auxv", O_RDONLY | O_CLOEXEC);
38534 if (fd < 0)
38535 return;
38536  
38537 for (;;) {
38538 do {
38539 int ret = read(fd, &((char *)auxbuf)[filled],
38540 sizeof(auxbuf) - filled);
38541 if (ret <= 0) {
38542 if (ret < 0 && errno == EINTR)
38543 continue;
38544 goto out;
38545 }
38546 filled += ret;
38547 } while (filled < 2 * sizeof(long));
38548  
38549 i = 0;
38550 do {
38551 unsigned long type = auxbuf[i];
38552 unsigned long value = auxbuf[i + 1];
38553  
38554 if (type == AT_HWCAP)
38555 *hwcap = value;
38556 else if (type == AT_HWCAP2)
38557 *hwcap2 = value;
38558 i += 2;
38559 filled -= 2 * sizeof(long);
38560 } while (filled >= 2 * sizeof(long));
38561  
38562 memmove(auxbuf, &auxbuf[i], filled);
38563 }
38564 out:
38565 close(fd);
38566 }
38567  
38568 static u32 query_arm_cpu_features(void)
38569 {
38570 u32 features = 0;
38571 unsigned long hwcap = 0;
38572 unsigned long hwcap2 = 0;
38573  
38574 scan_auxv(&hwcap, &hwcap2);
38575  
38576 #ifdef ARCH_ARM32
38577 STATIC_ASSERT(sizeof(long) == 4);
38578 if (hwcap & (1 << 12))
38579 features |= ARM_CPU_FEATURE_NEON;
38580 #else
38581 STATIC_ASSERT(sizeof(long) == 8);
38582 if (hwcap & (1 << 1))
38583 features |= ARM_CPU_FEATURE_NEON;
38584 if (hwcap & (1 << 4))
38585 features |= ARM_CPU_FEATURE_PMULL;
38586 if (hwcap & (1 << 7))
38587 features |= ARM_CPU_FEATURE_CRC32;
38588 if (hwcap & (1 << 17))
38589 features |= ARM_CPU_FEATURE_SHA3;
38590 if (hwcap & (1 << 20))
38591 features |= ARM_CPU_FEATURE_DOTPROD;
38592 #endif
38593 return features;
38594 }
38595  
38596 #ifdef ARCH_ARM64
38597  
38598 static bool arm64_cpu_is_neoverse_v_class(void)
38599 {
38600 int fd;
38601 char buf[32];
38602 ssize_t n;
38603 unsigned long midr;
38604 u32 part;
38605  
38606 fd = open("/sys/devices/system/cpu/cpu0/regs/identification/midr_el1",
38607 O_RDONLY | O_CLOEXEC);
38608 if (fd < 0)
38609 return false;
38610 do {
38611 n = read(fd, buf, sizeof(buf) - 1);
38612 } while (n < 0 && errno == EINTR);
38613 close(fd);
38614 if (n <= 0)
38615 return false;
38616 buf[n] = '\0';
38617 midr = strtoul(buf, NULL, 0);
38618  
38619
38620 if (((midr >> 24) & 0xff) != 0x41)
38621 return false;
38622 part = (midr >> 4) & 0xfff;
38623 switch (part) {
38624 case 0xd40:
38625 case 0xd4f:
38626 case 0xd83:
38627 case 0xd84:
38628 return true;
38629 }
38630 return false;
38631 }
38632 #endif
38633  
38634 #elif defined(__APPLE__)
38635  
38636  
38637 #include <sys/types.h>
38638 #include <sys/sysctl.h>
38639 #include <TargetConditionals.h>
38640  
38641 static const struct {
38642 const char *name;
38643 u32 feature;
38644 } feature_sysctls[] = {
38645 { "hw.optional.neon", ARM_CPU_FEATURE_NEON },
38646 { "hw.optional.AdvSIMD", ARM_CPU_FEATURE_NEON },
38647 { "hw.optional.arm.FEAT_PMULL", ARM_CPU_FEATURE_PMULL },
38648 { "hw.optional.armv8_crc32", ARM_CPU_FEATURE_CRC32 },
38649 { "hw.optional.armv8_2_sha3", ARM_CPU_FEATURE_SHA3 },
38650 { "hw.optional.arm.FEAT_SHA3", ARM_CPU_FEATURE_SHA3 },
38651 { "hw.optional.arm.FEAT_DotProd", ARM_CPU_FEATURE_DOTPROD },
38652 };
38653  
38654 static u32 query_arm_cpu_features(void)
38655 {
38656 u32 features = 0;
38657 size_t i;
38658  
38659 for (i = 0; i < ARRAY_LEN(feature_sysctls); i++) {
38660 const char *name = feature_sysctls[i].name;
38661 u32 val = 0;
38662 size_t valsize = sizeof(val);
38663  
38664 if (sysctlbyname(name, &val, &valsize, NULL, 0) == 0 &&
38665 valsize == sizeof(val) && val == 1)
38666 features |= feature_sysctls[i].feature;
38667 }
38668 return features;
38669 }
38670 #elif defined(_WIN32)
38671  
38672 #include <windows.h>
38673  
38674 #ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
38675 # define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
38676 #endif
38677  
38678 static u32 query_arm_cpu_features(void)
38679 {
38680 u32 features = ARM_CPU_FEATURE_NEON;
38681  
38682 if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
38683 features |= ARM_CPU_FEATURE_PMULL;
38684 if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
38685 features |= ARM_CPU_FEATURE_CRC32;
38686 if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE))
38687 features |= ARM_CPU_FEATURE_DOTPROD;
38688  
38689
38690  
38691 return features;
38692 }
38693 #else
38694 #error "unhandled case"
38695 #endif
38696  
38697 static const struct cpu_feature arm_cpu_feature_table[] = {
38698 {ARM_CPU_FEATURE_NEON, "neon"},
38699 {ARM_CPU_FEATURE_PMULL, "pmull"},
38700 {ARM_CPU_FEATURE_PREFER_PMULL, "prefer_pmull"},
38701 {ARM_CPU_FEATURE_CRC32, "crc32"},
38702 {ARM_CPU_FEATURE_SHA3, "sha3"},
38703 {ARM_CPU_FEATURE_DOTPROD, "dotprod"},
38704 };
38705  
38706  
38707 static bool arm_cpu_prefers_pmull(void)
38708 {
38709 #if defined(__APPLE__) && TARGET_OS_OSX
38710 return true;
38711 #elif defined(__linux__) && defined(ARCH_ARM64)
38712 if (arm64_cpu_is_neoverse_v_class())
38713 return true;
38714 #endif
38715 #ifdef TEST_SUPPORT__DO_NOT_USE
38716 return true;
38717 #endif
38718 return false;
38719 }
38720  
38721 volatile u32 libdeflate_arm_cpu_features = 0;
38722  
38723 void libdeflate_init_arm_cpu_features(void)
38724 {
38725 u32 features = query_arm_cpu_features();
38726  
38727 if (arm_cpu_prefers_pmull())
38728 features |= ARM_CPU_FEATURE_PREFER_PMULL;
38729  
38730 disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
38731 ARRAY_LEN(arm_cpu_feature_table));
38732  
38733 libdeflate_arm_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
38734 }
38735  
38736 #endif
38737 /* /usr/home/ben/projects/gzip-libdeflate/../../software/libdeflate/libdeflate-1.26/lib/x86/cpu_features.c */
38738  
38739  
38740 /* #include "cpu_features_common.h" - no include guard */
38741 /* #include "x86-cpu_features.h" */
38742  
38743  
38744 #ifndef LIB_X86_CPU_FEATURES_H
38745 #define LIB_X86_CPU_FEATURES_H
38746  
38747 /* #include "lib_common.h" */
38748  
38749  
38750 #ifndef LIB_LIB_COMMON_H
38751 #define LIB_LIB_COMMON_H
38752  
38753 #ifdef LIBDEFLATE_H
38754
38755 # error "lib_common.h must always be included before libdeflate.h"
38756 #endif
38757  
38758 #if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
38759 # define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
38760 #elif defined(__GNUC__)
38761 # define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
38762 #else
38763 # define LIBDEFLATE_EXPORT_SYM
38764 #endif
38765  
38766  
38767 #if defined(__GNUC__) && defined(__i386__)
38768 # define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
38769 #else
38770 # define LIBDEFLATE_ALIGN_STACK
38771 #endif
38772  
38773 #define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
38774  
38775 /* #include "../common_defs.h" */
38776  
38777  
38778 #ifndef COMMON_DEFS_H
38779 #define COMMON_DEFS_H
38780  
38781 /* #include "libdeflate.h" */
38782  
38783  
38784 #ifndef LIBDEFLATE_H
38785 #define LIBDEFLATE_H
38786  
38787 #include <stddef.h>
38788 #include <stdint.h>
38789  
38790 #ifdef __cplusplus
38791 extern "C" {
38792 #endif
38793  
38794 #define LIBDEFLATE_VERSION_MAJOR 1
38795 #define LIBDEFLATE_VERSION_MINOR 26
38796 #define LIBDEFLATE_VERSION_STRING "1.26"
38797  
38798  
38799 #ifndef LIBDEFLATEAPI
38800 # if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
38801 # define LIBDEFLATEAPI __declspec(dllimport)
38802 # else
38803 # define LIBDEFLATEAPI
38804 # endif
38805 #endif
38806  
38807  
38808  
38809  
38810  
38811 struct libdeflate_compressor;
38812 struct libdeflate_options;
38813  
38814  
38815 LIBDEFLATEAPI struct libdeflate_compressor *
38816 libdeflate_alloc_compressor(int compression_level);
38817  
38818  
38819 LIBDEFLATEAPI struct libdeflate_compressor *
38820 libdeflate_alloc_compressor_ex(int compression_level,
38821 const struct libdeflate_options *options);
38822  
38823  
38824 LIBDEFLATEAPI size_t
38825 libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
38826 const void *in, size_t in_nbytes,
38827 void *out, size_t out_nbytes_avail);
38828  
38829  
38830 LIBDEFLATEAPI size_t
38831 libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
38832 size_t in_nbytes);
38833  
38834  
38835 LIBDEFLATEAPI size_t
38836 libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
38837 const void *in, size_t in_nbytes,
38838 void *out, size_t out_nbytes_avail);
38839  
38840  
38841 LIBDEFLATEAPI size_t
38842 libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
38843 size_t in_nbytes);
38844  
38845  
38846 LIBDEFLATEAPI size_t
38847 libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
38848 const void *in, size_t in_nbytes,
38849 void *out, size_t out_nbytes_avail);
38850  
38851  
38852 LIBDEFLATEAPI size_t
38853 libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
38854 size_t in_nbytes);
38855  
38856  
38857 LIBDEFLATEAPI void
38858 libdeflate_free_compressor(struct libdeflate_compressor *compressor);
38859  
38860  
38861  
38862  
38863  
38864 struct libdeflate_decompressor;
38865 struct libdeflate_options;
38866  
38867  
38868 LIBDEFLATEAPI struct libdeflate_decompressor *
38869 libdeflate_alloc_decompressor(void);
38870  
38871  
38872 LIBDEFLATEAPI struct libdeflate_decompressor *
38873 libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
38874  
38875  
38876 enum libdeflate_result {
38877
38878 LIBDEFLATE_SUCCESS = 0,
38879  
38880
38881 LIBDEFLATE_BAD_DATA = 1,
38882  
38883
38884 LIBDEFLATE_SHORT_OUTPUT = 2,
38885  
38886
38887 LIBDEFLATE_INSUFFICIENT_SPACE = 3,
38888 };
38889  
38890  
38891 LIBDEFLATEAPI enum libdeflate_result
38892 libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
38893 const void *in, size_t in_nbytes,
38894 void *out, size_t out_nbytes_avail,
38895 size_t *actual_out_nbytes_ret);
38896  
38897  
38898 LIBDEFLATEAPI enum libdeflate_result
38899 libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
38900 const void *in, size_t in_nbytes,
38901 void *out, size_t out_nbytes_avail,
38902 size_t *actual_in_nbytes_ret,
38903 size_t *actual_out_nbytes_ret);
38904  
38905  
38906 LIBDEFLATEAPI enum libdeflate_result
38907 libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
38908 const void *in, size_t in_nbytes,
38909 void *out, size_t out_nbytes_avail,
38910 size_t *actual_out_nbytes_ret);
38911  
38912  
38913 LIBDEFLATEAPI enum libdeflate_result
38914 libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
38915 const void *in, size_t in_nbytes,
38916 void *out, size_t out_nbytes_avail,
38917 size_t *actual_in_nbytes_ret,
38918 size_t *actual_out_nbytes_ret);
38919  
38920  
38921 LIBDEFLATEAPI enum libdeflate_result
38922 libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
38923 const void *in, size_t in_nbytes,
38924 void *out, size_t out_nbytes_avail,
38925 size_t *actual_out_nbytes_ret);
38926  
38927  
38928 LIBDEFLATEAPI enum libdeflate_result
38929 libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
38930 const void *in, size_t in_nbytes,
38931 void *out, size_t out_nbytes_avail,
38932 size_t *actual_in_nbytes_ret,
38933 size_t *actual_out_nbytes_ret);
38934  
38935  
38936 LIBDEFLATEAPI void
38937 libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
38938  
38939  
38940  
38941  
38942  
38943  
38944 LIBDEFLATEAPI uint32_t
38945 libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
38946  
38947  
38948  
38949 LIBDEFLATEAPI uint32_t
38950 libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
38951  
38952  
38953  
38954  
38955  
38956  
38957 LIBDEFLATEAPI void
38958 libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
38959 void (*free_func)(void *));
38960  
38961  
38962 struct libdeflate_options {
38963  
38964
38965 size_t sizeof_options;
38966  
38967
38968 void *(*malloc_func)(size_t);
38969 void (*free_func)(void *);
38970 };
38971  
38972 #ifdef __cplusplus
38973 }
38974 #endif
38975  
38976 #endif
38977  
38978  
38979 #include <stdbool.h>
38980 #include <stddef.h>
38981 #include <stdint.h>
38982 #ifdef _MSC_VER
38983 # include <intrin.h>
38984 # include <stdlib.h>
38985
38986
38987 # pragma warning(disable : 4146)
38988
38989 # pragma warning(disable : 4018)
38990 # pragma warning(disable : 4244)
38991 # pragma warning(disable : 4267)
38992 # pragma warning(disable : 4310)
38993
38994 # pragma warning(disable : 4100)
38995 # pragma warning(disable : 4127)
38996 # pragma warning(disable : 4189)
38997 # pragma warning(disable : 4232)
38998 # pragma warning(disable : 4245)
38999 # pragma warning(disable : 4295)
39000 #endif
39001 #ifndef FREESTANDING
39002 # include <string.h>
39003 #endif
39004  
39005  
39006  
39007  
39008  
39009  
39010 #undef ARCH_X86_64
39011 #undef ARCH_X86_32
39012 #undef ARCH_ARM64
39013 #undef ARCH_ARM32
39014 #undef ARCH_RISCV
39015 #ifdef _MSC_VER
39016
39017 # if defined(_M_X64) && !defined(_M_ARM64EC)
39018 # define ARCH_X86_64
39019 # elif defined(_M_IX86)
39020 # define ARCH_X86_32
39021 # elif defined(_M_ARM64)
39022 # define ARCH_ARM64
39023 # elif defined(_M_ARM)
39024 # define ARCH_ARM32
39025 # endif
39026 #else
39027 # if defined(__x86_64__)
39028 # define ARCH_X86_64
39029 # elif defined(__i386__)
39030 # define ARCH_X86_32
39031 # elif defined(__aarch64__)
39032 # define ARCH_ARM64
39033 # elif defined(__arm__)
39034 # define ARCH_ARM32
39035 # elif defined(__riscv)
39036 # define ARCH_RISCV
39037 # endif
39038 #endif
39039  
39040  
39041  
39042  
39043  
39044  
39045 typedef uint8_t u8;
39046 typedef uint16_t u16;
39047 typedef uint32_t u32;
39048 typedef uint64_t u64;
39049 typedef int8_t s8;
39050 typedef int16_t s16;
39051 typedef int32_t s32;
39052 typedef int64_t s64;
39053  
39054  
39055 #ifdef _MSC_VER
39056 # ifdef _WIN64
39057 typedef long long ssize_t;
39058 # else
39059 typedef long ssize_t;
39060 # endif
39061 #endif
39062  
39063  
39064 typedef size_t machine_word_t;
39065  
39066  
39067 #define WORDBYTES ((int)sizeof(machine_word_t))
39068  
39069  
39070 #define WORDBITS (8 * WORDBYTES)
39071  
39072  
39073  
39074  
39075  
39076  
39077 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
39078 # define GCC_PREREQ(major, minor) \
39079 (__GNUC__ > (major) || \
39080 (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
39081 # if !GCC_PREREQ(4, 9)
39082 # error "gcc versions older than 4.9 are no longer supported"
39083 # endif
39084 #else
39085 # define GCC_PREREQ(major, minor) 0
39086 #endif
39087 #ifdef __clang__
39088 # ifdef __apple_build_version__
39089 # define CLANG_PREREQ(major, minor, apple_version) \
39090 (__apple_build_version__ >= (apple_version))
39091 # else
39092 # define CLANG_PREREQ(major, minor, apple_version) \
39093 (__clang_major__ > (major) || \
39094 (__clang_major__ == (major) && __clang_minor__ >= (minor)))
39095 # endif
39096 # if !CLANG_PREREQ(3, 9, 8000000)
39097 # error "clang versions older than 3.9 are no longer supported"
39098 # endif
39099 #else
39100 # define CLANG_PREREQ(major, minor, apple_version) 0
39101 #endif
39102 #ifdef _MSC_VER
39103 # define MSVC_PREREQ(version) (_MSC_VER >= (version))
39104 # if !MSVC_PREREQ(1928)
39105 # error "MSVC versions older than Visual Studio 2019 v16.8 are no longer supported"
39106 # endif
39107 #else
39108 # define MSVC_PREREQ(version) 0
39109 #endif
39110  
39111  
39112 #ifndef __has_attribute
39113 # define __has_attribute(attribute) 0
39114 #endif
39115  
39116  
39117 #ifndef __has_builtin
39118 # define __has_builtin(builtin) 0
39119 #endif
39120  
39121  
39122 #ifdef _MSC_VER
39123 # define inline __inline
39124 #endif
39125  
39126  
39127 #if defined(__GNUC__) || __has_attribute(always_inline)
39128 # define forceinline inline __attribute__((always_inline))
39129 #elif defined(_MSC_VER)
39130 # define forceinline __forceinline
39131 #else
39132 # define forceinline inline
39133 #endif
39134  
39135  
39136 #if defined(__GNUC__) || __has_attribute(unused)
39137 # define MAYBE_UNUSED __attribute__((unused))
39138 #else
39139 # define MAYBE_UNUSED
39140 #endif
39141  
39142  
39143 #if defined(__GNUC__) || __has_attribute(noreturn)
39144 # define NORETURN __attribute__((noreturn))
39145 #else
39146 # define NORETURN
39147 #endif
39148  
39149  
39150 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
39151 # define likely(expr) __builtin_expect(!!(expr), 1)
39152 #else
39153 # define likely(expr) (expr)
39154 #endif
39155  
39156  
39157 #if defined(__GNUC__) || __has_builtin(__builtin_expect)
39158 # define unlikely(expr) __builtin_expect(!!(expr), 0)
39159 #else
39160 # define unlikely(expr) (expr)
39161 #endif
39162  
39163  
39164 #undef prefetchr
39165 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
39166 # define prefetchr(addr) __builtin_prefetch((addr), 0)
39167 #elif defined(_MSC_VER)
39168 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
39169 # define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
39170 # elif defined(ARCH_ARM64)
39171 # define prefetchr(addr) __prefetch2((addr), 0x00 )
39172 # elif defined(ARCH_ARM32)
39173 # define prefetchr(addr) __prefetch(addr)
39174 # endif
39175 #endif
39176 #ifndef prefetchr
39177 # define prefetchr(addr)
39178 #endif
39179  
39180  
39181 #undef prefetchw
39182 #if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
39183 # define prefetchw(addr) __builtin_prefetch((addr), 1)
39184 #elif defined(_MSC_VER)
39185 # if defined(ARCH_X86_32) || defined(ARCH_X86_64)
39186 # define prefetchw(addr) _m_prefetchw(addr)
39187 # elif defined(ARCH_ARM64)
39188 # define prefetchw(addr) __prefetch2((addr), 0x10 )
39189 # elif defined(ARCH_ARM32)
39190 # define prefetchw(addr) __prefetchw(addr)
39191 # endif
39192 #endif
39193 #ifndef prefetchw
39194 # define prefetchw(addr)
39195 #endif
39196  
39197  
39198 #undef _aligned_attribute
39199 #if defined(__GNUC__) || __has_attribute(aligned)
39200 # define _aligned_attribute(n) __attribute__((aligned(n)))
39201 #elif defined(_MSC_VER)
39202 # define _aligned_attribute(n) __declspec(align(n))
39203 #endif
39204  
39205  
39206 #if defined(__GNUC__) || __has_attribute(target)
39207 # define _target_attribute(attrs) __attribute__((target(attrs)))
39208 #else
39209 # define _target_attribute(attrs)
39210 #endif
39211  
39212  
39213  
39214  
39215  
39216 #define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
39217 #define MIN(a, b) ((a) <= (b) ? (a) : (b))
39218 #define MAX(a, b) ((a) >= (b) ? (a) : (b))
39219 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
39220 #define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
39221 #define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
39222 #define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
39223  
39224  
39225  
39226  
39227  
39228  
39229 #if defined(__BYTE_ORDER__)
39230 # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
39231 #elif defined(_MSC_VER)
39232 # define CPU_IS_LITTLE_ENDIAN() true
39233 #else
39234 static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
39235 {
39236 union {
39237 u32 w;
39238 u8 b;
39239 } u;
39240  
39241 u.w = 1;
39242 return u.b;
39243 }
39244 #endif
39245  
39246  
39247 static forceinline u16 bswap16(u16 v)
39248 {
39249 #if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
39250 return __builtin_bswap16(v);
39251 #elif defined(_MSC_VER)
39252 return _byteswap_ushort(v);
39253 #else
39254 return (v << 8) | (v >> 8);
39255 #endif
39256 }
39257  
39258  
39259 static forceinline u32 bswap32(u32 v)
39260 {
39261 #if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
39262 return __builtin_bswap32(v);
39263 #elif defined(_MSC_VER)
39264 return _byteswap_ulong(v);
39265 #else
39266 return ((v & 0x000000FF) << 24) |
39267 ((v & 0x0000FF00) << 8) |
39268 ((v & 0x00FF0000) >> 8) |
39269 ((v & 0xFF000000) >> 24);
39270 #endif
39271 }
39272  
39273  
39274 static forceinline u64 bswap64(u64 v)
39275 {
39276 #if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
39277 return __builtin_bswap64(v);
39278 #elif defined(_MSC_VER)
39279 return _byteswap_uint64(v);
39280 #else
39281 return ((v & 0x00000000000000FF) << 56) |
39282 ((v & 0x000000000000FF00) << 40) |
39283 ((v & 0x0000000000FF0000) << 24) |
39284 ((v & 0x00000000FF000000) << 8) |
39285 ((v & 0x000000FF00000000) >> 8) |
39286 ((v & 0x0000FF0000000000) >> 24) |
39287 ((v & 0x00FF000000000000) >> 40) |
39288 ((v & 0xFF00000000000000) >> 56);
39289 #endif
39290 }
39291  
39292 #define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
39293 #define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
39294 #define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
39295 #define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
39296 #define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
39297 #define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
39298  
39299  
39300  
39301  
39302  
39303  
39304 #if (defined(__GNUC__) || defined(__clang__)) && \
39305 (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
39306 defined(__ARM_FEATURE_UNALIGNED) || \
39307 defined(__powerpc64__) || defined(__powerpc__) || defined(__POWERPC__) || \
39308 defined(__riscv_misaligned_fast) || \
39309 defined(__wasm__))
39310 # define UNALIGNED_ACCESS_IS_FAST 1
39311 #elif defined(_MSC_VER)
39312 # define UNALIGNED_ACCESS_IS_FAST 1
39313 #else
39314 # define UNALIGNED_ACCESS_IS_FAST 0
39315 #endif
39316  
39317  
39318  
39319 #ifdef FREESTANDING
39320 # define MEMCOPY __builtin_memcpy
39321 #else
39322 # define MEMCOPY memcpy
39323 #endif
39324  
39325  
39326  
39327 #define DEFINE_UNALIGNED_TYPE(type) \
39328 static forceinline type \
39329 load_##type##_unaligned(const void *p) \
39330 { \
39331 type v; \
39332 \
39333 MEMCOPY(&v, p, sizeof(v)); \
39334 return v; \
39335 } \
39336 \
39337 static forceinline void \
39338 store_##type##_unaligned(type v, void *p) \
39339 { \
39340 MEMCOPY(p, &v, sizeof(v)); \
39341 }
39342  
39343 DEFINE_UNALIGNED_TYPE(u16)
39344 DEFINE_UNALIGNED_TYPE(u32)
39345 DEFINE_UNALIGNED_TYPE(u64)
39346 DEFINE_UNALIGNED_TYPE(machine_word_t)
39347  
39348 #undef MEMCOPY
39349  
39350 #define load_word_unaligned load_machine_word_t_unaligned
39351 #define store_word_unaligned store_machine_word_t_unaligned
39352  
39353  
39354  
39355 static forceinline u16
39356 get_unaligned_le16(const u8 *p)
39357 {
39358 if (UNALIGNED_ACCESS_IS_FAST)
39359 return le16_bswap(load_u16_unaligned(p));
39360 else
39361 return ((u16)p[1] << 8) | p[0];
39362 }
39363  
39364 static forceinline u16
39365 get_unaligned_be16(const u8 *p)
39366 {
39367 if (UNALIGNED_ACCESS_IS_FAST)
39368 return be16_bswap(load_u16_unaligned(p));
39369 else
39370 return ((u16)p[0] << 8) | p[1];
39371 }
39372  
39373 static forceinline u32
39374 get_unaligned_le32(const u8 *p)
39375 {
39376 if (UNALIGNED_ACCESS_IS_FAST)
39377 return le32_bswap(load_u32_unaligned(p));
39378 else
39379 return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
39380 ((u32)p[1] << 8) | p[0];
39381 }
39382  
39383 static forceinline u32
39384 get_unaligned_be32(const u8 *p)
39385 {
39386 if (UNALIGNED_ACCESS_IS_FAST)
39387 return be32_bswap(load_u32_unaligned(p));
39388 else
39389 return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
39390 ((u32)p[2] << 8) | p[3];
39391 }
39392  
39393 static forceinline u64
39394 get_unaligned_le64(const u8 *p)
39395 {
39396 if (UNALIGNED_ACCESS_IS_FAST)
39397 return le64_bswap(load_u64_unaligned(p));
39398 else
39399 return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
39400 ((u64)p[5] << 40) | ((u64)p[4] << 32) |
39401 ((u64)p[3] << 24) | ((u64)p[2] << 16) |
39402 ((u64)p[1] << 8) | p[0];
39403 }
39404  
39405 static forceinline machine_word_t
39406 get_unaligned_leword(const u8 *p)
39407 {
39408 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
39409 if (WORDBITS == 32)
39410 return get_unaligned_le32(p);
39411 else
39412 return get_unaligned_le64(p);
39413 }
39414  
39415  
39416  
39417 static forceinline void
39418 put_unaligned_le16(u16 v, u8 *p)
39419 {
39420 if (UNALIGNED_ACCESS_IS_FAST) {
39421 store_u16_unaligned(le16_bswap(v), p);
39422 } else {
39423 p[0] = (u8)(v >> 0);
39424 p[1] = (u8)(v >> 8);
39425 }
39426 }
39427  
39428 static forceinline void
39429 put_unaligned_be16(u16 v, u8 *p)
39430 {
39431 if (UNALIGNED_ACCESS_IS_FAST) {
39432 store_u16_unaligned(be16_bswap(v), p);
39433 } else {
39434 p[0] = (u8)(v >> 8);
39435 p[1] = (u8)(v >> 0);
39436 }
39437 }
39438  
39439 static forceinline void
39440 put_unaligned_le32(u32 v, u8 *p)
39441 {
39442 if (UNALIGNED_ACCESS_IS_FAST) {
39443 store_u32_unaligned(le32_bswap(v), p);
39444 } else {
39445 p[0] = (u8)(v >> 0);
39446 p[1] = (u8)(v >> 8);
39447 p[2] = (u8)(v >> 16);
39448 p[3] = (u8)(v >> 24);
39449 }
39450 }
39451  
39452 static forceinline void
39453 put_unaligned_be32(u32 v, u8 *p)
39454 {
39455 if (UNALIGNED_ACCESS_IS_FAST) {
39456 store_u32_unaligned(be32_bswap(v), p);
39457 } else {
39458 p[0] = (u8)(v >> 24);
39459 p[1] = (u8)(v >> 16);
39460 p[2] = (u8)(v >> 8);
39461 p[3] = (u8)(v >> 0);
39462 }
39463 }
39464  
39465 static forceinline void
39466 put_unaligned_le64(u64 v, u8 *p)
39467 {
39468 if (UNALIGNED_ACCESS_IS_FAST) {
39469 store_u64_unaligned(le64_bswap(v), p);
39470 } else {
39471 p[0] = (u8)(v >> 0);
39472 p[1] = (u8)(v >> 8);
39473 p[2] = (u8)(v >> 16);
39474 p[3] = (u8)(v >> 24);
39475 p[4] = (u8)(v >> 32);
39476 p[5] = (u8)(v >> 40);
39477 p[6] = (u8)(v >> 48);
39478 p[7] = (u8)(v >> 56);
39479 }
39480 }
39481  
39482 static forceinline void
39483 put_unaligned_leword(machine_word_t v, u8 *p)
39484 {
39485 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
39486 if (WORDBITS == 32)
39487 put_unaligned_le32(v, p);
39488 else
39489 put_unaligned_le64(v, p);
39490 }
39491  
39492  
39493  
39494  
39495  
39496  
39497  
39498 static forceinline unsigned
39499 bsr32(u32 v)
39500 {
39501 #if defined(__GNUC__) || __has_builtin(__builtin_clz)
39502 return 31 - __builtin_clz(v);
39503 #elif defined(_MSC_VER)
39504 unsigned long i;
39505  
39506 _BitScanReverse(&i, v);
39507 return i;
39508 #else
39509 unsigned i = 0;
39510  
39511 while ((v >>= 1) != 0)
39512 i++;
39513 return i;
39514 #endif
39515 }
39516  
39517 static forceinline unsigned
39518 bsr64(u64 v)
39519 {
39520 #if defined(__GNUC__) || __has_builtin(__builtin_clzll)
39521 return 63 - __builtin_clzll(v);
39522 #elif defined(_MSC_VER) && defined(_WIN64)
39523 unsigned long i;
39524  
39525 _BitScanReverse64(&i, v);
39526 return i;
39527 #else
39528 unsigned i = 0;
39529  
39530 while ((v >>= 1) != 0)
39531 i++;
39532 return i;
39533 #endif
39534 }
39535  
39536 static forceinline unsigned
39537 bsrw(machine_word_t v)
39538 {
39539 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
39540 if (WORDBITS == 32)
39541 return bsr32(v);
39542 else
39543 return bsr64(v);
39544 }
39545  
39546  
39547  
39548 static forceinline unsigned
39549 bsf32(u32 v)
39550 {
39551 #if defined(__GNUC__) || __has_builtin(__builtin_ctz)
39552 return __builtin_ctz(v);
39553 #elif defined(_MSC_VER)
39554 unsigned long i;
39555  
39556 _BitScanForward(&i, v);
39557 return i;
39558 #else
39559 unsigned i = 0;
39560  
39561 for (; (v & 1) == 0; v >>= 1)
39562 i++;
39563 return i;
39564 #endif
39565 }
39566  
39567 static forceinline unsigned
39568 bsf64(u64 v)
39569 {
39570 #if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
39571 return __builtin_ctzll(v);
39572 #elif defined(_MSC_VER) && defined(_WIN64)
39573 unsigned long i;
39574  
39575 _BitScanForward64(&i, v);
39576 return i;
39577 #else
39578 unsigned i = 0;
39579  
39580 for (; (v & 1) == 0; v >>= 1)
39581 i++;
39582 return i;
39583 #endif
39584 }
39585  
39586 static forceinline unsigned
39587 bsfw(machine_word_t v)
39588 {
39589 STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
39590 if (WORDBITS == 32)
39591 return bsf32(v);
39592 else
39593 return bsf64(v);
39594 }
39595  
39596  
39597 #undef rbit32
39598 #if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
39599 (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
39600 static forceinline u32
39601 rbit32(u32 v)
39602 {
39603 __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
39604 return v;
39605 }
39606 #define rbit32 rbit32
39607 #elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
39608 static forceinline u32
39609 rbit32(u32 v)
39610 {
39611 __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
39612 return v;
39613 }
39614 #define rbit32 rbit32
39615 #endif
39616  
39617 #endif
39618  
39619  
39620 typedef void *(*malloc_func_t)(size_t);
39621 typedef void (*free_func_t)(void *);
39622  
39623 extern malloc_func_t libdeflate_default_malloc_func;
39624 extern free_func_t libdeflate_default_free_func;
39625  
39626 void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
39627 size_t alignment, size_t size);
39628 void libdeflate_aligned_free(free_func_t free_func, void *ptr);
39629  
39630 #ifdef FREESTANDING
39631  
39632 void *memset(void *s, int c, size_t n);
39633 #define memset(s, c, n) __builtin_memset((s), (c), (n))
39634  
39635 void *memcpy(void *dest, const void *src, size_t n);
39636 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
39637  
39638 void *memmove(void *dest, const void *src, size_t n);
39639 #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
39640  
39641 int memcmp(const void *s1, const void *s2, size_t n);
39642 #define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
39643  
39644 #undef LIBDEFLATE_ENABLE_ASSERTIONS
39645 #else
39646 # include <string.h>
39647
39648 # ifdef __clang_analyzer__
39649 # define LIBDEFLATE_ENABLE_ASSERTIONS
39650 # endif
39651 #endif
39652  
39653  
39654 #ifdef LIBDEFLATE_ENABLE_ASSERTIONS
39655 NORETURN void
39656 libdeflate_assertion_failed(const char *expr, const char *file, int line);
39657 #define ASSERT(expr) { if (unlikely(!(expr))) \
39658 libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
39659 #else
39660 #define ASSERT(expr) (void)(expr)
39661 #endif
39662  
39663 #define CONCAT_IMPL(a, b) a##b
39664 #define CONCAT(a, b) CONCAT_IMPL(a, b)
39665 #define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
39666  
39667 #endif
39668  
39669  
39670 #if defined(ARCH_X86_32) || defined(ARCH_X86_64)
39671  
39672 #define X86_CPU_FEATURE_SSE2 (1 << 0)
39673 #define X86_CPU_FEATURE_PCLMULQDQ (1 << 1)
39674 #define X86_CPU_FEATURE_AVX (1 << 2)
39675 #define X86_CPU_FEATURE_AVX2 (1 << 3)
39676 #define X86_CPU_FEATURE_BMI2 (1 << 4)
39677  
39678 #define X86_CPU_FEATURE_ZMM (1 << 5)
39679 #define X86_CPU_FEATURE_AVX512BW (1 << 6)
39680 #define X86_CPU_FEATURE_AVX512VL (1 << 7)
39681 #define X86_CPU_FEATURE_VPCLMULQDQ (1 << 8)
39682 #define X86_CPU_FEATURE_AVX512VNNI (1 << 9)
39683 #define X86_CPU_FEATURE_AVXVNNI (1 << 10)
39684  
39685 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
39686  
39687 # define X86_CPU_FEATURES_KNOWN (1U << 31)
39688 extern volatile u32 libdeflate_x86_cpu_features;
39689  
39690 void libdeflate_init_x86_cpu_features(void);
39691  
39692 static inline u32 get_x86_cpu_features(void)
39693 {
39694 if (libdeflate_x86_cpu_features == 0)
39695 libdeflate_init_x86_cpu_features();
39696 return libdeflate_x86_cpu_features;
39697 }
39698  
39699 # include <immintrin.h>
39700 # if defined(_MSC_VER) && defined(__clang__)
39701 # include <tmmintrin.h>
39702 # include <smmintrin.h>
39703 # include <wmmintrin.h>
39704 # include <avxintrin.h>
39705 # include <avx2intrin.h>
39706 # include <avx512fintrin.h>
39707 # include <avx512bwintrin.h>
39708 # include <avx512vlintrin.h>
39709 # if __has_include(<avx512vlbwintrin.h>)
39710 # include <avx512vlbwintrin.h>
39711 # endif
39712 # if __has_include(<vpclmulqdqintrin.h>)
39713 # include <vpclmulqdqintrin.h>
39714 # endif
39715 # if __has_include(<avx512vnniintrin.h>)
39716 # include <avx512vnniintrin.h>
39717 # endif
39718 # if __has_include(<avx512vlvnniintrin.h>)
39719 # include <avx512vlvnniintrin.h>
39720 # endif
39721 # if __has_include(<avxvnniintrin.h>)
39722 # include <avxvnniintrin.h>
39723 # endif
39724 # endif
39725 #else
39726 static inline u32 get_x86_cpu_features(void) { return 0; }
39727 #endif
39728  
39729 #if defined(__SSE2__) || \
39730 (defined(_MSC_VER) && \
39731 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
39732 # define HAVE_SSE2(features) 1
39733 # define HAVE_SSE2_NATIVE 1
39734 #else
39735 # define HAVE_SSE2(features) ((features) & X86_CPU_FEATURE_SSE2)
39736 # define HAVE_SSE2_NATIVE 0
39737 #endif
39738  
39739 #if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
39740 (defined(_MSC_VER) && defined(__AVX2__))
39741 # define HAVE_PCLMULQDQ(features) 1
39742 #else
39743 # define HAVE_PCLMULQDQ(features) ((features) & X86_CPU_FEATURE_PCLMULQDQ)
39744 #endif
39745  
39746 #ifdef __AVX__
39747 # define HAVE_AVX(features) 1
39748 #else
39749 # define HAVE_AVX(features) ((features) & X86_CPU_FEATURE_AVX)
39750 #endif
39751  
39752 #ifdef __AVX2__
39753 # define HAVE_AVX2(features) 1
39754 #else
39755 # define HAVE_AVX2(features) ((features) & X86_CPU_FEATURE_AVX2)
39756 #endif
39757  
39758 #if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
39759 # define HAVE_BMI2(features) 1
39760 # define HAVE_BMI2_NATIVE 1
39761 #else
39762 # define HAVE_BMI2(features) ((features) & X86_CPU_FEATURE_BMI2)
39763 # define HAVE_BMI2_NATIVE 0
39764 #endif
39765  
39766 #ifdef __AVX512BW__
39767 # define HAVE_AVX512BW(features) 1
39768 #else
39769 # define HAVE_AVX512BW(features) ((features) & X86_CPU_FEATURE_AVX512BW)
39770 #endif
39771  
39772 #ifdef __AVX512VL__
39773 # define HAVE_AVX512VL(features) 1
39774 #else
39775 # define HAVE_AVX512VL(features) ((features) & X86_CPU_FEATURE_AVX512VL)
39776 #endif
39777  
39778 #ifdef __VPCLMULQDQ__
39779 # define HAVE_VPCLMULQDQ(features) 1
39780 #else
39781 # define HAVE_VPCLMULQDQ(features) ((features) & X86_CPU_FEATURE_VPCLMULQDQ)
39782 #endif
39783  
39784 #ifdef __AVX512VNNI__
39785 # define HAVE_AVX512VNNI(features) 1
39786 #else
39787 # define HAVE_AVX512VNNI(features) ((features) & X86_CPU_FEATURE_AVX512VNNI)
39788 #endif
39789  
39790 #ifdef __AVXVNNI__
39791 # define HAVE_AVXVNNI(features) 1
39792 #else
39793 # define HAVE_AVXVNNI(features) ((features) & X86_CPU_FEATURE_AVXVNNI)
39794 #endif
39795  
39796 #endif
39797  
39798 #endif
39799  
39800  
39801 #ifdef X86_CPU_FEATURES_KNOWN
39802  
39803  
39804  
39805 static inline void
39806 8 cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
39807 {
39808 #ifdef _MSC_VER
39809 int result[4];
39810  
39811 __cpuidex(result, leaf, subleaf);
39812 *a = result[0];
39813 *b = result[1];
39814 *c = result[2];
39815 *d = result[3];
39816 #else
39817 8 __asm__ volatile("cpuid" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
39818 : "a" (leaf), "c" (subleaf));
39819 #endif
39820 8 }
39821  
39822  
39823 static inline u64
39824 2 read_xcr(u32 index)
39825 {
39826 #ifdef _MSC_VER
39827 return _xgetbv(index);
39828 #else
39829 u32 d, a;
39830  
39831
39832 2 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" :
39833 "=d" (d), "=a" (a) : "c" (index));
39834  
39835 2 return ((u64)d << 32) | a;
39836 #endif
39837 }
39838  
39839 static const struct cpu_feature x86_cpu_feature_table[] = {
39840 {X86_CPU_FEATURE_SSE2, "sse2"},
39841 {X86_CPU_FEATURE_PCLMULQDQ, "pclmulqdq"},
39842 {X86_CPU_FEATURE_AVX, "avx"},
39843 {X86_CPU_FEATURE_AVX2, "avx2"},
39844 {X86_CPU_FEATURE_BMI2, "bmi2"},
39845 {X86_CPU_FEATURE_ZMM, "zmm"},
39846 {X86_CPU_FEATURE_AVX512BW, "avx512bw"},
39847 {X86_CPU_FEATURE_AVX512VL, "avx512vl"},
39848 {X86_CPU_FEATURE_VPCLMULQDQ, "vpclmulqdq"},
39849 {X86_CPU_FEATURE_AVX512VNNI, "avx512_vnni"},
39850 {X86_CPU_FEATURE_AVXVNNI, "avx_vnni"},
39851 };
39852  
39853 volatile u32 libdeflate_x86_cpu_features = 0;
39854  
39855 static inline bool
39856 2 os_supports_avx512(u64 xcr0)
39857 {
39858 #ifdef __APPLE__
39859
39860 return false;
39861 #else
39862 2 return (xcr0 & 0xe6) == 0xe6;
39863 #endif
39864 }
39865  
39866  
39867 static inline bool
39868 0 allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
39869 {
39870 #ifdef TEST_SUPPORT__DO_NOT_USE
39871 return true;
39872 #endif
39873 0 ▶ if (memcmp(manufacturer, "GenuineIntel", 12) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (memcmp(manufacturer, "GenuineIntel", 12) != 0) 0 0 0
39874 0 return true;
39875 0 ▶ if (family != 6)
Branch: 1 branch
BranchTrueFalseTotal
if (family != 6) 0 0 0
39876 0 return true;
39877 0 ▶ switch (model) {
Branch: 1 branch
BranchTrueFalseTotal
switch (model) { 0 0 0
39878 0 case 85:
39879 case 106:
39880 case 108:
39881 case 126:
39882 case 140:
39883 case 141:
39884 0 return false;
39885 }
39886 0 return true;
39887 }
39888  
39889  
39890 2 void libdeflate_init_x86_cpu_features(void)
39891 {
39892 u32 max_leaf;
39893 u32 manufacturer[3];
39894 u32 family, model;
39895 u32 a, b, c, d;
39896 2 u64 xcr0 = 0;
39897 2 u32 features = 0;
39898  
39899
39900 2 cpuid(0, 0, &max_leaf, &manufacturer[0], &manufacturer[2],
39901 &manufacturer[1]);
39902 2 ▶ if (max_leaf < 1)
Branch: 1 branch
BranchTrueFalseTotal
if (max_leaf < 1) 0 2 2
39903 0 goto out;
39904  
39905
39906 2 cpuid(1, 0, &a, &b, &c, &d);
39907 2 family = (a >> 8) & 0xf;
39908 2 model = (a >> 4) & 0xf;
39909 2 ▶ if (family == 6 || family == 0xf)
Branch: 2 branches
BranchTrueFalseTotal
if (family == 6 || family == 0xf) 0 2 2
if (family == 6 || family == 0xf) 0 0 0
39910 2 model += (a >> 12) & 0xf0;
39911 2 ▶ if (family == 0xf)
Branch: 1 branch
BranchTrueFalseTotal
if (family == 0xf) 0 2 2
39912 0 family += (a >> 20) & 0xff;
39913 2 ▶ if (d & (1 << 26))
Branch: 1 branch
BranchTrueFalseTotal
if (d & (1 << 26)) 2 0 2
39914 2 features |= X86_CPU_FEATURE_SSE2;
39915
39916 2 ▶ if ((c & (1 << 1)) && (c & (1 << 19)))
Branch: 2 branches
BranchTrueFalseTotal
if ((c & (1 << 1)) && (c & (1 << 19))) 2 0 2
if ((c & (1 << 1)) && (c & (1 << 19))) 2 0 2
39917 2 features |= X86_CPU_FEATURE_PCLMULQDQ;
39918 2 ▶ if (c & (1 << 27))
Branch: 1 branch
BranchTrueFalseTotal
if (c & (1 << 27)) 2 0 2
39919 2 xcr0 = read_xcr(0);
39920 2 ▶ if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6))
Branch: 2 branches
BranchTrueFalseTotal
if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6)) 2 0 2
if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6)) 2 0 2
39921 2 features |= X86_CPU_FEATURE_AVX;
39922  
39923 2 ▶ if (max_leaf < 7)
Branch: 1 branch
BranchTrueFalseTotal
if (max_leaf < 7) 0 2 2
39924 0 goto out;
39925  
39926
39927 2 cpuid(7, 0, &a, &b, &c, &d);
39928 2 ▶ if (b & (1 << 8))
Branch: 1 branch
BranchTrueFalseTotal
if (b & (1 << 8)) 2 0 2
39929 2 features |= X86_CPU_FEATURE_BMI2;
39930 2 ▶ if ((xcr0 & 0x6) == 0x6) {
Branch: 1 branch
BranchTrueFalseTotal
if ((xcr0 & 0x6) == 0x6) { 2 0 2
39931 2 ▶ if (b & (1 << 5))
Branch: 1 branch
BranchTrueFalseTotal
if (b & (1 << 5)) 2 0 2
39932 2 features |= X86_CPU_FEATURE_AVX2;
39933 2 ▶ if (c & (1 << 10))
Branch: 1 branch
BranchTrueFalseTotal
if (c & (1 << 10)) 0 2 2
39934 0 features |= X86_CPU_FEATURE_VPCLMULQDQ;
39935 }
39936 2 ▶ if (os_supports_avx512(xcr0)) {
Branch: 1 branch
BranchTrueFalseTotal
if (os_supports_avx512(xcr0)) { 0 2 2
39937 0 ▶ if (allow_512bit_vectors(manufacturer, family, model))
Branch: 1 branch
BranchTrueFalseTotal
if (allow_512bit_vectors(manufacturer, family, model)) 0 0 0
39938 0 features |= X86_CPU_FEATURE_ZMM;
39939 0 ▶ if (b & (1 << 30))
Branch: 1 branch
BranchTrueFalseTotal
if (b & (1 << 30)) 0 0 0
39940 0 features |= X86_CPU_FEATURE_AVX512BW;
39941 0 ▶ if (b & (1U << 31))
Branch: 1 branch
BranchTrueFalseTotal
if (b & (1U << 31)) 0 0 0
39942 0 features |= X86_CPU_FEATURE_AVX512VL;
39943 0 ▶ if (c & (1 << 11))
Branch: 1 branch
BranchTrueFalseTotal
if (c & (1 << 11)) 0 0 0
39944 0 features |= X86_CPU_FEATURE_AVX512VNNI;
39945 }
39946  
39947
39948 2 cpuid(7, 1, &a, &b, &c, &d);
39949 2 ▶ if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6))
Branch: 2 branches
BranchTrueFalseTotal
if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6)) 2 0 2
if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6)) 0 0 0
39950 0 features |= X86_CPU_FEATURE_AVXVNNI;
39951  
39952 2 out:
39953 2 disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
39954 ARRAY_LEN(x86_cpu_feature_table));
39955  
39956 2 libdeflate_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
39957 2 }
39958  
39959 #endif
« blib/lib/Gzip/Libdeflate.pm ↑ Summary gzip-libdeflate-perl.c »