| 1 |
|
| /* po_crc32c.h - CRC-32C (Castagnoli), hardware where there is one. |
| 2 |
|
| * |
| 3 |
|
| * CRC-32C rather than the zlib CRC-32 because every architecture that matters |
| 4 |
|
| * has an instruction for it: SSE4.2 on x86-64, the v8 CRC extension on ARM. |
| 5 |
|
| * The polynomial is 0x1EDC6F41, reflected 0x82F63B78, and the reflected form |
| 6 |
|
| * is what both instructions implement. |
| 7 |
|
| * |
| 8 |
|
| * The probes are in Makefile.PL and they LINK rather than merely compile - |
| 9 |
|
| * MSVC exits 0 on an undeclared intrinsic and old gcc treats an implicit |
| 10 |
|
| * declaration as a warning, so a compile-only probe reports an instruction |
| 11 |
|
| * that is not there and the .so dies at load. |
| 12 |
|
| * |
| 13 |
|
| * The table fallback is not a lesser implementation, it is the SAME function. |
| 14 |
|
| * t/0004-wal.t asserts they agree bit for bit on the same input, because a WAL |
| 15 |
|
| * written on a machine with the instruction and replayed on one without has |
| 16 |
|
| * to verify - and that is not a hypothetical, it is a segment archive moved |
| 17 |
|
| * between hosts. |
| 18 |
|
| */ |
| 19 |
|
| #ifndef PO_CRC32C_H |
| 20 |
|
| #define PO_CRC32C_H |
| 21 |
|
| |
| 22 |
|
| #include "punk_observe/po_compat.h" |
| 23 |
|
| |
| 24 |
|
| #if defined(PO_HAVE_CRC32C_X86) |
| 25 |
|
| # include <nmmintrin.h> |
| 26 |
|
| #endif |
| 27 |
|
| #if defined(PO_HAVE_CRC32C_ARM) |
| 28 |
|
| # include <arm_acle.h> |
| 29 |
|
| #endif |
| 30 |
|
| |
| 31 |
|
| /* The reflected Castagnoli table, generated on first use rather than shipped |
| 32 |
|
| * as 1KB of literals nobody can check. */ |
| 33 |
|
| static uint32_t po_crc32c_tab[256]; |
| 34 |
|
| static int po_crc32c_ready = 0; |
| 35 |
|
| |
| 36 |
33081 |
| static void po_crc32c_init(void) { |
| 37 |
|
| uint32_t i, j, c; |
| 38 |
33081 |
▶ | if (po_crc32c_ready) return; |
Branch: 1 branch
| Branch | True | False | Total |
| if (po_crc32c_ready) return; |
33050 |
31 |
33081 |
|
| 39 |
7967 |
▶ | for (i = 0; i < 256; i++) { |
Branch: 1 branch
| Branch | True | False | Total |
| for (i = 0; i < 256; i++) { |
7936 |
31 |
7967 |
|
| 40 |
7936 |
| c = i; |
| 41 |
71424 |
▶ | for (j = 0; j < 8; j++) |
Branch: 1 branch
| Branch | True | False | Total |
| for (j = 0; j < 8; j++) |
63488 |
7936 |
71424 |
|
| 42 |
63488 |
| c = (c & 1) ? (0x82F63B78u ^ (c >> 1)) : (c >> 1); |
| 43 |
7936 |
| po_crc32c_tab[i] = c; |
| 44 |
|
| } |
| 45 |
31 |
| po_crc32c_ready = 1; |
| 46 |
|
| } |
| 47 |
|
| |
| 48 |
33081 |
| static uint32_t po_crc32c_table(uint32_t crc, const void *buf, size_t len) { |
| 49 |
33081 |
| const uint8_t *p = (const uint8_t *)buf; |
| 50 |
33081 |
| po_crc32c_init(); |
| 51 |
33081 |
| crc = ~crc; |
| 52 |
18427054 |
▶ | while (len--) crc = po_crc32c_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); |
Branch: 1 branch
| Branch | True | False | Total |
| while (len--) crc = po_crc32c_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); |
18393973 |
33081 |
18427054 |
|
| 53 |
33081 |
| return ~crc; |
| 54 |
|
| } |
| 55 |
|
| |
| 56 |
|
| /* The public entry. Same signature, same answer, whichever path runs. */ |
| 57 |
32863 |
| static uint32_t po_crc32c(uint32_t crc, const void *buf, size_t len) { |
| 58 |
|
| #if defined(PO_HAVE_CRC32C_X86) || defined(PO_HAVE_CRC32C_ARM) |
| 59 |
|
| const uint8_t *p = (const uint8_t *)buf; |
| 60 |
|
| uint32_t c = ~crc; |
| 61 |
|
| |
| 62 |
|
| /* Align to 8 bytes a byte at a time, then eat 8 at a time. Unaligned |
| 63 |
|
| * loads are legal on both architectures but the aligned path is the one |
| 64 |
|
| * the instruction is scheduled for. */ |
| 65 |
|
| while (len && ((uintptr_t)p & 7)) { |
| 66 |
|
| # if defined(PO_HAVE_CRC32C_X86) |
| 67 |
|
| c = (uint32_t)_mm_crc32_u8(c, *p); |
| 68 |
|
| # else |
| 69 |
|
| c = __crc32cb(c, *p); |
| 70 |
|
| # endif |
| 71 |
|
| p++; len--; |
| 72 |
|
| } |
| 73 |
|
| while (len >= 8) { |
| 74 |
|
| po_u64 v; |
| 75 |
|
| memcpy(&v, p, 8); /* memcpy, not a cast: strict aliasing */ |
| 76 |
|
| # if defined(PO_HAVE_CRC32C_X86) |
| 77 |
|
| c = (uint32_t)_mm_crc32_u64((po_u64)c, v); |
| 78 |
|
| # else |
| 79 |
|
| c = __crc32cd(c, v); |
| 80 |
|
| # endif |
| 81 |
|
| p += 8; len -= 8; |
| 82 |
|
| } |
| 83 |
|
| while (len--) { |
| 84 |
|
| # if defined(PO_HAVE_CRC32C_X86) |
| 85 |
|
| c = (uint32_t)_mm_crc32_u8(c, *p); |
| 86 |
|
| # else |
| 87 |
|
| c = __crc32cb(c, *p); |
| 88 |
|
| # endif |
| 89 |
|
| p++; |
| 90 |
|
| } |
| 91 |
|
| return ~c; |
| 92 |
|
| #else |
| 93 |
32863 |
| return po_crc32c_table(crc, buf, len); |
| 94 |
|
| #endif |
| 95 |
|
| } |
| 96 |
|
| |
| 97 |
1 |
| static int po_crc32c_hardware(void) { |
| 98 |
|
| #if defined(PO_HAVE_CRC32C_X86) || defined(PO_HAVE_CRC32C_ARM) |
| 99 |
|
| return 1; |
| 100 |
|
| #else |
| 101 |
1 |
| return 0; |
| 102 |
|
| #endif |
| 103 |
|
| } |
| 104 |
|
| |
| 105 |
|
| #endif /* PO_CRC32C_H */ |