shm_generic.h

filter: Statementstmt 65.4% 13057 / 19960 Branchbran 42.7% 6302 / 14740 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 55.7% 19359 / 34700 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« xs/si32.xs ↑ Summary xs/is.xs »
1 /*
2 * shm_generic.h -- macro template for shared-memory hash maps. Define first:
3 * SHM_PREFIX, SHM_NODE_TYPE, SHM_VARIANT_ID (unique, checked in the header)
4 * SHM_KEY_IS_INT + SHM_KEY_INT_TYPE, or neither for string keys
5 * SHM_VAL_IS_STR, or SHM_VAL_INT_TYPE
6 * SHM_HAS_COUNTERS (optional, integer values): incr/decr, max/min, cas
7 */
8  
9 /* ---- Part 1: shared definitions (included once) ---- */
10  
11 #ifndef SHM_DEFS_H
12 #define SHM_DEFS_H
13  
14 #include <stddef.h>
15 #include <stdint.h>
16 #include <stdbool.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <string.h>
20 #include <unistd.h>
21 #include <fcntl.h>
22 #include <time.h>
23 #include <sys/mman.h>
24 #include <sys/stat.h>
25  
26 #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
27 #error "shm_generic.h: inline string packing requires little-endian architecture"
28 #endif
29 #include <sys/file.h>
30 #include <pthread.h>
31 #include <sys/syscall.h>
32 #include <limits.h>
33 #include <signal.h>
34 #include <errno.h>
35 /* Optional (minimal containers lack it): both values used are kernel ABI. */
36 #ifdef SHM_HAVE_LINUX_FUTEX_H
37 #include <linux/futex.h>
38 #endif
39 #ifndef FUTEX_WAIT
40 #define FUTEX_WAIT 0
41 #endif
42 #ifndef FUTEX_WAKE
43 #define FUTEX_WAKE 1
44 #endif
45  
46 /* glibc < 2.27 lacks memfd_create and the sealing constants; values are kernel ABI. */
47 #ifndef MFD_CLOEXEC
48 #define MFD_CLOEXEC 0x0001U
49 #endif
50 #ifndef MFD_ALLOW_SEALING
51 #define MFD_ALLOW_SEALING 0x0002U
52 #endif
53 #ifndef F_ADD_SEALS
54 #define F_ADD_SEALS 1033 /* F_LINUX_SPECIFIC_BASE + 9 */
55 #endif
56 #ifndef F_SEAL_SHRINK
57 #define F_SEAL_SHRINK 0x0002
58 #endif
59 #ifndef F_SEAL_GROW
60 #define F_SEAL_GROW 0x0004
61 #endif
62 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
63 # if !__GLIBC_PREREQ(2, 27)
64 # define SHM_MEMFD_VIA_SYSCALL 1
65 # endif
66 #endif
67 #ifdef SHM_MEMFD_VIA_SYSCALL
68 # ifndef SYS_memfd_create
69 # error "memfd_create is unavailable: glibc < 2.27 and no SYS_memfd_create"
70 # endif
71 # define shm_memfd_create(name, flags) ((int)syscall(SYS_memfd_create, (name), (flags)))
72 #else
73 # define shm_memfd_create(name, flags) memfd_create((name), (flags))
74 #endif
75  
76 #ifdef __SSE2__
77 #include <emmintrin.h>
78 #endif
79  
80 #define XXH_INLINE_ALL
81 #include "xxhash.h"
82  
83 #define SHM_MAGIC 0x53484D31U /* "SHM1" */
84 #define SHM_VERSION 10U
85 /* new_sharded's ceiling: the power-of-two round-up must not overflow. */
86 #define SHM_MAX_SHARDS 4096U
87  
88 #ifndef SHM_READER_SLOTS
89 #define SHM_READER_SLOTS 1024 /* max concurrent reader processes for dead-process recovery */
90 #endif
91 /* Reader-slot occupancy bitmap: a bit is set on claim, cleared on clean release. */
92 #define SHM_OCC_WORDS (((SHM_READER_SLOTS) + 63) / 64)
93 #define SHM_OCC_BYTES ((uint64_t)SHM_OCC_WORDS * 8)
94 #define SHM_INITIAL_CAP 16
95 #define SHM_MAX_STR_LEN 0x3FFFFFFFU /* ~1GB, bit 30 reserved for inline flag */
96 #define SHM_LRU_NONE UINT32_MAX
97  
98 /* UINT32_MAX = use default TTL; 0 = no TTL; other = per-key TTL */
99 #define SHM_TTL_USE_DEFAULT UINT32_MAX
100  
101 /* Hash half that picks the shard; fixed per set, or keys would change shards. */
102 #define SHM_ROUTING_LEGACY 0 /* low half, the half the in-shard probe uses */
103 #define SHM_ROUTING_SPLIT 1 /* high half, which the slot index ignores */
104  
105 #define SHM_IS_EXPIRED(h, i, now) \
106 ((h)->expires_at && (h)->expires_at[(i)] && \
107 (now) >= (h)->expires_at[(i)])
108  
109 53479 static inline uint32_t shm_now(void) {
110 struct timespec ts;
111 53479 clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
112 53479 return (uint32_t)ts.tv_sec;
113 }
114  
115 45963 static inline uint32_t shm_expiry_ts(uint32_t ttl) {
116 45963 uint64_t sum = (uint64_t)shm_now() + ttl;
117 45963 ▶ return (sum > UINT32_MAX) ? UINT32_MAX : (uint32_t)sum;
Branch: 1 branch
BranchTrueFalseTotal
return (sum > UINT32_MAX) ? UINT32_MAX : (uint32_t)sum; 45960 3 45963
118 }
119  
120 #define SHM_EMPTY 0
121 #define SHM_TOMBSTONE 1
122 #define SHM_TAG_MIN 2 /* state values 2-255 = LIVE with hash tag */
123 #define SHM_IS_LIVE(st) ((st) >= SHM_TAG_MIN)
124 /* During PLACE (CLEAN left no tombstones): a live entry not yet re-placed. */
125 #define SHM_MOVE SHM_TOMBSTONE
126  
127 /* rz_phase, each restartable. CLEAN empties tombstones, MARK turns every live
128 * entry into SHM_MOVE, PLACE re-inserts them; CLEAR is a clear() under way. */
129 #define SHM_RZ_NONE 0
130 #define SHM_RZ_CLEAN 1
131 #define SHM_RZ_MARK 2
132 #define SHM_RZ_PLACE 3
133 #define SHM_RZ_CLEAR 4
134  
135 /* PLACE on a string-key table of at least SHM_RZ_PREFETCH_MIN slots prefetches
136 * the arena key of the entry this far ahead. */
137 #define SHM_RZ_PREFETCH_AHEAD 24
138 #define SHM_RZ_PREFETCH_MIN (1u << 16)
139  
140 10042 static inline int shm_rz_cap_ok(uint32_t cap, uint32_t max_mask) {
141 10042 ▶ return cap && (cap & (cap - 1)) == 0 && cap - 1 <= max_mask;
Branch: 3 branches
BranchTrueFalseTotal
return cap && (cap & (cap - 1)) == 0 && cap - 1 <= max_mask; 10042 0 10042
return cap && (cap & (cap - 1)) == 0 && cap - 1 <= max_mask; 10042 0 10042
return cap && (cap & (cap - 1)) == 0 && cap - 1 <= max_mask; 10042 0 10042
142 }
143  
144 2575 static inline uint32_t shm_count_live(const uint8_t *states, uint32_t cap) {
145 2575 uint32_t live = 0, i = 0;
146 #ifdef __SSE2__
147 5150 const __m128i one = _mm_set1_epi8(1), tag_min = _mm_set1_epi8(SHM_TAG_MIN);
148 2575 __m128i sum = _mm_setzero_si128();
149 69175 ▶ for (; i + 16 <= cap; i += 16) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i + 16 <= cap; i += 16) { 66600 2575 69175
150 133200 __m128i c = _mm_loadu_si128((const __m128i *)(states + i));
151 133200 __m128i is_live = _mm_cmpeq_epi8(_mm_min_epu8(c, tag_min), tag_min);
152 266400 sum = _mm_add_epi64(sum, _mm_sad_epu8(_mm_and_si128(is_live, one), _mm_setzero_si128()));
153 }
154 2575 live = (uint32_t)_mm_cvtsi128_si32(sum) + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(sum, 8));
155 #endif
156 2575 ▶ for (; i < cap; i++) live += SHM_IS_LIVE(states[i]);
Branch: 1 branch
BranchTrueFalseTotal
for (; i < cap; i++) live += SHM_IS_LIVE(states[i]); 0 2575 2575
157 2575 return live;
158 }
159  
160 2446 static inline void shm_states_clean(uint8_t *states, uint32_t cap) {
161 2446 uint32_t i = 0;
162 #ifdef __SSE2__
163 2446 const __m128i tomb = _mm_set1_epi8(SHM_TOMBSTONE);
164 60064 ▶ for (; i + 16 <= cap; i += 16) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i + 16 <= cap; i += 16) { 57618 2446 60064
165 115236 __m128i c = _mm_loadu_si128((const __m128i *)(states + i));
166 115236 _mm_storeu_si128((__m128i *)(states + i), _mm_andnot_si128(_mm_cmpeq_epi8(c, tomb), c));
167 }
168 #endif
169 2446 ▶ for (; i < cap; i++) states[i] = states[i] == SHM_TOMBSTONE ? SHM_EMPTY : states[i];
Branch: 2 branches
BranchTrueFalseTotal
for (; i < cap; i++) states[i] = states[i] == SHM_TOMBSTONE ? SHM_EMPTY : states[i]; 0 0 0
for (; i < cap; i++) states[i] = states[i] == SHM_TOMBSTONE ? SHM_EMPTY : states[i]; 0 2446 2446
170 2446 }
171  
172 2446 static inline void shm_states_mark(uint8_t *states, uint32_t cap) {
173 2446 uint32_t i = 0;
174 #ifdef __SSE2__
175 2446 const __m128i move = _mm_set1_epi8(SHM_MOVE);
176 60064 ▶ for (; i + 16 <= cap; i += 16) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i + 16 <= cap; i += 16) { 57618 2446 60064
177 115236 __m128i c = _mm_loadu_si128((const __m128i *)(states + i));
178 172854 _mm_storeu_si128((__m128i *)(states + i),
179 _mm_andnot_si128(_mm_cmpeq_epi8(c, _mm_setzero_si128()), move));
180 }
181 #endif
182 2446 ▶ for (; i < cap; i++) states[i] = states[i] == SHM_EMPTY ? SHM_EMPTY : SHM_MOVE;
Branch: 1 branch
BranchTrueFalseTotal
for (; i < cap; i++) states[i] = states[i] == SHM_EMPTY ? SHM_EMPTY : SHM_MOVE; 0 2446 2446
183 2446 }
184  
185 #define SHM_MAKE_TAG(hash) ((uint8_t)(((hash) >> 24) % 254 + SHM_TAG_MIN))
186 /* Tag-based probe filtering needs TOMBSTONE < TAG_MIN. */
187 typedef char shm_tag_invariant_check[(SHM_TOMBSTONE < SHM_TAG_MIN) ? 1 : -1];
188  
189 22779 static inline int shm_find_next_live(const uint8_t *states, uint32_t cap, uint32_t *pos) {
190 22779 uint32_t i = *pos;
191 #ifdef __SSE2__
192 22779 __m128i ones = _mm_set1_epi8(1);
193 22779 uint32_t align_end = (i + 15) & ~(uint32_t)15;
194 22779 ▶ if (align_end > cap) align_end = cap;
Branch: 1 branch
BranchTrueFalseTotal
if (align_end > cap) align_end = cap; 0 22779 22779
195 46522 ▶ for (; i < align_end; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i < align_end; i++) { 43002 3520 46522
196 43002 ▶ if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; }
Branch: 1 branch
BranchTrueFalseTotal
if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; } 19259 23743 43002
197 }
198 /* saturating byte - 1 is nonzero iff byte >= SHM_TAG_MIN */
199 19535 ▶ for (; i + 16 <= cap; i += 16) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i + 16 <= cap; i += 16) { 19476 59 19535
200 38952 __m128i chunk = _mm_loadu_si128((const __m128i *)(states + i));
201 19476 __m128i sub = _mm_subs_epu8(chunk, ones);
202 38952 int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(sub, _mm_setzero_si128()));
203 19476 mask = ~mask & 0xFFFF;
204 19476 ▶ if (mask) { *pos = i + __builtin_ctz(mask); return 1; }
Branch: 1 branch
BranchTrueFalseTotal
if (mask) { *pos = i + __builtin_ctz(mask); return 1; } 3461 16015 19476
205 }
206 #endif
207 59 ▶ for (; i < cap; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (; i < cap; i++) { 0 59 59
208 0 ▶ if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; }
Branch: 1 branch
BranchTrueFalseTotal
if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; } 0 0 0
209 }
210 59 return 0;
211 }
212  
213 /* 16 contiguous state bytes: the caller handles table wrap. */
214 #ifdef __SSE2__
215 377812 static inline void shm_probe_group(const uint8_t *states, uint32_t pos,
216 uint8_t tag, uint16_t *match_mask,
217 uint16_t *empty_mask) {
218 377812 __m128i group = _mm_loadu_si128((const __m128i *)(states + pos));
219 755624 __m128i tag_v = _mm_set1_epi8((char)tag);
220 377812 __m128i zero_v = _mm_setzero_si128();
221 755624 *match_mask = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(group, tag_v));
222 377812 *empty_mask = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(group, zero_v));
223 377812 }
224 #endif
225  
226 #define SHM_ARENA_NUM_CLASSES 16 /* 2^4..2^19 = 16..524288 */
227 #define SHM_EVICT_SEARCH 32 /* oldest entries searched for a block of the request's class */
228 #define SHM_LRU_SPARE_MAX 64 /* second chances one eviction grants before it takes an entry anyway */
229 #define SHM_ARENA_MIN_ALLOC 16
230 /* A free-list walk longer than this is following a corrupted cycle. */
231 #define SHM_ARENA_FREELIST_MAX(cap) ((uint32_t)((cap) / SHM_ARENA_MIN_ALLOC) + 1)
232  
233 /* ---- UTF-8 and inline-string flag packing ----
234 * key_len / val_len (uint32_t): bit 31 = UTF-8, bit 30 = inline, bits 0-29 =
235 * length. Inline: _off plus bits 0-23 of _len hold up to 7 bytes of data,
236 * bits 24-26 its length. Arena: _off is the offset, bits 0-29 the length. */
237  
238 #define SHM_UTF8_FLAG ((uint32_t)0x80000000U)
239 #define SHM_INLINE_FLAG ((uint32_t)0x40000000U)
240 #define SHM_LEN_MASK ((uint32_t)0x3FFFFFFFU)
241 #define SHM_INLINE_MAX 7
242  
243 #define SHM_PACK_LEN(len, utf8) ((uint32_t)(len) | ((utf8) ? SHM_UTF8_FLAG : 0))
244 #define SHM_UNPACK_LEN(packed) ((uint32_t)((packed) & SHM_LEN_MASK))
245 #define SHM_UNPACK_UTF8(packed) (((packed) & SHM_UTF8_FLAG) != 0)
246 #define SHM_IS_INLINE(packed) (((packed) & SHM_INLINE_FLAG) != 0)
247  
248 #define SHM_STR_LEN(packed) \
249 (SHM_IS_INLINE(packed) ? shm_inline_len(packed) : SHM_UNPACK_LEN(packed))
250  
251 162686 static inline void shm_inline_pack(uint32_t *off, uint32_t *len_field,
252 const char *str, uint32_t slen, bool utf8) {
253 162686 uint32_t lf = SHM_INLINE_FLAG | ((uint32_t)slen << 24);
254 162686 ▶ if (utf8) lf |= SHM_UTF8_FLAG;
Branch: 1 branch
BranchTrueFalseTotal
if (utf8) lf |= SHM_UTF8_FLAG; 1216 161470 162686
255 162686 uint32_t o = 0;
256 162686 memcpy(&o, str, slen > 4 ? 4 : slen);
257 162686 ▶ if (slen > 4) {
Branch: 1 branch
BranchTrueFalseTotal
if (slen > 4) { 138363 24323 162686
258 138363 uint32_t rest = 0;
259 138363 memcpy(&rest, str + 4, slen - 4);
260 138363 lf |= rest;
261 }
262 /* Pass through the inline-empty state, which shm_str_free ignores, so a killed
263 * writer leaves no (off,len) to misfree. Atomic so the interim store stays. */
264 162686 __atomic_store_n(len_field, SHM_INLINE_FLAG, __ATOMIC_RELEASE);
265 162686 __atomic_store_n(off, o, __ATOMIC_RELEASE);
266 162686 __atomic_store_n(len_field, lf, __ATOMIC_RELEASE);
267 162686 }
268  
269 8742154 static inline uint32_t shm_inline_len(uint32_t len_field) {
270 8742154 return (len_field >> 24) & 0x7;
271 }
272  
273 4368550 static inline const char *shm_inline_read(uint32_t off, uint32_t len_field,
274 char *buf) {
275 4368550 uint32_t slen = shm_inline_len(len_field);
276 4368550 memcpy(buf, &off, slen > 4 ? 4 : slen);
277 4368550 ▶ if (slen > 4) {
Branch: 1 branch
BranchTrueFalseTotal
if (slen > 4) { 3578575 789975 4368550
278 3578575 uint32_t rest = len_field & 0x00FFFFFFU;
279 3578575 memcpy(buf + 4, &rest, slen - 4);
280 }
281 4368550 return buf;
282 }
283  
284 2618575 static inline const char *shm_str_ptr(uint32_t off, uint32_t len_field,
285 const char *arena, uint64_t arena_cap,
286 char *inline_buf,
287 uint32_t *out_len) {
288 2618575 ▶ if (SHM_IS_INLINE(len_field)) {
Branch: 1 branch
BranchTrueFalseTotal
if (SHM_IS_INLINE(len_field)) { 257276 2361299 2618575
289 257276 *out_len = shm_inline_len(len_field);
290 257276 return shm_inline_read(off, len_field, inline_buf);
291 }
292 2361299 uint32_t len = SHM_UNPACK_LEN(len_field);
293 2361299 ▶ if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + len > arena_cap) {
Branch: 2 branches
BranchTrueFalseTotal
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + len > arena_cap) { 2361297 2 2361299
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + len > arena_cap) { 0 2361297 2361297
294 /* off/len are peer-writable: never return an out-of-bounds pointer. */
295 2 *out_len = 0;
296 2 return inline_buf;
297 }
298 2361297 *out_len = len;
299 2361297 return arena + off;
300 }
301  
302 /* ---- Shared memory header (256 bytes, 4 cache lines, in mmap) ---- */
303  
304 /* No trailing semicolon: the call site adds one; a stray one is invalid C89. */
305 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
306 #define SHM_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)
307 #else
308 #define SHM_SA_JOIN2(a, b) a##b
309 #define SHM_SA_JOIN(a, b) SHM_SA_JOIN2(a, b)
310 #define SHM_STATIC_ASSERT(cond, msg) \
311 typedef char SHM_SA_JOIN(shm_static_assert_, __LINE__)[(cond) ? 1 : -1]
312 #endif
313  
314 typedef struct {
315 /* ---- Cache line 0 (0-63): immutable after create ---- */
316 uint32_t magic; /* 0 */
317 uint32_t version; /* 4 */
318 uint32_t variant_id; /* 8 */
319 uint32_t node_size; /* 12 */
320 uint32_t max_table_cap; /* 16 */
321 uint32_t table_cap; /* 20: changes on resize only */
322 uint32_t max_size; /* 24: LRU capacity, 0 = disabled */
323 uint32_t default_ttl; /* 28: TTL seconds, 0 = disabled */
324 uint64_t total_size; /* 32 */
325 uint64_t nodes_off; /* 40 */
326 uint64_t states_off; /* 48 */
327 uint64_t arena_off; /* 56 */
328  
329 /* ---- Cache line 1 (64-127): seqlock + read-path data ---- */
330 uint32_t seq; /* 64: seqlock counter, odd = writer active */
331 uint32_t drain_seq; /* 68: futex a releasing reader bumps to wake a draining writer */
332 uint64_t arena_cap; /* 72: immutable */
333 uint64_t reader_slots_off;/* 80 */
334 uint32_t slotless_rdepth; /* 88: read-locks held by readers with no reader-slot */
335 uint32_t arena_large_free;/* 92: head of the >2^19 large-block free list (0 = empty) */
336 /* 96-127 were carved from the pad: a file written before them reads 0. */
337 uint8_t sealed; /* 96: 1 = frozen (read-only; lock-free reads) */
338 uint8_t routing; /* 97: SHM_ROUTING_* */
339 uint8_t shard_log2; /* 98: log2(shards)+1 once a set's creator stamped it, else 0 */
340 uint8_t rz_phase; /* 99: SHM_RZ_*, a resize in progress */
341 uint32_t pop_cursor; /* 100: next slot a non-LRU pop or drain examines (wraps) */
342 uint32_t shift_cursor; /* 104: the slot above the next one a non-LRU shift examines,
343 0 meaning the top of the table */
344 /* The resize record, meaningful while rz_phase is set. */
345 uint8_t rz_old_log2; /* 108 */
346 uint8_t rz_new_log2; /* 109 */
347 uint16_t rz_rsv; /* 110 */
348 uint64_t rz_move; /* 112: the move in flight, src | dst << 32 */
349 uint32_t rz_cursor; /* 120: no slot below it holds an entry still to move */
350 uint32_t rz_seq; /* 124: the odd seq of the write section that recorded it */
351  
352 /* ---- Cache line 2 (128-191): rwlock + write-hot fields ---- */
353 uint32_t wlock; /* 128: 0 or SHM_RWLOCK_WR(pid); readers count in their slots */
354 uint32_t rwait; /* 132: waiters parked on wlock; may over-count */
355 uint32_t size; /* 136 */
356 uint32_t tombstones; /* 140 */
357 uint32_t lru_head; /* 144: MRU slot index */
358 uint32_t lru_tail; /* 148: LRU slot index */
359 uint32_t flush_cursor; /* 152: partial flush_expired scan cursor */
360 uint32_t table_gen; /* 156: incremented on every resize */
361 uint64_t arena_bump; /* 160 */
362 uint64_t stat_evictions; /* 168 */
363 uint64_t stat_expired; /* 176 */
364 uint32_t stat_recoveries; /* 184 */
365 uint32_t lru_skip; /* 188: promotion skip mask (power-of-2 minus 1, 0=strict LRU) */
366  
367 /* ---- Cache line 3 (192-255): arena free lists ---- */
368 uint32_t arena_free[SHM_ARENA_NUM_CLASSES]; /* 192-255 */
369 } ShmHeader;
370  
371 SHM_STATIC_ASSERT(sizeof(ShmHeader) == 256, "ShmHeader must be exactly 256 bytes (4 cache lines)");
372 /* shm_create_sharded CASes shard_log2 across processes; libatomic's fallback
373 * lock is per process. C11 only: GCC before 5 cannot fold this builtin. */
374 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
375 SHM_STATIC_ASSERT(__atomic_always_lock_free(sizeof(((ShmHeader *)0)->shard_log2), 0),
376 "cross-process CAS on shard_log2 needs a lock-free byte atomic");
377 #endif
378  
379 /* A reader's whole share of the lock is its slot's rdepth, so clearing a dead
380 * reader's pid releases it. _rsv* keep the on-disk 16-byte slot size. */
381 typedef struct {
382 uint32_t pid; /* 0 = unclaimed */
383 uint32_t rdepth; /* read-locks this process holds */
384 uint32_t _rsv1;
385 uint32_t _rsv2;
386 } ShmReaderSlot;
387  
388 /* Consecutive pids start their slot search a cache line apart. */
389 #define SHM_READER_SLOT_STRIDE (64 / sizeof(ShmReaderSlot))
390  
391 typedef struct ShmHandle_s {
392 ShmHeader *hdr;
393 void *nodes;
394 uint8_t *states;
395 char *arena;
396 uint32_t *lru_prev;
397 uint32_t *lru_next;
398 uint8_t *lru_accessed; /* clock second-chance bits */
399 uint32_t *expires_at;
400 ShmReaderSlot *reader_slots;
401 uint64_t *occ;
402 uint32_t my_slot_idx; /* UINT32_MAX if all slots taken (no recovery for this handle) */
403 uint32_t cached_pid; /* getpid() cached at last slot claim */
404 uint32_t cached_fork_gen; /* shm_fork_gen at last slot claim; a mismatch reclaims */
405 uint32_t slotless_held; /* rwlock read-locks held with no reader-slot */
406 uint32_t occ_sweep_in; /* write locks left before the next occupancy sweep */
407 uint32_t slotless_retry_in; /* locks left before a slotless handle rescans */
408 uint32_t compact_backoff; /* refusals left before a dense arena is compacted again */
409 uint32_t compact_period; /* how long that wait currently is; grows while it does not pay */
410 uint32_t compact_size; /* the map's size at the last slide that did not pay */
411 uint32_t arena_need; /* block the refused store wanted; what a slide has to beat to count */
412 uint8_t arena_failed; /* the arena refused a store: compact before the next store */
413 uint8_t reclaim_wait; /* seconds the last expired flush that freed little backs off */
414 uint32_t reclaim_at; /* shm_now() before which an unforced expired flush is skipped */
415 uint32_t reclaim_cap; /* table_cap the backoff was measured at */
416 uint32_t reclaim_sec; /* shm_now() + 1 of the last flush; nothing expires twice in a second */
417 uint32_t lock_depth; /* RDLOCK_GUARD open on this handle */
418 uint64_t map_id; /* see shm_map_id_of */
419 dev_t file_dev; /* the backing file as opened: unlink removes the path */
420 ino_t file_ino; /* only while it still names this file */
421 int wrlock_ent; /* this map's shm_wrlock_maps entry, or SHM_WRLOCK_UNRESOLVED */
422 uint8_t pending_close; /* DESTROY arrived while lock_depth > 0; free at depth 0 */
423 int readonly; /* 1 = frozen PROT_READ view: never write the mapping
424 (no rdepth, no clock bit) */
425 size_t mmap_size;
426 uint32_t max_mask; /* max_table_cap - 1, for seqlock bounds clamping */
427 uint32_t iter_pos;
428 char *copy_buf;
429 uint32_t copy_buf_size;
430 uint32_t iterating; /* active iterator count (each + cursors) */
431 uint32_t iter_gen; /* table_gen snapshot for each() */
432 uint8_t iter_active;
433 uint8_t deferred; /* shrink/compact deferred while iterating */
434 uint8_t flush_done; /* shard of a set: its partial-flush cycle ended this round */
435 uint8_t flush_freed; /* this handle's partial flushes freed entries this cycle */
436 uint32_t flush_seen; /* flush_cursor as this handle's last partial flush left it */
437 uint32_t flush_gen; /* and the table_gen it was left at */
438 char *path;
439 int backing_fd; /* memfd to close on destroy, else -1 */
440 struct ShmHandle_s **shard_handles; /* non-NULL: a sharded map's dispatcher */
441 uint32_t num_shards;
442 uint32_t shard_mask;
443 uint8_t route_high; /* shard on the high hash half (see SHM_ROUTING_*) */
444 uint32_t shard_iter;
445 uint32_t shard_rr; /* pop/shift/drain start; apart from shard_iter so a
446 drain inside each() skips no shard */
447 void (*rz_resume)(struct ShmHandle_s *h); /* the variant's rz_run */
448 sigset_t sig_old; /* the signal mask to restore while sig_held */
449 uint8_t sig_held; /* signals are blocked until this write lock is released */
450 uint64_t sect_bytes; /* string bytes hashed, stored, compared or copied out
451 under this write lock */
452 volatile int *sig_count; /* the interpreter's PL_sig_pending */
453 int sig_seen; /* as this write lock found it */
454 uint64_t reclaim_kept; /* expired slot the last flush kept, table_gen << 32 | slot;
455 UINT64_MAX if none */
456 } ShmHandle;
457  
458 /* ---- Signals held across a long pass ----
459 * Perl croaks from its C signal handler once 120 signals are pending, stranding
460 * a held write lock. A write blocks signals once one arrives (shm_sig_check),
461 * or before a step over this many slots or SHM_SIG_HOLD_BYTES of strings; never
462 * before taking the lock, or a waiter cannot be killed. */
463 #define SHM_SIG_HOLD_MIN 4096u
464 #define SHM_SIG_HOLD_BYTES ((uint64_t)SHM_SIG_HOLD_MIN << 8)
465 #define SHM_SIG_CHECK_SLOTS 255u /* masks: a slot scan checks every 256th slot, */
466 #define SHM_SIG_CHECK_ENTRIES 15u /* a batch every 16th entry */
467 static volatile int shm_sig_none; /* sig_count of a handle no interpreter watches */
468 290 static __attribute__((noinline, cold)) void shm_sig_block(ShmHandle *h) {
469 sigset_t all;
470 290 sigfillset(&all);
471 290 sigdelset(&all, SIGSEGV);
472 290 sigdelset(&all, SIGBUS);
473 290 sigdelset(&all, SIGFPE);
474 290 sigdelset(&all, SIGILL);
475 290 ▶ if (pthread_sigmask(SIG_BLOCK, &all, &h->sig_old) == 0) h->sig_held = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (pthread_sigmask(SIG_BLOCK, &all, &h->sig_old) == 0) h->sig_held = 1; 290 0 290
476 290 }
477 4299 static inline void shm_sig_hold(ShmHandle *h, uint32_t work) {
478 4299 ▶ if (work >= SHM_SIG_HOLD_MIN && !h->sig_held) shm_sig_block(h);
Branch: 2 branches
BranchTrueFalseTotal
if (work >= SHM_SIG_HOLD_MIN && !h->sig_held) shm_sig_block(h); 299 4000 4299
if (work >= SHM_SIG_HOLD_MIN && !h->sig_held) shm_sig_block(h); 290 9 299
479 4299 }
480 static inline __attribute__((always_inline)) void shm_sig_check(ShmHandle *h) {
481 51353 ▶ if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h);
Branch: 172 branches
BranchTrueFalseTotal
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 2115 2115
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5 5
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 831 831
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 561 561
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 769 769
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 3368 3368
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 7 7
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 6 6
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 162 162
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5 5
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 1000 1000
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 6 6
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5 5
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 1 1
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 6 6
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5 5
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 1 1
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 20276 20276
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5000 5000
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5000 5000
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5000 5000
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 2075 2075
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 15 15
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 24 24
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 61 61
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 5 5
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 4617 4617
if (__builtin_expect(*h->sig_count != h->sig_seen, 0) && !h->sig_held) shm_sig_block(h); 0 0 0
482 51353 }
483 131246 static inline void shm_sig_check_every(ShmHandle *h, uint32_t done, uint32_t mask) {
484 131246 ▶ if (!(done & mask)) shm_sig_check(h);
Branch: 1 branch
BranchTrueFalseTotal
if (!(done & mask)) shm_sig_check(h); 4617 126629 131246
485 131246 }
486 557067 static inline void shm_sig_hold_bytes(ShmHandle *h, uint32_t bytes) {
487 557067 h->sect_bytes += bytes;
488 557067 ▶ if (h->sect_bytes >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN);
Branch: 1 branch
BranchTrueFalseTotal
if (h->sect_bytes >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN); 25 557042 557067
489 557067 }
490 427163 static inline void shm_sig_release(ShmHandle *h) {
491 427163 ▶ if (__builtin_expect(h->sig_held, 0)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(h->sig_held, 0)) { 290 426873 427163
492 290 h->sig_held = 0;
493 290 pthread_sigmask(SIG_SETMASK, &h->sig_old, NULL);
494 }
495 427163 }
496 44176 static inline void shm_set_sig_count(ShmHandle *h, volatile int *count) {
497 44176 h->sig_count = count;
498 46245 ▶ for (uint32_t i = 0; h->shard_handles && i < h->num_shards; i++)
Branch: 2 branches
BranchTrueFalseTotal
for (uint32_t i = 0; h->shard_handles && i < h->num_shards; i++) 2291 43954 46245
for (uint32_t i = 0; h->shard_handles && i < h->num_shards; i++) 2069 222 2291
499 2069 h->shard_handles[i]->sig_count = count;
500 44176 }
501  
502 typedef struct {
503 ShmHandle *handle; /* for sharded, the dispatcher */
504 ShmHandle *current; /* current shard handle (== handle for single maps) */
505 SV *owner; /* keeps the mmap/handle alive while the cursor lives */
506 uint32_t iter_pos;
507 uint32_t gen; /* table_gen snapshot -- reset on mismatch */
508 uint32_t shard_idx;
509 uint32_t shard_count; /* 1 for single maps */
510 char *copy_buf;
511 uint32_t copy_buf_size;
512 } ShmCursor;
513  
514 3022 static inline int shm_grow_buf(char **buf, uint32_t *cap, uint32_t needed) {
515 3022 ▶ if (needed == 0) needed = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (needed == 0) needed = 1; 1 3021 3022
516 3022 ▶ if (needed <= *cap) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (needed <= *cap) return 1; 2941 81 3022
517 81 ▶ uint32_t ns = *cap ? *cap : 64;
Branch: 1 branch
BranchTrueFalseTotal
uint32_t ns = *cap ? *cap : 64; 53 28 81
518 284 ▶ while (ns < needed) {
Branch: 1 branch
BranchTrueFalseTotal
while (ns < needed) { 203 81 284
519 203 uint32_t next = ns * 2;
520 203 ▶ if (next <= ns) { ns = needed; break; }
Branch: 1 branch
BranchTrueFalseTotal
if (next <= ns) { ns = needed; break; } 0 203 203
521 203 ns = next;
522 }
523 81 char *nb = (char *)realloc(*buf, ns);
524 81 ▶ if (!nb) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!nb) return 0; 0 81 81
525 81 *buf = nb;
526 81 *cap = ns;
527 81 return 1;
528 }
529  
530 1942 static inline int shm_ensure_copy_buf(ShmHandle *h, uint32_t needed) {
531 1942 return shm_grow_buf(&h->copy_buf, &h->copy_buf_size, needed);
532 }
533  
534 1020 static inline int shm_cursor_ensure_copy_buf(ShmCursor *c, uint32_t needed) {
535 1020 return shm_grow_buf(&c->copy_buf, &c->copy_buf_size, needed);
536 }
537  
538 5679416 static inline uint64_t shm_hash_int64(int64_t key) {
539 5679416 return XXH3_64bits(&key, sizeof(key));
540 }
541  
542 8280220 static inline uint64_t shm_hash_string(const char *data, uint32_t len) {
543 8280220 return XXH3_64bits(data, (size_t)len);
544 }
545  
546 /* ---- Futex-based read-write lock ---- */
547  
548 #define SHM_RWLOCK_SPIN_LIMIT 32
549 #define SHM_LOCK_TIMEOUT_SEC 2 /* FUTEX_WAIT timeout for stale lock detection */
550  
551 9106776 static inline void shm_rwlock_spin_pause(void) {
552 #if defined(__x86_64__) || defined(__i386__)
553 9106776 __asm__ volatile("pause" ::: "memory");
554 #elif defined(__aarch64__)
555 __asm__ volatile("yield" ::: "memory");
556 #else
557 __asm__ volatile("" ::: "memory");
558 #endif
559 9106776 }
560  
561 #define SHM_RWLOCK_WRITER_BIT 0x80000000U
562 #define SHM_RWLOCK_PID_MASK 0x7FFFFFFFU
563 #define SHM_RWLOCK_WR(pid) (SHM_RWLOCK_WRITER_BIT | ((uint32_t)(pid) & SHM_RWLOCK_PID_MASK))
564  
565 /* kill(pid,0) succeeds for an unreaped zombie. An unreadable /proc says "not a
566 * zombie", never force-recovering a possibly-live holder. */
567 716 static inline int shm_pid_is_zombie(uint32_t pid) {
568 char path[32], buf[256];
569 716 snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
570 716 int fd = open(path, O_RDONLY | O_CLOEXEC);
571 716 ▶ if (fd < 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return 0; 0 716 716
572 716 ssize_t n = read(fd, buf, sizeof(buf) - 1);
573 716 close(fd);
574 716 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 716 716
575 716 buf[n] = '\0';
576 /* "pid (comm) state ..."; comm may contain ')' */
577 716 char *rp = strrchr(buf, ')');
578 716 ▶ if (!rp || rp + 2 >= buf + n) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (!rp || rp + 2 >= buf + n) return 0; 716 0 716
if (!rp || rp + 2 >= buf + n) return 0; 0 716 716
579 716 ▶ return rp[1] == ' ' && rp[2] == 'Z';
Branch: 2 branches
BranchTrueFalseTotal
return rp[1] == ' ' && rp[2] == 'Z'; 716 0 716
return rp[1] == ' ' && rp[2] == 'Z'; 0 716 716
580 }
581  
582 /* Per map, write locks this process (any handle or thread) holds or is
583 * acquiring: our own pid in its lock word is a recycled ghost only at 0.
584 * Entries are never freed (that races an acquirer); a process past
585 * SHM_WRLOCK_MAPS maps reads every own-pid lock as held. */
586 #define SHM_WRLOCK_MAPS 256
587 #define SHM_WRLOCK_NO_ENTRY (-1)
588 #define SHM_WRLOCK_UNRESOLVED (-2) /* a handle that has not write-locked yet */
589 /* Low 32 bits count holds; the generation above lets a repairer see one come and go. */
590 #define SHM_WRLOCK_HOLD ((1ULL << 32) | 1)
591 typedef struct { uint64_t id; uint64_t holds; } ShmWrlockMap;
592 static ShmWrlockMap shm_wrlock_maps[SHM_WRLOCK_MAPS];
593 /* Holds through handles with no entry; non-zero makes every own-pid lock read as held. */
594 static uint32_t shm_wrlock_untracked = 0;
595 /* An anonymous map (shared only by inheritance) uses its address. A collision
596 * costs recovery on both maps, never correctness. */
597 46127 static inline uint64_t shm_map_id_of(uint64_t dev, uint64_t ino, const void *base) {
598 92059 uint64_t id = ino ? (dev * 0x9E3779B97F4A7C15ULL) ^ (ino + 0x165667B19E3779F9ULL)
599 46127 ▶ : (uint64_t)(uintptr_t)base;
Branch: 1 branch
BranchTrueFalseTotal
: (uint64_t)(uintptr_t)base; 45932 195 46127
600 46127 ▶ return id ? id : 1; /* 0 marks a free slot */
Branch: 1 branch
BranchTrueFalseTotal
return id ? id : 1; /* 0 marks a free slot */ 46127 0 46127
601 }
602  
603 2261 static inline int shm_wrlock_entry(uint64_t id) {
604 304094 ▶ for (int i = 0; i < SHM_WRLOCK_MAPS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int i = 0; i < SHM_WRLOCK_MAPS; i++) { 303161 933 304094
605 303161 ▶ if (__atomic_load_n(&shm_wrlock_maps[i].id, __ATOMIC_SEQ_CST) == id)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&shm_wrlock_maps[i].id, __ATOMIC_SEQ_CST) == id) 134 303027 303161
606 1328 return i;
607 303027 uint64_t expected = 0;
608 303027 ▶ if (__atomic_compare_exchange_n(&shm_wrlock_maps[i].id, &expected, id, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&shm_wrlock_maps[i].id, &expected, id, 0, 1194 301833 303027
609 __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
610 1194 return i;
611 301833 ▶ if (expected == id) return i;
Branch: 1 branch
BranchTrueFalseTotal
if (expected == id) return i; 0 301833 301833
612 }
613 933 return SHM_WRLOCK_NO_ENTRY;
614 }
615  
616 /* Resolved at the first write lock or repair, so read-only use spends no entry.
617 * A handle is one thread's. */
618 427163 static inline int shm_wrlock_ent_of(ShmHandle *h) {
619 427163 ▶ if (h->wrlock_ent == SHM_WRLOCK_UNRESOLVED)
Branch: 1 branch
BranchTrueFalseTotal
if (h->wrlock_ent == SHM_WRLOCK_UNRESOLVED) 2261 424902 427163
620 2261 h->wrlock_ent = shm_wrlock_entry(h->map_id);
621 427163 return h->wrlock_ent;
622 }
623  
624 /* Raise before the lock word can carry our pid and drop after it is clear, or
625 * another thread of ours reads our live lock as a ghost. */
626 427165 static inline void shm_wrlock_hold(int ent) {
627 427165 ▶ if (ent == SHM_WRLOCK_NO_ENTRY)
Branch: 1 branch
BranchTrueFalseTotal
if (ent == SHM_WRLOCK_NO_ENTRY) 15339 411826 427165
628 15339 __atomic_add_fetch(&shm_wrlock_untracked, 1, __ATOMIC_SEQ_CST);
629 else
630 411826 __atomic_add_fetch(&shm_wrlock_maps[ent].holds, SHM_WRLOCK_HOLD, __ATOMIC_SEQ_CST);
631 427165 }
632  
633 427165 static inline void shm_wrlock_unhold(int ent) {
634 427165 ▶ if (ent == SHM_WRLOCK_NO_ENTRY)
Branch: 1 branch
BranchTrueFalseTotal
if (ent == SHM_WRLOCK_NO_ENTRY) 15339 411826 427165
635 15339 __atomic_sub_fetch(&shm_wrlock_untracked, 1, __ATOMIC_SEQ_CST);
636 else
637 411826 __atomic_sub_fetch(&shm_wrlock_maps[ent].holds, 1, __ATOMIC_SEQ_CST);
638 427165 }
639  
640 1 static inline int shm_wrlock_busy(int ent) {
641 1 return (uint32_t)__atomic_load_n(&shm_wrlock_maps[ent].holds, __ATOMIC_SEQ_CST) != 0;
642 }
643  
644 /* Without allocating: a map with no entry was never write-locked by us. */
645 1 static inline int shm_wrlock_held_by_id(uint64_t id) {
646 1 ▶ if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0) 0 1 1
647 0 return 1;
648 11 ▶ for (int i = 0; i < SHM_WRLOCK_MAPS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int i = 0; i < SHM_WRLOCK_MAPS; i++) { 11 0 11
649 11 uint64_t slot = __atomic_load_n(&shm_wrlock_maps[i].id, __ATOMIC_SEQ_CST);
650 11 ▶ if (slot == id)
Branch: 1 branch
BranchTrueFalseTotal
if (slot == id) 1 10 11
651 1 return shm_wrlock_busy(i);
652 10 ▶ if (slot == 0) return 0; /* entries are appended, so a gap ends the search */
Branch: 1 branch
BranchTrueFalseTotal
if (slot == 0) return 0; /* entries are appended, so a gap ends the search */ 0 10 10
653 }
654 0 return 0;
655 }
656  
657 /* Acquiring callers drop their own hold before asking. */
658 0 static inline int shm_wrlock_ours(const ShmHandle *h) {
659 0 int ent = h->wrlock_ent;
660 0 ▶ if (ent == SHM_WRLOCK_UNRESOLVED) return shm_wrlock_held_by_id(h->map_id);
Branch: 1 branch
BranchTrueFalseTotal
if (ent == SHM_WRLOCK_UNRESOLVED) return shm_wrlock_held_by_id(h->map_id); 0 0 0
661 0 ▶ if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0) return 1; 0 0 0
662 0 ▶ if (ent == SHM_WRLOCK_NO_ENTRY) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (ent == SHM_WRLOCK_NO_ENTRY) return 1; 0 0 0
663 0 return shm_wrlock_busy(ent);
664 }
665  
666 /* Repairer's token, holds 0 -> 1: no thread of ours held or was acquiring the
667 * map, and other repairers are barred. Acquirers are not: while holds still
668 * equal *claimed, none has started since. */
669 0 static inline int shm_wrlock_claim_idle(int ent, uint64_t *claimed) {
670 uint64_t v;
671 0 ▶ if (ent == SHM_WRLOCK_NO_ENTRY) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (ent == SHM_WRLOCK_NO_ENTRY) return 0; 0 0 0
672 0 ▶ if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&shm_wrlock_untracked, __ATOMIC_SEQ_CST) != 0) return 0; 0 0 0
673 0 v = __atomic_load_n(&shm_wrlock_maps[ent].holds, __ATOMIC_SEQ_CST);
674 0 ▶ if ((uint32_t)v != 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if ((uint32_t)v != 0) return 0; 0 0 0
675 0 *claimed = v + 1;
676 0 return __atomic_compare_exchange_n(&shm_wrlock_maps[ent].holds, &v, v + 1, 0,
677 __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
678 }
679  
680 /* 1 if alive or unknown. A recycled pid reads alive until that process exits. */
681 718 static inline int shm_pid_alive(uint32_t pid) {
682 718 ▶ if (pid == 0) return 0; /* kill(0,0) would ask about our process group */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) return 0; /* kill(0,0) would ask about our process group */ 0 718 718
683 718 ▶ if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; 2 716 718
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; 2 0 2
684 716 return !shm_pid_is_zombie(pid);
685 }
686  
687 /* Our own pid in a lock word is a dead writer's, recycled to us, unless a
688 * thread of ours holds this map. */
689 607 static inline int shm_holder_blocks_us(const ShmHandle *h, uint32_t pid) {
690 607 ▶ if (pid == h->cached_pid && !shm_wrlock_ours(h))
Branch: 2 branches
BranchTrueFalseTotal
if (pid == h->cached_pid && !shm_wrlock_ours(h)) 0 607 607
if (pid == h->cached_pid && !shm_wrlock_ours(h)) 0 0 0
691 0 return 0;
692 607 return shm_pid_alive(pid);
693 }
694  
695 static void shm_lru_rebuild_if_corrupt(ShmHandle *h);
696 static void shm_recount_counters(ShmHandle *h);
697 static inline void shm_seqlock_write_begin(uint32_t *seq);
698 static inline void shm_seqlock_write_end(uint32_t *seq);
699  
700 /* Force-recover a stale write lock left by a dead process: CAS to our own pid,
701 * so a later recoverer can re-recover if we crash mid-repair. */
702 1 static inline void shm_recover_stale_lock(ShmHandle *h, uint32_t observed_wlock) {
703 1 ShmHeader *hdr = h->hdr;
704 1 uint32_t mypid = SHM_RWLOCK_WR((uint32_t)getpid());
705 1 int ent = shm_wrlock_ent_of(h);
706 1 ▶ if (observed_wlock == mypid) {
Branch: 1 branch
BranchTrueFalseTotal
if (observed_wlock == mypid) { 0 1 1
707 /* Own-pid ghost: the CAS below succeeds for every thread of ours, so the
708 * map's holds arbitrate instead. */
709 uint64_t claimed;
710 0 ▶ if (!shm_wrlock_claim_idle(ent, &claimed))
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_wrlock_claim_idle(ent, &claimed)) 0 0 0
711 0 return;
712 /* A hold since the claim moves the generation even if the word looks
713 * unchanged; the word is read first, so a hold that wrote it is seen. */
714 0 ▶ if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) != mypid ||
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) != mypid || 0 0 0
715 0 ▶ __atomic_load_n(&shm_wrlock_maps[ent].holds, __ATOMIC_SEQ_CST) != claimed) {
Branch: 1 branch
BranchTrueFalseTotal
__atomic_load_n(&shm_wrlock_maps[ent].holds, __ATOMIC_SEQ_CST) != claimed) { 0 0 0
716 0 shm_wrlock_unhold(ent);
717 0 return;
718 }
719 } else {
720 1 shm_wrlock_hold(ent); /* the repair is a hold */
721 }
722 /* SEQ_CST like the acquire path's CAS: it must publish the hold above to a
723 * thread of ours that reads our pid here. */
724 1 ▶ if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock, 0 1 1
725 mypid, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED)) {
726 0 shm_wrlock_unhold(ent);
727 0 return;
728 }
729 1 h->sig_seen = *h->sig_count;
730 1 shm_sig_hold(h, hdr->table_cap);
731 /* Even out the dead writer's seq first so readers proceed: the repair below
732 * touches nothing lock-free readers read. A resize does: finish it, seq odd. */
733 1 uint32_t seq = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
734 1 ▶ if (__atomic_load_n(&hdr->rz_phase, __ATOMIC_ACQUIRE) != SHM_RZ_NONE && h->rz_resume) {
Branch: 2 branches
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rz_phase, __ATOMIC_ACQUIRE) != SHM_RZ_NONE && h->rz_resume) { 0 1 1
if (__atomic_load_n(&hdr->rz_phase, __ATOMIC_ACQUIRE) != SHM_RZ_NONE && h->rz_resume) { 0 0 0
735 0 ▶ if (!(seq & 1))
Branch: 1 branch
BranchTrueFalseTotal
if (!(seq & 1)) 0 0 0
736 0 shm_seqlock_write_begin(&hdr->seq);
737 0 h->rz_resume(h);
738 0 shm_seqlock_write_end(&hdr->seq);
739 1 ▶ } else if (seq & 1)
Branch: 1 branch
BranchTrueFalseTotal
} else if (seq & 1) 0 1 1
740 0 __atomic_store_n(&hdr->seq, seq + 1, __ATOMIC_RELEASE);
741 1 shm_lru_rebuild_if_corrupt(h);
742 1 shm_recount_counters(h);
743 1 __atomic_add_fetch(&hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
744 1 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
745 1 shm_wrlock_unhold(ent); /* after the word is 0, never before */
746 1 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 0 1 1
747 0 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
748 1 shm_sig_release(h);
749 }
750  
751 static const struct timespec shm_lock_timeout = { SHM_LOCK_TIMEOUT_SEC, 0 };
752 /* A draining writer waits this long before probing the readers still holding. */
753 static const struct timespec shm_drain_probe_after = { 0, 20 * 1000 * 1000 };
754  
755 /* Bumped in the atfork child so a handle detects a fork without getpid(). */
756 static uint32_t shm_fork_gen = 1;
757 static pthread_once_t shm_atfork_once = PTHREAD_ONCE_INIT;
758 0 static void shm_on_fork_child(void) {
759 0 __atomic_add_fetch(&shm_fork_gen, 1, __ATOMIC_RELAXED);
760 /* The child holds no write lock: every published hold is a parent thread's. */
761 0 __atomic_store_n(&shm_wrlock_untracked, 0, __ATOMIC_RELAXED);
762 0 ▶ for (int i = 0; i < SHM_WRLOCK_MAPS; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (int i = 0; i < SHM_WRLOCK_MAPS; i++) 0 0 0
763 0 __atomic_store_n(&shm_wrlock_maps[i].holds, 0, __ATOMIC_RELAXED);
764 0 }
765 82 static void shm_atfork_init(void) {
766 82 pthread_atfork(NULL, NULL, shm_on_fork_child);
767 82 }
768  
769 /* SEQ_CST: the bit is set before the slot's rdepth can go non-zero, so a
770 * writer's bitmap scan never misses a committed reader. */
771 2295 static inline void shm_occ_set(ShmHandle *h, uint32_t s) {
772 2295 __atomic_fetch_or(&h->occ[s >> 6], (uint64_t)1 << (s & 63), __ATOMIC_SEQ_CST);
773 2295 }
774 2435 static inline void shm_occ_clear(ShmHandle *h, uint32_t s) {
775 2435 __atomic_fetch_and(&h->occ[s >> 6], ~((uint64_t)1 << (s & 63)), __ATOMIC_SEQ_CST);
776 2435 }
777  
778 /* Locks between a slotless handle's rescans for a free slot. */
779 #define SHM_SLOTLESS_RETRY_EVERY 256u
780 568305 static inline void shm_claim_reader_slot(ShmHandle *h) {
781 568305 uint32_t cur_gen = __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED);
782 568305 ▶ if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1))
Branch: 2 branches
BranchTrueFalseTotal
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 566010 2295 568305
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 566010 0 566010
783 566010 return;
784 /* A fork always rescans: the child owns none of the parent's slots. */
785 2295 ▶ if (h->my_slot_idx == UINT32_MAX && cur_gen == h->cached_fork_gen
Branch: 2 branches
BranchTrueFalseTotal
if (h->my_slot_idx == UINT32_MAX && cur_gen == h->cached_fork_gen 2295 0 2295
if (h->my_slot_idx == UINT32_MAX && cur_gen == h->cached_fork_gen 0 2295 2295
786 0 ▶ && h->slotless_retry_in) {
Branch: 1 branch
BranchTrueFalseTotal
&& h->slotless_retry_in) { 0 0 0
787 0 h->slotless_retry_in--;
788 0 return;
789 }
790 2295 pthread_once(&shm_atfork_once, shm_atfork_init);
791 /* Re-read after pthread_once: shm_on_fork_child may have bumped it. */
792 2295 cur_gen = __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED);
793 2295 uint32_t now_pid = (uint32_t)getpid();
794 2295 h->cached_pid = now_pid;
795 2295 ▶ if (cur_gen != h->cached_fork_gen) h->slotless_held = 0;
Branch: 1 branch
BranchTrueFalseTotal
if (cur_gen != h->cached_fork_gen) h->slotless_held = 0; 2295 0 2295
796 2295 h->cached_fork_gen = cur_gen;
797 2295 h->my_slot_idx = UINT32_MAX;
798 2295 uint32_t start = (uint32_t)(now_pid * SHM_READER_SLOT_STRIDE % SHM_READER_SLOTS);
799 2337 ▶ for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) { 2337 0 2337
800 2337 uint32_t s = (start + i) % SHM_READER_SLOTS;
801 2337 uint32_t expected = 0;
802 2337 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[s].pid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[s].pid, 2295 42 2337
803 &expected, now_pid, 0,
804 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
805 /* A drained dead predecessor may have left rdepth set. */
806 2295 __atomic_store_n(&h->reader_slots[s].rdepth, 0, __ATOMIC_RELAXED);
807 2295 shm_occ_set(h, s);
808 2295 h->my_slot_idx = s;
809 2295 return;
810 }
811 }
812 /* Pass 2: no free slot, so reclaim one whose owner is dead. Safe even at
813 * rdepth>0: a writer scan ignores rdepth when pid==0. */
814 0 ▶ for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) { 0 0 0
815 0 uint32_t dpid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
816 /* kill() alone, as in shm_occ_sweep: a zombie's slot frees once reaped. */
817 0 ▶ if (dpid == 0 || dpid == now_pid || kill((pid_t)dpid, 0) == 0 || errno != ESRCH) continue;
Branch: 4 branches
BranchTrueFalseTotal
if (dpid == 0 || dpid == now_pid || kill((pid_t)dpid, 0) == 0 || errno != ESRCH) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || kill((pid_t)dpid, 0) == 0 || errno != ESRCH) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || kill((pid_t)dpid, 0) == 0 || errno != ESRCH) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || kill((pid_t)dpid, 0) == 0 || errno != ESRCH) continue; 0 0 0
818 0 uint32_t expected = dpid;
819 0 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0, 0 0 0
820 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
821 0 __atomic_store_n(&h->reader_slots[i].rdepth, 0, __ATOMIC_RELAXED);
822 0 shm_occ_set(h, i);
823 0 h->my_slot_idx = i;
824 0 return;
825 }
826 }
827 /* Slotless: the lock still works, but this reader's death is not recoverable. */
828 0 h->slotless_retry_in = SHM_SLOTLESS_RETRY_EVERY;
829 }
830  
831 /* Dead readers need nothing here: the draining writer clears them. */
832 0 static inline void shm_recover_after_timeout(ShmHandle *h) {
833 0 ShmHeader *hdr = h->hdr;
834 /* ACQUIRE at every load feeding shm_holder_blocks_us: it pairs with the
835 * holder's CAS, so a thread of ours that sees our pid also sees the hold. */
836 0 uint32_t val = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
837 0 ▶ if (val >= SHM_RWLOCK_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= SHM_RWLOCK_WRITER_BIT) { 0 0 0
838 0 uint32_t pid = val & SHM_RWLOCK_PID_MASK;
839 0 ▶ if (!shm_holder_blocks_us(h, pid))
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_holder_blocks_us(h, pid)) 0 0 0
840 0 shm_recover_stale_lock(h, val);
841 }
842 0 }
843  
844 /* rwait may over-count (a waiter killed while parked), never under-count. */
845 606 static inline void shm_park(ShmHandle *h) {
846 606 __atomic_add_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
847 606 }
848 606 static inline void shm_unpark(ShmHandle *h) {
849 606 __atomic_sub_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
850 606 }
851  
852 /* SEQ_CST: Dekker pairs with the writer's wlock CAS and rdepth scan (inc() with
853 * rdlock's re-check, dec() with shm_reader_wake_drain's load). dec() peels
854 * slotless first, so a slot claimed mid-hold cannot take the decrement. */
855 141909 static inline void shm_rdepth_inc(ShmHandle *h) {
856 141909 ▶ if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->my_slot_idx != UINT32_MAX) { 141909 0 141909
857 141909 __atomic_add_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_SEQ_CST);
858 } else {
859 0 __atomic_add_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
860 0 h->slotless_held++;
861 }
862 141909 }
863 141909 static inline void shm_rdepth_dec(ShmHandle *h) {
864 141909 ▶ if (h->slotless_held > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->slotless_held > 0) { 0 141909 141909
865 0 h->slotless_held--;
866 0 __atomic_sub_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
867 141909 ▶ } else if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (h->my_slot_idx != UINT32_MAX) { 141909 0 141909
868 141909 uint32_t *rdepth = &h->reader_slots[h->my_slot_idx].rdepth;
869 /* Below zero: shm_rdlock_drop_unwound took this lock back from under
870 * its section, beneath a signal handler that interrupted it. */
871 141909 ▶ if (__atomic_sub_fetch(rdepth, 1, __ATOMIC_SEQ_CST) == UINT32_MAX)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_sub_fetch(rdepth, 1, __ATOMIC_SEQ_CST) == UINT32_MAX) 0 141909 141909
872 0 __atomic_add_fetch(rdepth, 1, __ATOMIC_SEQ_CST);
873 }
874 141909 }
875  
876 141909 static inline void shm_reader_wake_drain(ShmHandle *h) {
877 141909 ▶ if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_SEQ_CST) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_SEQ_CST) != 0) { 8508 133401 141909
878 8508 __atomic_add_fetch(&h->hdr->drain_seq, 1, __ATOMIC_RELEASE);
879 8508 syscall(SYS_futex, &h->hdr->drain_seq, FUTEX_WAKE, 1, NULL, NULL, 0);
880 }
881 141909 }
882  
883 /* The read locks this handle holds: its slot is its own. */
884 54110 static inline uint32_t shm_rd_held(const ShmHandle *h) {
885 108220 ▶ return h->slotless_held + (h->my_slot_idx == UINT32_MAX ? 0 :
Branch: 1 branch
BranchTrueFalseTotal
return h->slotless_held + (h->my_slot_idx == UINT32_MAX ? 0 : 54110 0 54110
886 54110 __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_RELAXED));
887 }
888  
889 /* At a lock call a handle's read locks are its open guards' (lock_depth); any
890 * more were left by a call croaked out of its section. */
891 54110 static inline void shm_rdlock_drop_unwound(ShmHandle *h) {
892 54110 ▶ if (__builtin_expect(shm_rd_held(h) <= h->lock_depth, 1)) return;
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(shm_rd_held(h) <= h->lock_depth, 1)) return; 54110 0 54110
893 0 ▶ while (shm_rd_held(h) > h->lock_depth) shm_rdepth_dec(h);
Branch: 1 branch
BranchTrueFalseTotal
while (shm_rd_held(h) > h->lock_depth) shm_rdepth_dec(h); 0 0 0
894 0 shm_reader_wake_drain(h);
895 }
896  
897 146215 static inline void shm_rwlock_rdlock(ShmHandle *h) {
898 /* Frozen: no writer exists, and PROT_READ would fault on the rdepth publish. */
899 146215 ▶ if (h->readonly) return;
Branch: 1 branch
BranchTrueFalseTotal
if (h->readonly) return; 5072 141143 146215
900 141143 shm_claim_reader_slot(h);
901 141143 ShmHeader *hdr = h->hdr;
902 193762 for (int spin = 0; ; spin++) {
903 193762 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
904 193762 ▶ if (cur == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) { 141909 51853 193762
905 141909 shm_rdepth_inc(h);
906 141909 ▶ if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0) 141143 766 141909
907 141143 return;
908 /* A writer arrived meanwhile: yield (write-preferring). */
909 766 shm_rdepth_dec(h);
910 766 shm_reader_wake_drain(h);
911 766 spin = 0;
912 766 continue;
913 }
914 /* The writer may be draining a read lock this handle was croaked out of. */
915 51853 shm_rdlock_drop_unwound(h);
916 51853 ▶ if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) { 51247 606 51853
917 51247 shm_rwlock_spin_pause();
918 51247 continue;
919 }
920 1212 if (cur >= SHM_RWLOCK_WRITER_BIT &&
921 606 !shm_holder_blocks_us(h, cur & SHM_RWLOCK_PID_MASK)) {
922 0 shm_recover_stale_lock(h, cur);
923 0 spin = 0;
924 0 continue;
925 }
926 606 shm_park(h);
927 606 cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
928 606 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 279 327 606
929 279 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
930 &shm_lock_timeout, NULL, 0);
931 279 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 204 75 279
if (rc == -1 && errno == ETIMEDOUT) { 0 204 204
932 0 shm_unpark(h);
933 0 shm_recover_after_timeout(h);
934 0 spin = 0;
935 0 continue;
936 }
937 }
938 606 shm_unpark(h);
939 606 spin = 0;
940 }
941 }
942  
943 146215 static inline void shm_rwlock_rdunlock(ShmHandle *h) {
944 146215 ▶ if (h->readonly) return;
Branch: 1 branch
BranchTrueFalseTotal
if (h->readonly) return; 5072 141143 146215
945 141143 shm_rdepth_dec(h);
946 141143 shm_reader_wake_drain(h);
947 }
948  
949 /* Clears the bits of unowned slots. Clear, re-read the pid, restore the bit if
950 * it was claimed meanwhile: a claimant sets pid before the bit. */
951 #define SHM_OCC_SWEEP_EVERY 1024u
952 2588 static inline void shm_occ_sweep(ShmHandle *h) {
953 43996 ▶ for (uint32_t w = 0; w < SHM_OCC_WORDS; w++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t w = 0; w < SHM_OCC_WORDS; w++) { 41408 2588 43996
954 41408 uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
955 44163 ▶ while (word) {
Branch: 1 branch
BranchTrueFalseTotal
while (word) { 2755 41408 44163
956 2755 uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
957 2755 word &= word - 1;
958 2755 uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
959 /* Pid before rdepth: a reader killed mid-lock is drained to pid 0
960 * with rdepth left set. */
961 2755 ▶ if (pid != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (pid != 0) { 2655 100 2755
962 5270 ▶ if (__atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST) != 0) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST) != 0) continue; 0 2655 2655
963 /* kill() alone: a zombie's idle slot frees once reaped. */
964 2655 ▶ if (pid == h->cached_pid || kill((pid_t)pid, 0) == 0 || errno != ESRCH) continue;
Branch: 3 branches
BranchTrueFalseTotal
if (pid == h->cached_pid || kill((pid_t)pid, 0) == 0 || errno != ESRCH) continue; 40 2615 2655
if (pid == h->cached_pid || kill((pid_t)pid, 0) == 0 || errno != ESRCH) continue; 40 0 40
if (pid == h->cached_pid || kill((pid_t)pid, 0) == 0 || errno != ESRCH) continue; 0 40 40
965 40 uint32_t ep = pid;
966 40 ▶ if (!__atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0, 0 40 40
967 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
968 0 continue;
969 }
970 140 shm_occ_clear(h, i);
971 140 ▶ if (__atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_SEQ_CST) != 0) 0 140 140
972 0 shm_occ_set(h, i);
973 }
974 }
975 2588 }
976  
977 427162 static inline void shm_rwlock_wrlock(ShmHandle *h) {
978 /* No Perl API: xt/slotless_reader_recovery.t compiles this with plain cc.
979 * A frozen handle never arrives: every mutator croaks before locking. */
980 427162 shm_claim_reader_slot(h); /* refresh cached_pid across fork */
981 427162 ShmHeader *hdr = h->hdr;
982 /* The owner pid is in the lock word: no crash window between taking and owning. */
983 427162 uint32_t mypid = SHM_RWLOCK_WR(h->cached_pid);
984 427162 int ent = shm_wrlock_ent_of(h);
985 /* Phase 1: exclude other writers. */
986 427164 for (int spin = 0; ; spin++) {
987 427164 uint32_t expected = 0;
988 427164 shm_wrlock_hold(ent);
989 427164 ▶ if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid, 427162 2 427164
990 0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE))
991 427162 break;
992 2 shm_wrlock_unhold(ent); /* before the test below, which asks about others */
993 2 shm_rdlock_drop_unwound(h); /* the holder may be draining a read lock we left */
994 2 ▶ if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) { 1 1 2
995 2 ▶ for (uint32_t k = 1u << (spin < 8 ? spin : 8); k; k--) shm_rwlock_spin_pause();
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t k = 1u << (spin < 8 ? spin : 8); k; k--) shm_rwlock_spin_pause(); 1 1 2
996 32 ▶ while (__atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED) != 0 && ++spin < SHM_RWLOCK_SPIN_LIMIT)
Branch: 2 branches
BranchTrueFalseTotal
while (__atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED) != 0 && ++spin < SHM_RWLOCK_SPIN_LIMIT) 32 0 32
while (__atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED) != 0 && ++spin < SHM_RWLOCK_SPIN_LIMIT) 31 1 32
997 31 shm_rwlock_spin_pause();
998 2 continue;
999 }
1000 2 if (expected >= SHM_RWLOCK_WRITER_BIT &&
1001 1 !shm_holder_blocks_us(h, expected & SHM_RWLOCK_PID_MASK)) {
1002 1 shm_recover_stale_lock(h, expected);
1003 1 spin = 0;
1004 1 continue;
1005 }
1006 0 shm_park(h);
1007 0 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
1008 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
1009 0 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
1010 &shm_lock_timeout, NULL, 0);
1011 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
1012 0 shm_unpark(h);
1013 0 shm_recover_after_timeout(h);
1014 0 spin = 0;
1015 0 continue;
1016 }
1017 }
1018 0 shm_unpark(h);
1019 0 spin = 0;
1020 }
1021 /* Phase 2: no new reader joins; drain those holding. */
1022 427162 for (int probe = 0;;) {
1023 /* ACQUIRE, or ARM64 may read it after the scan below and wait on a
1024 * bump it has already missed, until the timeout. */
1025 427162 uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_ACQUIRE);
1026 427162 int busy = 0;
1027 7261754 ▶ for (uint32_t w = 0; w < SHM_OCC_WORDS; w++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t w = 0; w < SHM_OCC_WORDS; w++) { 6834592 427162 7261754
1028 6834592 uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
1029 7263193 ▶ while (word) {
Branch: 1 branch
BranchTrueFalseTotal
while (word) { 428601 6834592 7263193
1030 428601 uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
1031 428601 word &= word - 1;
1032 428601 uint32_t rd = __atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST);
1033 428601 ▶ if (rd == 0) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (rd == 0) continue; 428501 100 428601
1034 100 uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
1035 100 ▶ if (pid == 0) continue; /* stale rdepth on a freed slot */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) continue; /* stale rdepth on a freed slot */ 100 0 100
1036 0 ▶ if (probe && !shm_pid_alive(pid)) {
Branch: 2 branches
BranchTrueFalseTotal
if (probe && !shm_pid_alive(pid)) { 0 0 0
if (probe && !shm_pid_alive(pid)) { 0 0 0
1037 /* Dead reader: drop its pid; leave the occ bit, not to race a claimant. */
1038 0 uint32_t ep = pid;
1039 0 __atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
1040 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
1041 0 continue;
1042 }
1043 0 busy = 1;
1044 }
1045 }
1046 /* A dead slotless reader is never drained (documented limitation). */
1047 427162 ▶ if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0) 0 427162 427162
1048 0 busy = 1;
1049 427162 ▶ if (!busy) {
Branch: 1 branch
BranchTrueFalseTotal
if (!busy) { 427162 0 427162
1050 427162 ▶ if (h->occ_sweep_in-- == 0) { h->occ_sweep_in = SHM_OCC_SWEEP_EVERY; shm_occ_sweep(h); }
Branch: 1 branch
BranchTrueFalseTotal
if (h->occ_sweep_in-- == 0) { h->occ_sweep_in = SHM_OCC_SWEEP_EVERY; shm_occ_sweep(h); } 2588 424574 427162
1051 427162 h->sect_bytes = 0;
1052 427162 h->sig_seen = *h->sig_count;
1053 427162 return;
1054 }
1055 /* One may be ours, left by a croaked call; dropping it ends the wait at once. */
1056 0 shm_rdlock_drop_unwound(h);
1057 /* Probe liveness after the first timeout, or EINTR: signals restarting
1058 * the relative wait would otherwise postpone it forever. */
1059 0 ▶ if (syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v,
Branch: 2 branches
BranchTrueFalseTotal
if (syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, 0 0 0
if (syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, 0 0 0
1060 0 probe ? &shm_lock_timeout : &shm_drain_probe_after, NULL, 0) == -1 &&
1061 0 ▶ (errno == ETIMEDOUT || errno == EINTR))
Branch: 2 branches
BranchTrueFalseTotal
(errno == ETIMEDOUT || errno == EINTR)) 0 0 0
(errno == ETIMEDOUT || errno == EINTR)) 0 0 0
1062 0 probe = 1;
1063 }
1064 }
1065  
1066 427162 static inline void shm_rwlock_wrunlock(ShmHandle *h) {
1067 427162 ShmHeader *hdr = h->hdr;
1068 427162 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
1069 427162 shm_wrlock_unhold(h->wrlock_ent); /* after the word is 0, never before */
1070 /* Never reset rwait when a wake finds nobody: a waiter between its rwait++
1071 * and FUTEX_WAIT would then sleep to its timeout, as no unlock wakes it. */
1072 427162 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 1 427161 427162
1073 1 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1074 427162 shm_sig_release(h);
1075 427162 }
1076  
1077 /* ---- Seqlock (lock-free readers) ---- */
1078  
1079 /* Torn lock-free passes before get/exists take the read lock. */
1080 #define SHM_READ_TRIES_UNLOCKED 8
1081 /* A value this long that tore once is re-read under the read lock at once. */
1082 #define SHM_READ_TORN_BIG (64u * 1024)
1083  
1084 3666089 static inline uint32_t shm_seqlock_read_begin(ShmHandle *h) {
1085 3666089 ShmHeader *hdr = h->hdr;
1086 3666089 int spin = 0, budget = 100000;
1087 9055605 for (;;) {
1088 12721694 uint32_t s = __atomic_load_n(&hdr->seq, __ATOMIC_ACQUIRE);
1089 12721694 ▶ if (__builtin_expect((s & 1) == 0, 1)) return s;
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect((s & 1) == 0, 1)) return s; 3666089 9055605 12721694
1090 9055605 ▶ if (__builtin_expect(spin < budget, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < budget, 1)) { 9055497 108 9055605
1091 9055497 shm_rwlock_spin_pause();
1092 9055497 spin++;
1093 9055497 continue;
1094 }
1095 /* Prolonged odd seq -- check for dead writer */
1096 108 uint32_t val = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
1097 108 ▶ if (val >= SHM_RWLOCK_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= SHM_RWLOCK_WRITER_BIT) { 108 0 108
1098 108 uint32_t pid = val & SHM_RWLOCK_PID_MASK;
1099 /* shm_holder_blocks_us's rule, with getpid(): cached_pid is 0 before
1100 * a handle's first lock, and the parent's in a fork child until then. */
1101 108 ▶ if (pid == 0
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0 108 0 108
1102 108 ▶ || (pid == (uint32_t)getpid() && !shm_wrlock_ours(h))
Branch: 2 branches
BranchTrueFalseTotal
|| (pid == (uint32_t)getpid() && !shm_wrlock_ours(h)) 0 108 108
|| (pid == (uint32_t)getpid() && !shm_wrlock_ours(h)) 0 0 0
1103 108 ▶ || !shm_pid_alive(pid)) {
Branch: 1 branch
BranchTrueFalseTotal
|| !shm_pid_alive(pid)) { 0 108 108
1104 /* A PROT_READ view cannot recover: return stale rather than spin. */
1105 0 ▶ if (h->readonly) return s;
Branch: 1 branch
BranchTrueFalseTotal
if (h->readonly) return s; 0 0 0
1106 0 shm_recover_stale_lock(h, val);
1107 0 spin = 0;
1108 0 continue;
1109 }
1110 0 ▶ } else if (h->readonly) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (h->readonly) { 0 0 0
1111 0 return s; /* odd seq with no holder: nothing will ever republish */
1112 }
1113 108 struct timespec ts = {0, 1000000};
1114 108 nanosleep(&ts, NULL);
1115 108 spin = 0;
1116 108 budget = 2000;
1117 }
1118 }
1119  
1120 3666047 static inline int shm_seqlock_read_retry(uint32_t *seq, uint32_t start) {
1121 3666047 __atomic_thread_fence(__ATOMIC_ACQUIRE);
1122 3666047 return __atomic_load_n(seq, __ATOMIC_RELAXED) != start;
1123 }
1124  
1125 427121 static inline void shm_seqlock_write_begin(uint32_t *seq) {
1126 427121 __atomic_add_fetch(seq, 1, __ATOMIC_RELEASE);
1127 /* StoreStore: on ARM64 the odd seq must be visible before the writes that follow. */
1128 427121 __atomic_thread_fence(__ATOMIC_RELEASE);
1129 427121 }
1130  
1131 427121 static inline void shm_seqlock_write_end(uint32_t *seq) {
1132 427121 __atomic_add_fetch(seq, 1, __ATOMIC_RELEASE);
1133 427121 }
1134  
1135 /* ---- Arena allocator ---- */
1136  
1137 static inline uint32_t shm_next_pow2(uint32_t v);
1138  
1139 1625172 static inline uint32_t shm_arena_round_up(uint32_t len) {
1140 1625172 ▶ if (len < SHM_ARENA_MIN_ALLOC) return SHM_ARENA_MIN_ALLOC;
Branch: 1 branch
BranchTrueFalseTotal
if (len < SHM_ARENA_MIN_ALLOC) return SHM_ARENA_MIN_ALLOC; 134867 1490305 1625172
1141 1490305 return shm_next_pow2(len);
1142 }
1143  
1144 /* Offset 0 is reserved: a block of exactly arena_cap never fits. */
1145 90326 static inline int shm_arena_can_hold(const ShmHeader *hdr, uint32_t slen) {
1146 177995 ▶ return slen <= SHM_INLINE_MAX ||
Branch: 1 branch
BranchTrueFalseTotal
return slen <= SHM_INLINE_MAX || 87669 2657 90326
1147 87669 ▶ (uint64_t)SHM_ARENA_MIN_ALLOC + shm_arena_round_up(slen) <= hdr->arena_cap;
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)SHM_ARENA_MIN_ALLOC + shm_arena_round_up(slen) <= hdr->arena_cap; 87652 17 87669
1148 }
1149  
1150 290241 static inline int shm_arena_class_index(uint32_t alloc_size) {
1151 290241 ▶ if (alloc_size <= SHM_ARENA_MIN_ALLOC) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (alloc_size <= SHM_ARENA_MIN_ALLOC) return 0; 85352 204889 290241
1152 204889 ▶ if (alloc_size > (SHM_ARENA_MIN_ALLOC << (SHM_ARENA_NUM_CLASSES - 1))) return -1;
Branch: 1 branch
BranchTrueFalseTotal
if (alloc_size > (SHM_ARENA_MIN_ALLOC << (SHM_ARENA_NUM_CLASSES - 1))) return -1; 83 204806 204889
1153 204806 return 32 - __builtin_clz(alloc_size - 1) - 4; /* log2(alloc_size) - 4 */
1154 }
1155  
1156 231125 static inline uint32_t shm_arena_alloc(ShmHeader *hdr, char *arena, uint32_t len) {
1157 231125 uint32_t asize = shm_arena_round_up(len);
1158 231125 int cls = shm_arena_class_index(asize);
1159  
1160 231125 ▶ if (cls >= 0 && hdr->arena_free[cls] != 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (cls >= 0 && hdr->arena_free[cls] != 0) { 231082 43 231125
if (cls >= 0 && hdr->arena_free[cls] != 0) { 42633 188449 231082
1161 42633 uint32_t head = hdr->arena_free[cls];
1162 /* Heads are peer-writable: treat a wild one as an empty class. */
1163 42633 ▶ if (head >= SHM_ARENA_MIN_ALLOC && (uint64_t)head + asize <= hdr->arena_cap) {
Branch: 2 branches
BranchTrueFalseTotal
if (head >= SHM_ARENA_MIN_ALLOC && (uint64_t)head + asize <= hdr->arena_cap) { 42633 0 42633
if (head >= SHM_ARENA_MIN_ALLOC && (uint64_t)head + asize <= hdr->arena_cap) { 42633 0 42633
1164 uint32_t next;
1165 42633 memcpy(&next, arena + head, sizeof(uint32_t));
1166 42633 hdr->arena_free[cls] = next;
1167 42633 return head;
1168 }
1169 }
1170 188492 ▶ if (cls < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cls < 0) { 43 188449 188492
1171 43 uint32_t prev = 0, cur = hdr->arena_large_free;
1172 43 uint32_t guard = SHM_ARENA_FREELIST_MAX(hdr->arena_cap);
1173 43 uint32_t steps = 0;
1174 44 ▶ while (cur != 0 && steps++ < guard) {
Branch: 2 branches
BranchTrueFalseTotal
while (cur != 0 && steps++ < guard) { 3 41 44
while (cur != 0 && steps++ < guard) { 3 0 3
1175 uint32_t next, blk;
1176 /* cur is peer-writable. Bound by the 8-byte [next][size] header, not
1177 * asize: a smaller valid block near the end is skipped, not fatal. */
1178 3 ▶ if (cur < SHM_ARENA_MIN_ALLOC ||
Branch: 1 branch
BranchTrueFalseTotal
if (cur < SHM_ARENA_MIN_ALLOC || 3 0 3
1179 3 ▶ (uint64_t)cur + 2 * sizeof(uint32_t) > hdr->arena_cap) break;
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)cur + 2 * sizeof(uint32_t) > hdr->arena_cap) break; 3 0 3
1180 3 memcpy(&next, arena + cur, sizeof(uint32_t));
1181 3 memcpy(&blk, arena + cur + sizeof(uint32_t), sizeof(uint32_t));
1182 3 ▶ if (!blk || (uint64_t)cur + blk > hdr->arena_cap) break;
Branch: 2 branches
BranchTrueFalseTotal
if (!blk || (uint64_t)cur + blk > hdr->arena_cap) break; 3 0 3
if (!blk || (uint64_t)cur + blk > hdr->arena_cap) break; 3 0 3
1183 /* Exact size: a larger block would be refiled at the smaller size on free. */
1184 3 ▶ if (blk == asize) {
Branch: 1 branch
BranchTrueFalseTotal
if (blk == asize) { 2 1 3
1185 2 ▶ if (prev == 0) hdr->arena_large_free = next;
Branch: 1 branch
BranchTrueFalseTotal
if (prev == 0) hdr->arena_large_free = next; 1 1 2
1186 1 else memcpy(arena + prev, &next, sizeof(uint32_t));
1187 2 return cur;
1188 }
1189 1 prev = cur; cur = next;
1190 }
1191 }
1192  
1193 188490 uint64_t off = hdr->arena_bump;
1194 188490 ▶ if (off + asize > hdr->arena_cap || off + asize > (uint64_t)UINT32_MAX)
Branch: 2 branches
BranchTrueFalseTotal
if (off + asize > hdr->arena_cap || off + asize > (uint64_t)UINT32_MAX) 96757 91733 188490
if (off + asize > hdr->arena_cap || off + asize > (uint64_t)UINT32_MAX) 0 96757 96757
1195 91733 return 0;
1196 96757 hdr->arena_bump = off + asize;
1197 96757 return (uint32_t)off;
1198 }
1199  
1200 59116 static inline void shm_arena_free_block(ShmHeader *hdr, char *arena,
1201 uint32_t off, uint32_t len) {
1202 59116 uint32_t asize = shm_arena_round_up(len);
1203 59116 int cls = shm_arena_class_index(asize);
1204 59156 ▶ if (off < SHM_ARENA_MIN_ALLOC) return;
Branch: 1 branch
BranchTrueFalseTotal
if (off < SHM_ARENA_MIN_ALLOC) return; 0 59116 59116
1205 /* off is peer-writable: leak a wild block rather than write out of bounds. */
1206 59116 ▶ if ((uint64_t)off + asize > hdr->arena_cap) return;
Branch: 1 branch
BranchTrueFalseTotal
if ((uint64_t)off + asize > hdr->arena_cap) return; 0 59116 59116
1207 59116 ▶ if (cls < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cls < 0) { 40 59076 59116
1208 40 uint32_t old_head = hdr->arena_large_free;
1209 40 memcpy(arena + off, &old_head, sizeof(uint32_t)); /* next */
1210 40 memcpy(arena + off + sizeof(uint32_t), &asize, sizeof(uint32_t)); /* size */
1211 /* The head publishes the block: its link must already be in it. */
1212 40 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1213 40 hdr->arena_large_free = off;
1214 40 return;
1215 }
1216 59076 uint32_t old_head = hdr->arena_free[cls];
1217 59076 memcpy(arena + off, &old_head, sizeof(uint32_t));
1218 59076 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1219 59076 hdr->arena_free[cls] = off;
1220 }
1221  
1222 /* The tag publishes a slot: whatever it covers, its TTL included, is stored before it. */
1223 816323 static inline void shm_publish_tag(uint8_t *states, uint32_t pos, uint8_t tag) {
1224 816323 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1225 816323 states[pos] = tag;
1226 816323 }
1227  
1228 /* Live block: ref = node index << 1 | is_value; at 2^31 slots max the shift has
1229 * no room for another flag. Snapshotted free block: ref = size. */
1230 typedef struct { uint32_t off; uint32_t ref; } ShmArenaRef;
1231  
1232 1757820 static int shm_arena_ref_cmp(const void *a, const void *b) {
1233 1757820 uint32_t x = ((const ShmArenaRef *)a)->off, y = ((const ShmArenaRef *)b)->off;
1234 1757820 ▶ return x < y ? -1 : x > y ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
return x < y ? -1 : x > y ? 1 : 0; 900506 857314 1757820
1235 }
1236  
1237 /* So a gap compaction cannot close is relisted as the same blocks: classes
1238 * never split. `out` NULL only counts. */
1239 182 static uint32_t shm_arena_snapshot_free(const ShmHeader *hdr, const char *arena,
1240 ShmArenaRef *out, uint32_t cap) {
1241 182 uint32_t n = 0;
1242 182 uint32_t guard = SHM_ARENA_FREELIST_MAX(hdr->arena_cap);
1243 3094 ▶ for (int c = 0; c < SHM_ARENA_NUM_CLASSES; c++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int c = 0; c < SHM_ARENA_NUM_CLASSES; c++) { 2912 182 3094
1244 2912 uint32_t size = (uint32_t)SHM_ARENA_MIN_ALLOC << c;
1245 2912 uint32_t off = hdr->arena_free[c], steps = 0;
1246 19672 ▶ while (off && steps++ < guard) {
Branch: 2 branches
BranchTrueFalseTotal
while (off && steps++ < guard) { 16760 2912 19672
while (off && steps++ < guard) { 16760 0 16760
1247 16760 ▶ if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) break;
Branch: 2 branches
BranchTrueFalseTotal
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) break; 16760 0 16760
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) break; 16760 0 16760
1248 16760 ▶ if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; }
Branch: 2 branches
BranchTrueFalseTotal
if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; } 8380 8380 16760
if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; } 0 8380 8380
1249 16760 n++;
1250 16760 memcpy(&off, arena + off, sizeof(uint32_t));
1251 }
1252 }
1253 182 uint32_t off = hdr->arena_large_free, steps = 0;
1254 182 ▶ while (off && steps++ < guard) {
Branch: 2 branches
BranchTrueFalseTotal
while (off && steps++ < guard) { 0 182 182
while (off && steps++ < guard) { 0 0 0
1255 uint32_t next, size;
1256 0 ▶ if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + 2 * sizeof(uint32_t) > hdr->arena_cap) break;
Branch: 2 branches
BranchTrueFalseTotal
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + 2 * sizeof(uint32_t) > hdr->arena_cap) break; 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + 2 * sizeof(uint32_t) > hdr->arena_cap) break; 0 0 0
1257 0 memcpy(&next, arena + off, sizeof(uint32_t));
1258 0 memcpy(&size, arena + off + sizeof(uint32_t), sizeof(uint32_t));
1259 0 ▶ if (!size || (uint64_t)off + size > hdr->arena_cap) break;
Branch: 2 branches
BranchTrueFalseTotal
if (!size || (uint64_t)off + size > hdr->arena_cap) break; 0 0 0
if (!size || (uint64_t)off + size > hdr->arena_cap) break; 0 0 0
1260 0 ▶ if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; }
Branch: 2 branches
BranchTrueFalseTotal
if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; } 0 0 0
if (out) { if (n >= cap) return n; out[n].off = off; out[n].ref = size; } 0 0 0
1261 0 n++;
1262 0 off = next;
1263 }
1264 182 return n;
1265 }
1266  
1267 /* Each block is clipped to [from, to) and cut into chunks no larger than itself,
1268 * so blocks never merge. Gaps and blocks ascend: one cursor serves every gap. */
1269 764 static void shm_arena_relist_gap(ShmHeader *hdr, char *arena, const ShmArenaRef *blocks,
1270 uint32_t n, uint32_t *ci, uint32_t from, uint32_t to) {
1271 20456 ▶ while (*ci < n && (uint64_t)blocks[*ci].off + blocks[*ci].ref <= from) (*ci)++;
Branch: 2 branches
BranchTrueFalseTotal
while (*ci < n && (uint64_t)blocks[*ci].off + blocks[*ci].ref <= from) (*ci)++; 20159 297 20456
while (*ci < n && (uint64_t)blocks[*ci].off + blocks[*ci].ref <= from) (*ci)++; 19692 467 20159
1272 764 uint32_t i = *ci;
1273 2777 ▶ while (i < n && blocks[i].off < to) {
Branch: 2 branches
BranchTrueFalseTotal
while (i < n && blocks[i].off < to) { 2381 396 2777
while (i < n && blocks[i].off < to) { 2013 368 2381
1274 2013 uint32_t s = blocks[i].off > from ? blocks[i].off : from;
1275 2013 uint64_t e = (uint64_t)blocks[i].off + blocks[i].ref;
1276 2013 ▶ if (e > to) e = to;
Branch: 1 branch
BranchTrueFalseTotal
if (e > to) e = to; 0 2013 2013
1277 4026 ▶ while (e > s && e - s >= SHM_ARENA_MIN_ALLOC) {
Branch: 2 branches
BranchTrueFalseTotal
while (e > s && e - s >= SHM_ARENA_MIN_ALLOC) { 2013 2013 4026
while (e > s && e - s >= SHM_ARENA_MIN_ALLOC) { 2013 0 2013
1278 2013 uint32_t span = (uint32_t)(e - s), chunk = blocks[i].ref;
1279 2013 ▶ while (chunk > span) chunk >>= 1;
Branch: 1 branch
BranchTrueFalseTotal
while (chunk > span) chunk >>= 1; 0 2013 2013
1280 2013 shm_arena_free_block(hdr, arena, s, chunk);
1281 2013 s += chunk;
1282 }
1283 2013 i++;
1284 }
1285 764 *ci = i;
1286 764 }
1287  
1288 /* Only the smallest class with no free block and no headroom can gain from a
1289 * slide, and only if it fits headroom plus holes. */
1290 178 static inline int shm_arena_slide_useless(const ShmHeader *hdr, uint64_t live) {
1291 178 uint64_t headroom = hdr->arena_cap - hdr->arena_bump;
1292 178 uint64_t used = hdr->arena_bump - SHM_ARENA_MIN_ALLOC;
1293 178 ▶ if (live > used) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (live > used) return 0; 0 178 178
1294 966 ▶ for (int c = 0; c < SHM_ARENA_NUM_CLASSES; c++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int c = 0; c < SHM_ARENA_NUM_CLASSES; c++) { 966 0 966
1295 966 uint64_t size = (uint64_t)SHM_ARENA_MIN_ALLOC << c;
1296 966 ▶ if (hdr->arena_free[c] || size <= headroom) continue;
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->arena_free[c] || size <= headroom) continue; 827 139 966
if (hdr->arena_free[c] || size <= headroom) continue; 649 178 827
1297 178 return size > headroom + (used - live);
1298 }
1299 0 return 0;
1300 }
1301  
1302 596 static inline void shm_arena_reset(ShmHeader *hdr) {
1303 596 memset(hdr->arena_free, 0, sizeof(hdr->arena_free));
1304 596 hdr->arena_large_free = 0;
1305 596 hdr->arena_bump = SHM_ARENA_MIN_ALLOC;
1306 596 }
1307  
1308 393811 static inline int shm_str_store(ShmHeader *hdr, char *arena,
1309 uint32_t *off, uint32_t *len_field,
1310 const char *str, uint32_t slen, bool utf8) {
1311 393811 ▶ if (slen <= SHM_INLINE_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (slen <= SHM_INLINE_MAX) { 162686 231125 393811
1312 162686 shm_inline_pack(off, len_field, str, slen, utf8);
1313 162686 return 1;
1314 }
1315 231125 uint32_t aoff = shm_arena_alloc(hdr, arena, slen);
1316 231125 ▶ if (aoff == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (aoff == 0) return 0; 91733 139392 231125
1317 139392 memcpy(arena + aoff, str, slen);
1318 /* Interim inline-empty state -- see shm_inline_pack. */
1319 139392 __atomic_store_n(len_field, SHM_INLINE_FLAG, __ATOMIC_RELEASE);
1320 139392 __atomic_store_n(off, aoff, __ATOMIC_RELEASE);
1321 139392 ▶ __atomic_store_n(len_field, SHM_PACK_LEN(slen, utf8), __ATOMIC_RELEASE);
Branch: 1 branch
BranchTrueFalseTotal
__atomic_store_n(len_field, SHM_PACK_LEN(slen, utf8), __ATOMIC_RELEASE); 9778 129614 139392
1322 139392 return 1;
1323 }
1324  
1325 156411 static inline void shm_str_free(ShmHeader *hdr, char *arena,
1326 uint32_t off, uint32_t len_field) {
1327 156411 ▶ if (!SHM_IS_INLINE(len_field))
Branch: 1 branch
BranchTrueFalseTotal
if (!SHM_IS_INLINE(len_field)) 57103 99308 156411
1328 57103 shm_arena_free_block(hdr, arena, off, SHM_UNPACK_LEN(len_field));
1329 156411 }
1330  
1331 5505 static inline void shm_str_copy(char *dst, uint32_t off, uint32_t len_field,
1332 const char *arena, uint64_t arena_cap, uint32_t len) {
1333 5505 ▶ if (SHM_IS_INLINE(len_field)) {
Branch: 1 branch
BranchTrueFalseTotal
if (SHM_IS_INLINE(len_field)) { 4710 795 5505
1334 4710 shm_inline_read(off, len_field, dst);
1335 795 ▶ } else if (off >= SHM_ARENA_MIN_ALLOC && (uint64_t)off + len <= arena_cap) {
Branch: 2 branches
BranchTrueFalseTotal
} else if (off >= SHM_ARENA_MIN_ALLOC && (uint64_t)off + len <= arena_cap) { 795 0 795
} else if (off >= SHM_ARENA_MIN_ALLOC && (uint64_t)off + len <= arena_cap) { 795 0 795
1336 795 memcpy(dst, arena + off, len);
1337 } else {
1338 /* A poisoned record reads as zeros. Readers that take an arena pointer
1339 * instead bound off+len against arena_cap themselves. */
1340 0 memset(dst, 0, len);
1341 }
1342 5505 }
1343  
1344 1536309 static inline uint32_t shm_next_pow2(uint32_t v) {
1345 1536309 ▶ if (v < 2) return 2;
Branch: 1 branch
BranchTrueFalseTotal
if (v < 2) return 2; 0 1536309 1536309
1346 1536309 ▶ if (v > 0x80000000U) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (v > 0x80000000U) return 0; 0 1536309 1536309
1347 1536309 v--;
1348 1536309 v |= v >> 1; v |= v >> 2; v |= v >> 4; v |= v >> 8; v |= v >> 16;
1349 1536309 return v + 1;
1350 }
1351  
1352 /* Cap at 2^31 before next_pow2, which returns 0 above it. */
1353 45974 static inline uint32_t shm_max_tcap_from_entries(uint32_t max_entries) {
1354 45974 uint64_t want = (uint64_t)max_entries * 4 / 3 + 1;
1355 45974 ▶ uint32_t cap = (want > 0x80000000ULL) ? 0x80000000U : shm_next_pow2((uint32_t)want);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t cap = (want > 0x80000000ULL) ? 0x80000000U : shm_next_pow2((uint32_t)want); 45974 0 45974
1356 45974 return cap < SHM_INITIAL_CAP ? SHM_INITIAL_CAP : cap;
1357 }
1358  
1359 404169 static inline int shm_over_load(uint32_t size, uint32_t tomb, uint32_t cap) {
1360 404169 return (uint64_t)(size + tomb) * 4 > (uint64_t)cap * 3;
1361 }
1362  
1363 /* The last test matters at max capacity: nothing else stops churn from filling
1364 * every empty slot, after which each miss walks the whole table. */
1365 399938 static inline int shm_needs_compaction(uint32_t size, uint32_t tomb, uint32_t cap) {
1366 399938 ▶ return tomb > size || tomb > cap / 4 || (uint64_t)tomb * 2 > (uint64_t)cap - size;
Branch: 3 branches
BranchTrueFalseTotal
return tomb > size || tomb > cap / 4 || (uint64_t)tomb * 2 > (uint64_t)cap - size; 399726 212 399938
return tomb > size || tomb > cap / 4 || (uint64_t)tomb * 2 > (uint64_t)cap - size; 398368 1358 399726
return tomb > size || tomb > cap / 4 || (uint64_t)tomb * 2 > (uint64_t)cap - size; 145 398223 398368
1367 }
1368  
1369 53584 static inline int shm_under_load(uint32_t size, uint32_t cap) {
1370 53584 ▶ return cap > SHM_INITIAL_CAP && (uint64_t)size * 4 < cap;
Branch: 2 branches
BranchTrueFalseTotal
return cap > SHM_INITIAL_CAP && (uint64_t)size * 4 < cap; 53312 272 53584
return cap > SHM_INITIAL_CAP && (uint64_t)size * 4 < cap; 10330 42982 53312
1371 }
1372  
1373 /* All the way down in one call: batch removers reach it once per batch. */
1374 73 static inline uint32_t shm_shrink_target(uint32_t size, uint32_t cap) {
1375 248 ▶ while (shm_under_load(size, cap)) cap /= 2;
Branch: 1 branch
BranchTrueFalseTotal
while (shm_under_load(size, cap)) cap /= 2; 175 73 248
1376 73 ▶ if (cap < SHM_INITIAL_CAP) cap = SHM_INITIAL_CAP;
Branch: 1 branch
BranchTrueFalseTotal
if (cap < SHM_INITIAL_CAP) cap = SHM_INITIAL_CAP; 0 73 73
1377 73 return cap;
1378 }
1379  
1380 /* lru_skip percentage to the promotion mask: 50 -> 1 (every 2nd), 90 -> 15,
1381 * 95 -> 31. Outside 1..99 skipping is off. */
1382 3215 static inline uint32_t shm_lru_skip_to_mask(uint32_t lru_skip) {
1383 3215 ▶ if (lru_skip == 0 || lru_skip >= 100) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (lru_skip == 0 || lru_skip >= 100) return 0; 65 3150 3215
if (lru_skip == 0 || lru_skip >= 100) return 0; 0 65 65
1384 65 uint32_t interval = 100 / (100 - lru_skip);
1385 65 uint32_t p = 1;
1386 325 ▶ while (p < interval) p <<= 1;
Branch: 1 branch
BranchTrueFalseTotal
while (p < interval) p <<= 1; 260 65 325
1387 65 return p - 1;
1388 }
1389  
1390 /* ---- LRU helpers ---- */
1391  
1392 43840 static inline void shm_lru_unlink(ShmHandle *h, uint32_t idx) {
1393 43840 uint32_t *prev = h->lru_prev;
1394 43840 uint32_t *next = h->lru_next;
1395 43840 ShmHeader *hdr = h->hdr;
1396 43840 uint32_t p = prev[idx], n = next[idx];
1397 43840 ▶ if (p == SHM_LRU_NONE && n == SHM_LRU_NONE && hdr->lru_head != idx) return;
Branch: 3 branches
BranchTrueFalseTotal
if (p == SHM_LRU_NONE && n == SHM_LRU_NONE && hdr->lru_head != idx) return; 2 43838 43840
if (p == SHM_LRU_NONE && n == SHM_LRU_NONE && hdr->lru_head != idx) return; 1 1 2
if (p == SHM_LRU_NONE && n == SHM_LRU_NONE && hdr->lru_head != idx) return; 0 1 1
1398 43840 ▶ if (p != SHM_LRU_NONE) next[p] = n;
Branch: 1 branch
BranchTrueFalseTotal
if (p != SHM_LRU_NONE) next[p] = n; 43838 2 43840
1399 2 else hdr->lru_head = n;
1400 43840 ▶ if (n != SHM_LRU_NONE) prev[n] = p;
Branch: 1 branch
BranchTrueFalseTotal
if (n != SHM_LRU_NONE) prev[n] = p; 15437 28403 43840
1401 28403 else hdr->lru_tail = p;
1402 43840 prev[idx] = next[idx] = SHM_LRU_NONE;
1403 }
1404  
1405 55083 static inline void shm_lru_push_front(ShmHandle *h, uint32_t idx) {
1406 55083 uint32_t *prev = h->lru_prev;
1407 55083 uint32_t *next = h->lru_next;
1408 55083 ShmHeader *hdr = h->hdr;
1409 55083 ▶ if (h->lru_accessed) __atomic_store_n(&h->lru_accessed[idx], 0, __ATOMIC_RELAXED);
Branch: 1 branch
BranchTrueFalseTotal
if (h->lru_accessed) __atomic_store_n(&h->lru_accessed[idx], 0, __ATOMIC_RELAXED); 55083 0 55083
1410 55083 prev[idx] = SHM_LRU_NONE;
1411 55083 next[idx] = hdr->lru_head;
1412 55083 ▶ if (hdr->lru_head != SHM_LRU_NONE) prev[hdr->lru_head] = idx;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->lru_head != SHM_LRU_NONE) prev[hdr->lru_head] = idx; 54928 155 55083
1413 155 else hdr->lru_tail = idx;
1414 55083 hdr->lru_head = idx;
1415 55083 }
1416  
1417 75 static inline void shm_lru_promote(ShmHandle *h, uint32_t idx) {
1418 75 ShmHeader *hdr = h->hdr;
1419 75 ▶ if (hdr->lru_head == idx) return;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->lru_head == idx) return; 7 68 75
1420 /* Promote every (mask+1)th access, set the clock bit on the rest; never skip
1421 * the tail, or eviction would take a hot entry. */
1422 68 ▶ if (hdr->lru_skip > 0 && idx != hdr->lru_tail) {
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->lru_skip > 0 && idx != hdr->lru_tail) { 65 3 68
if (hdr->lru_skip > 0 && idx != hdr->lru_tail) { 64 1 65
1423 static __thread uint32_t promote_ctr = 0;
1424 64 ▶ if ((++promote_ctr & hdr->lru_skip) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ((++promote_ctr & hdr->lru_skip) != 0) { 60 4 64
1425 60 __atomic_store_n(&h->lru_accessed[idx], 1, __ATOMIC_RELAXED);
1426 60 return;
1427 }
1428 }
1429 8 shm_lru_unlink(h, idx);
1430 8 shm_lru_push_front(h, idx);
1431 }
1432  
1433 /* Never on a frozen map (PROT_READ; the file stays as sealed). A set bit is
1434 * not rewritten, so a hot key's line stays shared. */
1435 7477529 static inline void shm_lru_mark(ShmHandle *h, uint32_t idx) {
1436 7477529 ▶ if (h->lru_accessed && !h->readonly &&
Branch: 2 branches
BranchTrueFalseTotal
if (h->lru_accessed && !h->readonly && 2300 7475229 7477529
if (h->lru_accessed && !h->readonly && 1299 1001 2300
1437 1299 ▶ !__atomic_load_n(&h->lru_accessed[idx], __ATOMIC_RELAXED) &&
Branch: 1 branch
BranchTrueFalseTotal
!__atomic_load_n(&h->lru_accessed[idx], __ATOMIC_RELAXED) && 1297 2 1299
1438 1297 ▶ !__atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED))
Branch: 1 branch
BranchTrueFalseTotal
!__atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED)) 1294 3 1297
1439 1294 __atomic_store_n(&h->lru_accessed[idx], 1, __ATOMIC_RELAXED);
1440 7477529 }
1441  
1442 /* Recount size/tombstones from states[]: a writer killed between publishing
1443 * states[idx] and bumping the counters leaves them behind the table. */
1444 1 static void shm_recount_counters(ShmHandle *h) {
1445 1 ShmHeader *hdr = h->hdr;
1446 1 uint32_t cap = hdr->table_cap, live = 0, tomb = 0;
1447 17 ▶ for (uint32_t i = 0; i < cap; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap; i++) { 16 1 17
1448 16 uint8_t st = h->states[i];
1449 16 ▶ if (st == SHM_TOMBSTONE) tomb++;
Branch: 1 branch
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) tomb++; 0 16 16
1450 16 ▶ else if (SHM_IS_LIVE(st)) live++;
Branch: 1 branch
BranchTrueFalseTotal
else if (SHM_IS_LIVE(st)) live++; 7 9 16
1451 }
1452 /* Bracket only the stores: an attach validates size and tombstones together. */
1453 1 shm_seqlock_write_begin(&hdr->seq);
1454 /* Decreasing counter first: an increase can transiently exceed table_cap,
1455 * and a crash there is unrepairable. Atomic, or gcc merges the arms. */
1456 1 ▶ if (tomb <= hdr->tombstones) {
Branch: 1 branch
BranchTrueFalseTotal
if (tomb <= hdr->tombstones) { 1 0 1
1457 1 __atomic_store_n(&hdr->tombstones, tomb, __ATOMIC_RELEASE);
1458 1 __atomic_store_n(&hdr->size, live, __ATOMIC_RELEASE);
1459 } else {
1460 0 __atomic_store_n(&hdr->size, live, __ATOMIC_RELEASE);
1461 0 __atomic_store_n(&hdr->tombstones, tomb, __ATOMIC_RELEASE);
1462 }
1463 1 shm_seqlock_write_end(&hdr->seq);
1464 1 }
1465  
1466 /* A writer killed mid-unlink/push_front leaves a break that would loop the next
1467 * eviction. The rebuild is in slot order: recency is lost, not entries. */
1468 1 static void shm_lru_rebuild_if_corrupt(ShmHandle *h) {
1469 1 ▶ if (!h->lru_prev) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h->lru_prev) return; 1 0 1
1470 0 ShmHeader *hdr = h->hdr;
1471 0 uint32_t cap = hdr->table_cap;
1472 0 uint32_t head = hdr->lru_head;
1473 0 uint32_t tail = hdr->lru_tail;
1474 0 int corrupt = 0;
1475 0 uint32_t chain_len = 0;
1476  
1477 0 ▶ if ((head != SHM_LRU_NONE && head >= cap) ||
Branch: 3 branches
BranchTrueFalseTotal
if ((head != SHM_LRU_NONE && head >= cap) || 0 0 0
if ((head != SHM_LRU_NONE && head >= cap) || 0 0 0
if ((head != SHM_LRU_NONE && head >= cap) || 0 0 0
1478 0 ▶ (tail != SHM_LRU_NONE && tail >= cap)) {
Branch: 1 branch
BranchTrueFalseTotal
(tail != SHM_LRU_NONE && tail >= cap)) { 0 0 0
1479 0 corrupt = 1;
1480 } else {
1481 0 uint32_t prev_idx = SHM_LRU_NONE;
1482 0 uint32_t idx = head;
1483 0 ▶ while (idx != SHM_LRU_NONE) {
Branch: 1 branch
BranchTrueFalseTotal
while (idx != SHM_LRU_NONE) { 0 0 0
1484 0 ▶ if (idx >= cap || !SHM_IS_LIVE(h->states[idx]) ||
Branch: 2 branches
BranchTrueFalseTotal
if (idx >= cap || !SHM_IS_LIVE(h->states[idx]) || 0 0 0
if (idx >= cap || !SHM_IS_LIVE(h->states[idx]) || 0 0 0
1485 0 ▶ h->lru_prev[idx] != prev_idx) { corrupt = 1; break; }
Branch: 1 branch
BranchTrueFalseTotal
h->lru_prev[idx] != prev_idx) { corrupt = 1; break; } 0 0 0
1486 0 prev_idx = idx;
1487 0 idx = h->lru_next[idx];
1488 0 ▶ if (++chain_len > cap) { corrupt = 1; break; }
Branch: 1 branch
BranchTrueFalseTotal
if (++chain_len > cap) { corrupt = 1; break; } 0 0 0
1489 }
1490 0 ▶ if (!corrupt && prev_idx != tail) corrupt = 1;
Branch: 2 branches
BranchTrueFalseTotal
if (!corrupt && prev_idx != tail) corrupt = 1; 0 0 0
if (!corrupt && prev_idx != tail) corrupt = 1; 0 0 0
1491 }
1492 /* Count live states, not hdr->size, which a killed writer can leave behind. */
1493 0 uint32_t live_count = 0;
1494 0 ▶ if (!corrupt) {
Branch: 1 branch
BranchTrueFalseTotal
if (!corrupt) { 0 0 0
1495 0 ▶ for (uint32_t i = 0; i < cap; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap; i++) 0 0 0
1496 0 ▶ if (SHM_IS_LIVE(h->states[i])) live_count++;
Branch: 1 branch
BranchTrueFalseTotal
if (SHM_IS_LIVE(h->states[i])) live_count++; 0 0 0
1497 0 ▶ if (chain_len != live_count) corrupt = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (chain_len != live_count) corrupt = 1; 0 0 0
1498 }
1499 0 ▶ if (!corrupt) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!corrupt) return; 0 0 0
1500  
1501 0 memset(h->lru_prev, 0xFF, (size_t)cap * sizeof(uint32_t));
1502 0 memset(h->lru_next, 0xFF, (size_t)cap * sizeof(uint32_t));
1503 0 ▶ if (h->lru_accessed) memset(h->lru_accessed, 0, cap);
Branch: 1 branch
BranchTrueFalseTotal
if (h->lru_accessed) memset(h->lru_accessed, 0, cap); 0 0 0
1504 0 uint32_t prev = SHM_LRU_NONE;
1505 0 uint32_t new_head = SHM_LRU_NONE;
1506 0 ▶ for (uint32_t i = 0; i < cap; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap; i++) { 0 0 0
1507 0 ▶ if (!SHM_IS_LIVE(h->states[i])) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (!SHM_IS_LIVE(h->states[i])) continue; 0 0 0
1508 0 h->lru_prev[i] = prev;
1509 0 ▶ if (prev != SHM_LRU_NONE) h->lru_next[prev] = i;
Branch: 1 branch
BranchTrueFalseTotal
if (prev != SHM_LRU_NONE) h->lru_next[prev] = i; 0 0 0
1510 0 else new_head = i;
1511 0 prev = i;
1512 }
1513 0 hdr->lru_head = new_head;
1514 0 hdr->lru_tail = prev;
1515 }
1516  
1517 /* Rerun from the SHM_RZ_CLEAR record after a crash: table_cap drops only once
1518 * every state is empty, and the stores after it repeat harmlessly. */
1519 11 static void shm_clear_run(ShmHandle *h) {
1520 11 ShmHeader *hdr = h->hdr;
1521 11 memset(h->states, SHM_EMPTY, hdr->table_cap);
1522 11 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1523 11 hdr->size = 0;
1524 11 hdr->tombstones = 0;
1525 11 ▶ if (hdr->table_cap > SHM_INITIAL_CAP)
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->table_cap > SHM_INITIAL_CAP) 2 9 11
1526 2 hdr->table_cap = SHM_INITIAL_CAP;
1527  
1528 11 ▶ if (h->arena)
Branch: 1 branch
BranchTrueFalseTotal
if (h->arena) 6 5 11
1529 6 shm_arena_reset(hdr);
1530  
1531 /* LRU links and TTLs are read only after an insert or move writes them. */
1532 11 ▶ if (h->lru_prev) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->lru_prev) { 1 10 11
1533 1 memset(h->lru_prev, 0xFF, SHM_INITIAL_CAP * sizeof(uint32_t));
1534 1 memset(h->lru_next, 0xFF, SHM_INITIAL_CAP * sizeof(uint32_t));
1535 1 ▶ if (h->lru_accessed) memset(h->lru_accessed, 0, SHM_INITIAL_CAP);
Branch: 1 branch
BranchTrueFalseTotal
if (h->lru_accessed) memset(h->lru_accessed, 0, SHM_INITIAL_CAP); 1 0 1
1536 1 hdr->lru_head = SHM_LRU_NONE;
1537 1 hdr->lru_tail = SHM_LRU_NONE;
1538 }
1539  
1540 11 ▶ if (h->expires_at) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->expires_at) { 0 11 11
1541 0 memset(h->expires_at, 0, SHM_INITIAL_CAP * sizeof(uint32_t));
1542 0 hdr->flush_cursor = 0;
1543 }
1544 11 hdr->pop_cursor = 0;
1545 11 hdr->shift_cursor = 0;
1546  
1547 11 hdr->table_gen++;
1548 11 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1549 11 __atomic_store_n(&hdr->rz_phase, SHM_RZ_NONE, __ATOMIC_RELEASE);
1550 11 }
1551  
1552 /* ---- Create / Open / Close ---- */
1553  
1554 #define SHM_ERR_BUFLEN (PATH_MAX + 256)
1555  
1556 typedef struct {
1557 uint64_t nodes_off, states_off;
1558 uint64_t lru_prev_off, lru_next_off, lru_accessed_off;
1559 uint64_t expires_off;
1560 uint64_t reader_slots_off;
1561 uint64_t occ_off;
1562 uint64_t arena_off, arena_cap;
1563 uint64_t total_size;
1564 uint64_t end_off; /* end of the LRU/TTL region; the file must reach it */
1565 } ShmLayout;
1566  
1567 1521 static inline uint64_t shm_clamp_arena_cap(uint64_t want) {
1568 1521 ▶ if (want < 4096) return 4096;
Branch: 1 branch
BranchTrueFalseTotal
if (want < 4096) return 4096; 208 1313 1521
1569 1313 ▶ if (want > UINT32_MAX) return UINT32_MAX;
Branch: 1 branch
BranchTrueFalseTotal
if (want > UINT32_MAX) return UINT32_MAX; 0 1313 1313
1570 1313 return want;
1571 }
1572  
1573 /* Reopen checks the file against end_off and takes reader_slots_off and
1574 * total_size from the header. */
1575 88847 static inline void shm_compute_layout(ShmLayout *lo, uint32_t max_tcap,
1576 uint32_t node_size, int has_lru,
1577 int has_ttl, int has_arena,
1578 uint32_t max_entries,
1579 uint64_t arena_cap_override) {
1580 88847 lo->nodes_off = sizeof(ShmHeader);
1581 88847 lo->states_off = lo->nodes_off + (uint64_t)max_tcap * node_size;
1582 88847 uint64_t off = lo->states_off + max_tcap;
1583 88847 lo->lru_prev_off = lo->lru_next_off = lo->lru_accessed_off = 0;
1584 88847 lo->expires_off = 0;
1585 88847 ▶ if (has_lru) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_lru) { 1056 87791 88847
1586 1056 off = (off + 3) & ~(uint64_t)3;
1587 1056 lo->lru_prev_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
1588 1056 lo->lru_next_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
1589 1056 lo->lru_accessed_off = off; off += max_tcap;
1590 }
1591 88847 ▶ if (has_ttl) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_ttl) { 538 88309 88847
1592 538 off = (off + 3) & ~(uint64_t)3;
1593 538 lo->expires_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
1594 }
1595 88847 lo->end_off = off;
1596 88847 off = (off + 7) & ~(uint64_t)7;
1597 88847 lo->reader_slots_off = off;
1598 88847 off += (uint64_t)SHM_READER_SLOTS * sizeof(ShmReaderSlot);
1599 88847 lo->occ_off = off;
1600 88847 off += SHM_OCC_BYTES;
1601 88847 lo->arena_off = lo->arena_cap = 0;
1602 88847 ▶ if (has_arena) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_arena) { 1521 87326 88847
1603 1521 lo->arena_off = (off + 7) & ~(uint64_t)7;
1604 1521 uint64_t want = arena_cap_override ? arena_cap_override
1605 1521 ▶ : (uint64_t)max_entries * 128;
Branch: 1 branch
BranchTrueFalseTotal
: (uint64_t)max_entries * 128; 1437 84 1521
1606 1521 lo->arena_cap = shm_clamp_arena_cap(want);
1607 1521 lo->total_size = lo->arena_off + lo->arena_cap;
1608 } else {
1609 87326 lo->total_size = off;
1610 }
1611 88847 }
1612  
1613 3215 static inline void shm_init_header(ShmHeader *hdr, void *base,
1614 const ShmLayout *lo, uint32_t max_tcap,
1615 uint32_t node_size, uint32_t variant_id,
1616 int has_arena, int has_lru, int has_ttl,
1617 uint32_t max_size, uint32_t default_ttl,
1618 uint32_t lru_skip) {
1619 3215 memset(hdr, 0, sizeof(ShmHeader));
1620 3215 hdr->version = SHM_VERSION;
1621 3215 hdr->variant_id = variant_id;
1622 3215 hdr->node_size = node_size;
1623 3215 hdr->max_table_cap = max_tcap;
1624 3215 hdr->table_cap = SHM_INITIAL_CAP;
1625 3215 hdr->total_size = lo->total_size;
1626 3215 hdr->nodes_off = lo->nodes_off;
1627 3215 hdr->states_off = lo->states_off;
1628 3215 ▶ hdr->arena_off = has_arena ? lo->arena_off : 0;
Branch: 1 branch
BranchTrueFalseTotal
hdr->arena_off = has_arena ? lo->arena_off : 0; 1318 1897 3215
1629 3215 hdr->arena_cap = lo->arena_cap;
1630 3215 hdr->reader_slots_off = lo->reader_slots_off;
1631 3215 hdr->arena_bump = SHM_ARENA_MIN_ALLOC; /* reserve offset 0 */
1632 3215 hdr->routing = SHM_ROUTING_SPLIT;
1633 3215 hdr->max_size = max_size;
1634 3215 hdr->default_ttl = default_ttl;
1635 3215 hdr->lru_skip = shm_lru_skip_to_mask(lru_skip);
1636 3215 hdr->lru_head = SHM_LRU_NONE;
1637 3215 hdr->lru_tail = SHM_LRU_NONE;
1638 /* Only the initial table: see shm_clear_run. */
1639 3215 ▶ if (has_lru) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_lru) { 896 2319 3215
1640 896 memset((char *)base + lo->lru_prev_off, 0xFF, SHM_INITIAL_CAP * sizeof(uint32_t));
1641 896 memset((char *)base + lo->lru_next_off, 0xFF, SHM_INITIAL_CAP * sizeof(uint32_t));
1642 896 memset((char *)base + lo->lru_accessed_off, 0, SHM_INITIAL_CAP);
1643 }
1644 3215 ▶ if (has_ttl)
Branch: 1 branch
BranchTrueFalseTotal
if (has_ttl) 461 2754 3215
1645 461 memset((char *)base + lo->expires_off, 0, SHM_INITIAL_CAP * sizeof(uint32_t));
1646 3215 memset((char *)base + lo->reader_slots_off, 0,
1647 SHM_READER_SLOTS * sizeof(ShmReaderSlot));
1648 /* Create does not memset the mapping: no OS zero-fill to rely on. */
1649 3215 memset((char *)base + lo->occ_off, 0, SHM_OCC_BYTES);
1650 /* Magic last, as a release store: the commit point, so a creator killed
1651 before it leaves magic == 0, never a file mistaken for a valid one. */
1652 3215 __atomic_store_n(&hdr->magic, SHM_MAGIC, __ATOMIC_RELEASE);
1653 3215 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1654 3215 }
1655  
1656 /* The caller has checked the file size against sizeof(ShmHeader) and total_size. */
1657 42905 static inline int shm_validate_header(const ShmHeader *hdr,
1658 uint32_t variant_id, uint32_t node_size) {
1659 85810 return (hdr->magic == SHM_MAGIC &&
1660 42905 ▶ hdr->version == SHM_VERSION &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->version == SHM_VERSION && 42905 0 42905
1661 42905 ▶ hdr->variant_id == variant_id &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->variant_id == variant_id && 42903 2 42905
1662 42903 ▶ hdr->node_size == node_size &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->node_size == node_size && 42903 0 42903
1663 42903 ▶ hdr->nodes_off >= sizeof(ShmHeader) &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->nodes_off >= sizeof(ShmHeader) && 42903 0 42903
1664 42903 ▶ hdr->states_off > hdr->nodes_off &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->states_off > hdr->nodes_off && 42903 0 42903
1665 42903 ▶ hdr->states_off < hdr->total_size &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->states_off < hdr->total_size && 42903 0 42903
1666 42903 ▶ (!hdr->arena_off || (hdr->arena_off < hdr->total_size &&
Branch: 2 branches
BranchTrueFalseTotal
(!hdr->arena_off || (hdr->arena_off < hdr->total_size && 162 42741 42903
(!hdr->arena_off || (hdr->arena_off < hdr->total_size && 162 0 162
1667 162 ▶ hdr->arena_off + hdr->arena_cap <= hdr->total_size &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->arena_off + hdr->arena_cap <= hdr->total_size && 162 0 162
1668 162 ▶ hdr->arena_bump <= hdr->arena_cap &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->arena_bump <= hdr->arena_cap && 162 0 162
1669 162 ▶ hdr->arena_bump >= SHM_ARENA_MIN_ALLOC)) &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->arena_bump >= SHM_ARENA_MIN_ALLOC)) && 162 0 162
1670 42903 ▶ hdr->max_table_cap > 0 &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->max_table_cap > 0 && 42903 0 42903
1671 42903 ▶ (hdr->max_table_cap & (hdr->max_table_cap - 1)) == 0 &&
Branch: 1 branch
BranchTrueFalseTotal
(hdr->max_table_cap & (hdr->max_table_cap - 1)) == 0 && 42903 0 42903
1672 42903 ▶ hdr->table_cap > 0 &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->table_cap > 0 && 42903 0 42903
1673 42903 ▶ (hdr->table_cap & (hdr->table_cap - 1)) == 0 &&
Branch: 1 branch
BranchTrueFalseTotal
(hdr->table_cap & (hdr->table_cap - 1)) == 0 && 42902 1 42903
1674 42902 ▶ hdr->table_cap <= hdr->max_table_cap &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->table_cap <= hdr->max_table_cap && 42902 0 42902
1675 42902 ▶ hdr->states_off + hdr->max_table_cap <= hdr->total_size &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->states_off + hdr->max_table_cap <= hdr->total_size && 42902 0 42902
1676 42902 ▶ hdr->nodes_off + (uint64_t)hdr->max_table_cap * hdr->node_size <= hdr->states_off &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->nodes_off + (uint64_t)hdr->max_table_cap * hdr->node_size <= hdr->states_off && 42902 0 42902
1677 42902 ▶ hdr->size <= hdr->table_cap &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->size <= hdr->table_cap && 42881 21 42902
1678 128690 ▶ hdr->tombstones <= hdr->table_cap - hdr->size &&
Branch: 2 branches
BranchTrueFalseTotal
hdr->tombstones <= hdr->table_cap - hdr->size && 42905 0 42905
hdr->tombstones <= hdr->table_cap - hdr->size && 42880 1 42881
1679 42880 ▶ (!hdr->max_size ||
Branch: 1 branch
BranchTrueFalseTotal
(!hdr->max_size || 141 42739 42880
1680 141 ▶ ((hdr->lru_head == SHM_LRU_NONE || hdr->lru_head < hdr->max_table_cap) &&
Branch: 2 branches
BranchTrueFalseTotal
((hdr->lru_head == SHM_LRU_NONE || hdr->lru_head < hdr->max_table_cap) && 6 135 141
((hdr->lru_head == SHM_LRU_NONE || hdr->lru_head < hdr->max_table_cap) && 6 0 6
1681 141 ▶ (hdr->lru_tail == SHM_LRU_NONE || hdr->lru_tail < hdr->max_table_cap))));
Branch: 2 branches
BranchTrueFalseTotal
(hdr->lru_tail == SHM_LRU_NONE || hdr->lru_tail < hdr->max_table_cap)))); 6 135 141
(hdr->lru_tail == SHM_LRU_NONE || hdr->lru_tail < hdr->max_table_cap)))); 6 0 6
1682 }
1683  
1684 /* size + tombstones <= table_cap can read torn on a healthy saturated map, so
1685 * reject only when the seqlock shows no writer was active across the check. */
1686 42905 static inline int shm_validate_header_live(const ShmHeader *hdr,
1687 uint32_t variant_id, uint32_t node_size,
1688 uint64_t map_id) {
1689 42905 ▶ for (int attempt = 0; attempt < 4096; attempt++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int attempt = 0; attempt < 4096; attempt++) { 42905 0 42905
1690 42905 uint32_t s1 = __atomic_load_n(&hdr->seq, __ATOMIC_ACQUIRE);
1691 42905 int ok = shm_validate_header(hdr, variant_id, node_size);
1692 42905 __atomic_thread_fence(__ATOMIC_ACQUIRE);
1693 42905 uint32_t s2 = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
1694 42905 ▶ if (ok) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (ok) return 1; 42880 25 42905
1695 /* Only a live holder can still republish, and a repair scan runs with
1696 * seq even, so seq stability alone proves nothing. */
1697 25 uint32_t w = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
1698 /* shm_holder_blocks_us's rule, by map id */
1699 25 uint32_t wpid = w & SHM_RWLOCK_PID_MASK;
1700 0 ▶ int live_holder = (w >= SHM_RWLOCK_WRITER_BIT) && wpid != 0 &&
Branch: 1 branch
BranchTrueFalseTotal
int live_holder = (w >= SHM_RWLOCK_WRITER_BIT) && wpid != 0 && 0 0 0
1701 25 ▶ (wpid != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id)) &&
Branch: 2 branches
BranchTrueFalseTotal
(wpid != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id)) && 0 25 25
(wpid != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id)) && 0 0 0
1702 0 shm_pid_alive(wpid);
1703 25 ▶ if (!live_holder) {
Branch: 1 branch
BranchTrueFalseTotal
if (!live_holder) { 25 0 25
1704 /* Stable even seq: a real violation. Stable odd: a dead writer's
1705 * leftover, one more read decides. A moved seq means torn: retry. */
1706 25 ▶ if (s1 == s2)
Branch: 1 branch
BranchTrueFalseTotal
if (s1 == s2) 25 0 25
1707 25 ▶ return (s1 & 1) ? shm_validate_header(hdr, variant_id, node_size) : 0;
Branch: 1 branch
BranchTrueFalseTotal
return (s1 & 1) ? shm_validate_header(hdr, variant_id, node_size) : 0; 0 25 25
1708 }
1709 0 ▶ if (attempt >= 8) { struct timespec ts = { 0, 500000 }; nanosleep(&ts, NULL); }
Branch: 1 branch
BranchTrueFalseTotal
if (attempt >= 8) { struct timespec ts = { 0, 500000 }; nanosleep(&ts, NULL); } 0 0 0
1710 }
1711 0 return shm_validate_header(hdr, variant_id, node_size);
1712 }
1713  
1714 4 static inline const char *shm_variant_name(uint32_t id) {
1715 static const char *const names[] = { "?", "I16", "I32", "II", "I16S", "I32S",
1716 "IS", "SI16", "SI32", "SI", "SS" };
1717 4 ▶ return id < sizeof(names) / sizeof(*names) ? names[id] : "?";
Branch: 1 branch
BranchTrueFalseTotal
return id < sizeof(names) / sizeof(*names) ? names[id] : "?"; 4 0 4
1718 }
1719  
1720 31 static inline void shm_format_header_error(char *errbuf, const char *prefix,
1721 const ShmHeader *hdr,
1722 uint32_t variant_id, uint64_t file_size) {
1723 31 ▶ if (!errbuf) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!errbuf) return; 0 31 31
1724 31 ▶ if (hdr->magic != SHM_MAGIC)
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->magic != SHM_MAGIC) 2 29 31
1725 2 snprintf(errbuf, SHM_ERR_BUFLEN, "%s: bad magic (not a HashMap::Shared file)", prefix);
1726 29 ▶ else if (hdr->version != SHM_VERSION)
Branch: 1 branch
BranchTrueFalseTotal
else if (hdr->version != SHM_VERSION) 0 29 29
1727 0 snprintf(errbuf, SHM_ERR_BUFLEN, "%s: version mismatch (file=%u, expected=%u)",
1728 0 prefix, hdr->version, SHM_VERSION);
1729 29 ▶ else if (hdr->variant_id != variant_id)
Branch: 1 branch
BranchTrueFalseTotal
else if (hdr->variant_id != variant_id) 2 27 29
1730 4 snprintf(errbuf, SHM_ERR_BUFLEN, "%s: variant mismatch (the file is %s, not %s)",
1731 2 prefix, shm_variant_name(hdr->variant_id), shm_variant_name(variant_id));
1732 27 ▶ else if (hdr->total_size != file_size)
Branch: 1 branch
BranchTrueFalseTotal
else if (hdr->total_size != file_size) 4 23 27
1733 4 snprintf(errbuf, SHM_ERR_BUFLEN,
1734 "%s: the file is %llu bytes but its header says %llu (a truncated or extended copy?)",
1735 4 prefix, (unsigned long long)file_size, (unsigned long long)hdr->total_size);
1736 else
1737 23 snprintf(errbuf, SHM_ERR_BUFLEN, "%s: corrupt header", prefix);
1738 }
1739  
1740 42873 static inline int shm_validate_layout_regions(ShmLayout *lo, const ShmHeader *hdr,
1741 int has_arena, uint64_t mapped_size,
1742 char *errbuf, const char *prefix) {
1743 42873 int has_lru = (hdr->max_size > 0);
1744 42873 int has_ttl = (hdr->default_ttl > 0);
1745 42873 shm_compute_layout(lo, hdr->max_table_cap, hdr->node_size,
1746 has_lru, has_ttl, has_arena, 0, 0);
1747 42873 ▶ if (lo->end_off > mapped_size) {
Branch: 1 branch
BranchTrueFalseTotal
if (lo->end_off > mapped_size) { 1 42872 42873
1748 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 1 0 1
1749 "%s: file too small for LRU/TTL arrays", prefix);
1750 1 return 0;
1751 }
1752 /* Pointers come from the local layout, never the peer-writable
1753 * hdr->reader_slots_off, which is only sanity-gated. */
1754 42872 uint64_t rs_off = hdr->reader_slots_off;
1755 42872 ▶ if (!rs_off || rs_off < lo->end_off ||
Branch: 2 branches
BranchTrueFalseTotal
if (!rs_off || rs_off < lo->end_off || 42871 1 42872
if (!rs_off || rs_off < lo->end_off || 42871 0 42871
1756 42871 ▶ lo->reader_slots_off + SHM_READER_SLOTS * sizeof(ShmReaderSlot) > mapped_size ||
Branch: 1 branch
BranchTrueFalseTotal
lo->reader_slots_off + SHM_READER_SLOTS * sizeof(ShmReaderSlot) > mapped_size || 42871 0 42871
1757 42871 ▶ lo->occ_off + SHM_OCC_BYTES > mapped_size) {
Branch: 1 branch
BranchTrueFalseTotal
lo->occ_off + SHM_OCC_BYTES > mapped_size) { 0 42871 42871
1758 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 1 0 1
1759 "%s: reader_slots region missing or out of bounds", prefix);
1760 1 return 0;
1761 }
1762 42871 return 1;
1763 }
1764  
1765 46084 static ShmHandle *shm_alloc_handle(void *base, uint64_t total_size,
1766 int has_arena, int has_lru, int has_ttl,
1767 const ShmLayout *lo,
1768 const char *path, int backing_fd,
1769 uint64_t map_id, char *errbuf) {
1770 46084 ShmHeader *hdr = (ShmHeader *)base;
1771 46084 ShmHandle *h = (ShmHandle *)calloc(1, sizeof(ShmHandle));
1772 46084 ▶ if (!h) {
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { 0 46084 46084
1773 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); 0 0 0
1774 0 munmap(base, (size_t)total_size);
1775 0 ▶ if (backing_fd >= 0) close(backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (backing_fd >= 0) close(backing_fd); 0 0 0
1776 0 return NULL;
1777 }
1778 46084 h->hdr = hdr;
1779 46084 h->nodes = (char *)hdr + hdr->nodes_off;
1780 46084 h->states = (uint8_t *)((char *)hdr + hdr->states_off);
1781 46084 ▶ h->arena = has_arena ? (char *)hdr + lo->arena_off : NULL; /* local layout, not the header */
Branch: 1 branch
BranchTrueFalseTotal
h->arena = has_arena ? (char *)hdr + lo->arena_off : NULL; /* local layout, not the header */ 1480 44604 46084
1782 46084 ▶ h->lru_prev = has_lru ? (uint32_t *)((char *)hdr + lo->lru_prev_off) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->lru_prev = has_lru ? (uint32_t *)((char *)hdr + lo->lru_prev_off) : NULL; 1036 45048 46084
1783 46084 ▶ h->lru_next = has_lru ? (uint32_t *)((char *)hdr + lo->lru_next_off) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->lru_next = has_lru ? (uint32_t *)((char *)hdr + lo->lru_next_off) : NULL; 1036 45048 46084
1784 46084 ▶ h->lru_accessed = has_lru ? (uint8_t *)((char *)hdr + lo->lru_accessed_off) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->lru_accessed = has_lru ? (uint8_t *)((char *)hdr + lo->lru_accessed_off) : NULL; 1036 45048 46084
1785 46084 ▶ h->expires_at = has_ttl ? (uint32_t *)((char *)hdr + lo->expires_off) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->expires_at = has_ttl ? (uint32_t *)((char *)hdr + lo->expires_off) : NULL; 499 45585 46084
1786 46084 h->reader_slots = (ShmReaderSlot *)((char *)hdr + lo->reader_slots_off);
1787 46084 h->occ = (uint64_t *)((char *)hdr + lo->occ_off);
1788 46084 h->my_slot_idx = UINT32_MAX;
1789 /* occ_sweep_in stays 0, so the handle's first write lock sweeps. */
1790 46084 h->cached_pid = 0;
1791 46084 h->map_id = map_id;
1792 46084 h->wrlock_ent = SHM_WRLOCK_UNRESOLVED;
1793 46084 h->mmap_size = (size_t)total_size;
1794 46084 h->max_mask = hdr->max_table_cap - 1;
1795 46084 h->iter_pos = 0;
1796 46084 h->backing_fd = backing_fd;
1797 46084 h->sig_count = &shm_sig_none;
1798 46084 (void)madvise(base, (size_t)total_size, MADV_RANDOM);
1799 46084 ▶ if (path) {
Branch: 1 branch
BranchTrueFalseTotal
if (path) { 45629 455 46084
1800 45629 h->path = strdup(path);
1801 45629 ▶ if (!h->path) {
Branch: 1 branch
BranchTrueFalseTotal
if (!h->path) { 0 45629 45629
1802 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory"); 0 0 0
1803 0 munmap(base, (size_t)total_size);
1804 0 ▶ if (backing_fd >= 0) close(backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (backing_fd >= 0) close(backing_fd); 0 0 0
1805 0 free(h);
1806 0 return NULL;
1807 }
1808 }
1809 46084 ▶ if (has_arena) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_arena) { 1480 44604 46084
1810 1480 h->copy_buf = (char *)malloc(256);
1811 1480 ▶ if (h->copy_buf) h->copy_buf_size = 256;
Branch: 1 branch
BranchTrueFalseTotal
if (h->copy_buf) h->copy_buf_size = 256; 1480 0 1480
1812 }
1813 46084 return h;
1814 }
1815  
1816 /* O_NOFOLLOW refuses a symlink at the path as ELOOP, whose text blames a loop. */
1817 7 static const char *shm_open_reason(const char *path, int err) {
1818 struct stat st;
1819 7 ▶ if (err == ELOOP && lstat(path, &st) == 0 && S_ISLNK(st.st_mode))
Branch: 3 branches
BranchTrueFalseTotal
if (err == ELOOP && lstat(path, &st) == 0 && S_ISLNK(st.st_mode)) 3 4 7
if (err == ELOOP && lstat(path, &st) == 0 && S_ISLNK(st.st_mode)) 3 0 3
if (err == ELOOP && lstat(path, &st) == 0 && S_ISLNK(st.st_mode)) 3 0 3
1820 3 return "is a symbolic link";
1821 4 return strerror(err);
1822 }
1823  
1824 45644 static int shm_secure_open(const char *path, mode_t file_mode, char *errbuf) {
1825 45644 ▶ for (int attempt = 0; attempt < 100; attempt++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int attempt = 0; attempt < 100; attempt++) { 45644 0 45644
1826 45644 int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, file_mode);
1827 45644 ▶ if (fd >= 0) { (void)fchmod(fd, file_mode); return fd; } /* undo umask */
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { (void)fchmod(fd, file_mode); return fd; } /* undo umask */ 2863 42781 45644
1828 42781 ▶ if (errno != EEXIST) {
Branch: 1 branch
BranchTrueFalseTotal
if (errno != EEXIST) { 2 42779 42781
1829 2 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "create(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "create(%s): %s", path, strerror(errno)); 2 0 2
1830 2 return -1;
1831 }
1832 42779 fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
1833 42779 ▶ if (fd >= 0) return fd;
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) return fd; 42777 2 42779
1834 2 ▶ if (errno == ENOENT) continue; /* unlinked between the two opens */
Branch: 1 branch
BranchTrueFalseTotal
if (errno == ENOENT) continue; /* unlinked between the two opens */ 0 2 2
1835 2 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, shm_open_reason(path, errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, shm_open_reason(path, errno)); 2 0 2
1836 2 return -1;
1837 }
1838 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): create/attach kept racing", path);
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): create/attach kept racing", path); 0 0 0
1839 0 return -1;
1840 }
1841  
1842 /* Read, not mapped: a read fault on a tmpfs hole allocates the page. */
1843 4 static int shm_file_is_zero(int fd, uint64_t size) {
1844 char buf[16384];
1845 33 ▶ for (uint64_t off = 0; off < size; ) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint64_t off = 0; off < size; ) { 30 3 33
1846 30 ssize_t n = pread(fd, buf, size - off < sizeof buf ? (size_t)(size - off) : sizeof buf, (off_t)off);
1847 30 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 30 30
1848 462365 ▶ for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0;
Branch: 2 branches
BranchTrueFalseTotal
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 1 462335 462336
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 462336 29 462365
1849 29 off += (uint64_t)n;
1850 }
1851 3 return 1;
1852 }
1853  
1854 3252 static int shm_reserving(void) {
1855 3252 const char *sp = getenv("DATA_HASHMAP_SHARED_SPARSE");
1856 3252 ▶ return sp && !strcmp(sp, "0");
Branch: 2 branches
BranchTrueFalseTotal
return sp && !strcmp(sp, "0"); 30 3222 3252
return sp && !strcmp(sp, "0"); 30 0 30
1857 }
1858  
1859 /* No fallocate here (musl passes EOPNOTSUPP through): write a zero byte into
1860 * each block of the still all-zero file, as glibc does. */
1861 0 static int shm_reserve_by_writing(int fd, uint64_t size) {
1862 struct stat st;
1863 0 uint64_t step = 4096;
1864 0 ▶ if (fstat(fd, &st) == 0 && st.st_blksize > 0 && st.st_blksize < 4096 &&
Branch: 3 branches
BranchTrueFalseTotal
if (fstat(fd, &st) == 0 && st.st_blksize > 0 && st.st_blksize < 4096 && 0 0 0
if (fstat(fd, &st) == 0 && st.st_blksize > 0 && st.st_blksize < 4096 && 0 0 0
if (fstat(fd, &st) == 0 && st.st_blksize > 0 && st.st_blksize < 4096 && 0 0 0
1865 0 ▶ !(st.st_blksize & (st.st_blksize - 1)))
Branch: 1 branch
BranchTrueFalseTotal
!(st.st_blksize & (st.st_blksize - 1))) 0 0 0
1866 0 step = (uint64_t)st.st_blksize;
1867 0 ▶ for (uint64_t off = 0; off < size; off += step) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint64_t off = 0; off < size; off += step) { 0 0 0
1868 ssize_t n;
1869 0 ▶ do n = pwrite(fd, "", 1, (off_t)off); while (n < 0 && errno == EINTR);
Branch: 2 branches
BranchTrueFalseTotal
do n = pwrite(fd, "", 1, (off_t)off); while (n < 0 && errno == EINTR); 0 0 0
do n = pwrite(fd, "", 1, (off_t)off); while (n < 0 && errno == EINTR); 0 0 0
1870 0 ▶ if (n != 1) return n < 0 ? errno : EIO;
Branch: 2 branches
BranchTrueFalseTotal
if (n != 1) return n < 0 ? errno : EIO; 0 0 0
if (n != 1) return n < 0 ? errno : EIO; 0 0 0
1871 }
1872 0 return 0;
1873 }
1874  
1875 3021 static int shm_reserve(int fd, uint64_t size) {
1876 3021 ▶ if (!shm_reserving()) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_reserving()) return 0; 2993 28 3021
1877 /* Before Linux 6.11 tmpfs gives up at any pending signal and undoes the allocation, so a
1878 * periodic one could keep it from ever completing; SIGSTOP cannot be blocked. */
1879 sigset_t all, old;
1880 28 sigfillset(&all);
1881 28 sigprocmask(SIG_BLOCK, &all, &old);
1882 int e;
1883 28 ▶ do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR);
Branch: 1 branch
BranchTrueFalseTotal
do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR); 0 28 28
1884 28 sigprocmask(SIG_SETMASK, &old, NULL);
1885 28 ▶ if (e == EOPNOTSUPP || e == EINVAL) e = shm_reserve_by_writing(fd, size);
Branch: 2 branches
BranchTrueFalseTotal
if (e == EOPNOTSUPP || e == EINVAL) e = shm_reserve_by_writing(fd, size); 28 0 28
if (e == EOPNOTSUPP || e == EINVAL) e = shm_reserve_by_writing(fd, size); 0 28 28
1886 28 ▶ if (e == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (e == 0) return 0; 28 0 28
1887 0 errno = e;
1888 0 return -1;
1889 }
1890  
1891 #define SHM_CREATE_RACE_TRIES 250
1892 static const struct timespec shm_create_race_nap = { 0, 2 * 1000 * 1000 };
1893  
1894 45840 static ShmHandle *shm_create_map(const char *path, uint32_t max_entries,
1895 uint32_t node_size, uint32_t variant_id,
1896 int has_arena, uint32_t max_size,
1897 uint32_t default_ttl, uint32_t lru_skip,
1898 uint64_t arena_cap_override, mode_t file_mode, char *errbuf,
1899 int *hold_fd) {
1900 45840 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 45840 0 45840
1901 45840 uint32_t max_tcap = shm_max_tcap_from_entries(max_entries);
1902  
1903 45840 int has_lru = (max_size > 0);
1904 45840 int has_ttl = (default_ttl > 0);
1905  
1906 ShmLayout lo;
1907 45840 shm_compute_layout(&lo, max_tcap, node_size, has_lru, has_ttl, has_arena, max_entries, arena_cap_override);
1908  
1909 #define SHM_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while(0)
1910  
1911 /* Bounds the path every message below prints, so the reason always fits. */
1912 45840 ▶ if (path && strlen(path) >= PATH_MAX) {
Branch: 2 branches
BranchTrueFalseTotal
if (path && strlen(path) >= PATH_MAX) { 45645 195 45840
if (path && strlen(path) >= PATH_MAX) { 1 45644 45645
1913 1 ▶ SHM_ERR("%.64s...: %s", path, strerror(ENAMETOOLONG));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%.64s...: %s", path, strerror(ENAMETOOLONG)); 1 0 1
1914 1 return NULL;
1915 }
1916  
1917 45839 int anonymous = (path == NULL);
1918 45839 int fd = -1;
1919 45839 int is_new, created = 0;
1920 45839 struct stat st = { 0 };
1921 void *base;
1922  
1923 45839 ▶ if (anonymous) {
Branch: 1 branch
BranchTrueFalseTotal
if (anonymous) { 195 45644 45839
1924 195 base = mmap(NULL, lo.total_size, PROT_READ | PROT_WRITE,
1925 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1926 195 ▶ if (base == MAP_FAILED) { SHM_ERR("mmap(anon): %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { SHM_ERR("mmap(anon): %s", strerror(errno)); return NULL; } 0 195 195
if (base == MAP_FAILED) { SHM_ERR("mmap(anon): %s", strerror(errno)); return NULL; } 0 0 0
1927 195 is_new = 1;
1928 } else {
1929 45644 fd = shm_secure_open(path, file_mode, errbuf);
1930 45644 ▶ if (fd < 0) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return NULL; 4 45640 45644
1931  
1932 45640 for (int tries = 0;; tries++) {
1933 45640 ▶ while (flock(fd, LOCK_EX) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
while (flock(fd, LOCK_EX) < 0) { 0 45640 45640
1934 0 ▶ if (errno == EINTR) continue;
Branch: 1 branch
BranchTrueFalseTotal
if (errno == EINTR) continue; 0 0 0
1935 0 ▶ SHM_ERR("flock(%s): %s", path, strerror(errno)); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("flock(%s): %s", path, strerror(errno)); close(fd); return NULL; 0 0 0
1936 }
1937  
1938 45640 ▶ if (fstat(fd, &st) < 0) { SHM_ERR("fstat(%s): %s", path, strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { SHM_ERR("fstat(%s): %s", path, strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 45640 45640
if (fstat(fd, &st) < 0) { SHM_ERR("fstat(%s): %s", path, strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 0 0
1939  
1940 45640 is_new = created = (st.st_size == 0);
1941 /* Another user's empty file is their create between open and flock. */
1942 45640 ▶ if (!is_new || st.st_uid == geteuid() || tries == SHM_CREATE_RACE_TRIES) break;
Branch: 3 branches
BranchTrueFalseTotal
if (!is_new || st.st_uid == geteuid() || tries == SHM_CREATE_RACE_TRIES) break; 2884 42756 45640
if (!is_new || st.st_uid == geteuid() || tries == SHM_CREATE_RACE_TRIES) break; 0 2884 2884
if (!is_new || st.st_uid == geteuid() || tries == SHM_CREATE_RACE_TRIES) break; 0 0 0
1943 0 flock(fd, LOCK_UN);
1944 0 nanosleep(&shm_create_race_nap, NULL);
1945 }
1946  
1947 45640 ▶ if (!is_new && (uint64_t)st.st_size < sizeof(ShmHeader)) {
Branch: 2 branches
BranchTrueFalseTotal
if (!is_new && (uint64_t)st.st_size < sizeof(ShmHeader)) { 42756 2884 45640
if (!is_new && (uint64_t)st.st_size < sizeof(ShmHeader)) { 1 42755 42756
1948 1 ▶ SHM_ERR("%s: file too small (%lld bytes, need %zu)", path,
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: file too small (%lld bytes, need %zu)", path, 1 0 1
1949 (long long)st.st_size, sizeof(ShmHeader));
1950 1 flock(fd, LOCK_UN); close(fd); return NULL;
1951 }
1952  
1953 45639 ▶ if (is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_new) { 2884 42755 45639
1954 2884 ▶ if (st.st_uid != geteuid()) {
Branch: 1 branch
BranchTrueFalseTotal
if (st.st_uid != geteuid()) { 0 2884 2884
1955 0 ▶ SHM_ERR("%s: refusing to initialize file not owned by us", path);
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: refusing to initialize file not owned by us", path); 0 0 0
1956 0 flock(fd, LOCK_UN); close(fd); return NULL;
1957 }
1958 2884 ▶ if (fchmod(fd, file_mode) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fchmod(fd, file_mode) < 0) { 0 2884 2884
1959 0 ▶ SHM_ERR("fchmod(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("fchmod(%s): %s", path, strerror(errno)); 0 0 0
1960 0 flock(fd, LOCK_UN); close(fd); return NULL;
1961 }
1962 2884 ▶ if (ftruncate(fd, (off_t)lo.total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)lo.total_size) < 0) { 0 2884 2884
1963 0 ▶ SHM_ERR("ftruncate(%s, %llu): %s", path, (unsigned long long)lo.total_size, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("ftruncate(%s, %llu): %s", path, (unsigned long long)lo.total_size, strerror(errno)); 0 0 0
1964 0 flock(fd, LOCK_UN); close(fd); return NULL;
1965 }
1966 2884 ▶ if (shm_reserve(fd, lo.total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (shm_reserve(fd, lo.total_size) < 0) { 0 2884 2884
1967 0 ▶ SHM_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)lo.total_size, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)lo.total_size, strerror(errno)); 0 0 0
1968 0 if (ftruncate(fd, 0) < 0) { /* best effort */ }
1969 0 flock(fd, LOCK_UN); close(fd); return NULL;
1970 }
1971 }
1972  
1973 45639 ▶ base = mmap(NULL, is_new ? lo.total_size : (size_t)st.st_size,
Branch: 1 branch
BranchTrueFalseTotal
base = mmap(NULL, is_new ? lo.total_size : (size_t)st.st_size, 2884 42755 45639
1974 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1975 45639 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 1 45638 45639
1976 1 ▶ SHM_ERR("mmap(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("mmap(%s): %s", path, strerror(errno)); 1 0 1
1977 1 ▶ if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ }
Branch: 1 branch
BranchTrueFalseTotal
if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ } 1 0 1
1978 1 flock(fd, LOCK_UN); close(fd); return NULL;
1979 }
1980 }
1981  
1982 45833 ShmHeader *hdr = (ShmHeader *)base;
1983 45833 uint64_t map_id = shm_map_id_of(st.st_dev, st.st_ino, base);
1984 45833 ▶ uint64_t mapped_size = is_new ? lo.total_size : (uint64_t)st.st_size;
Branch: 1 branch
BranchTrueFalseTotal
uint64_t mapped_size = is_new ? lo.total_size : (uint64_t)st.st_size; 3078 42755 45833
1985  
1986 45833 ▶ if (is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_new) { 3078 42755 45833
1987 3078 shm_init_header(hdr, base, &lo, max_tcap, node_size, variant_id,
1988 has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);
1989 } else {
1990 85502 int ok = (hdr->total_size == (uint64_t)st.st_size &&
1991 42747 shm_validate_header_live(hdr, variant_id, node_size, map_id));
1992 42755 ▶ if (ok) {
Branch: 1 branch
BranchTrueFalseTotal
if (ok) { 42723 32 42755
1993 42723 has_lru = (hdr->max_size > 0);
1994 42723 has_ttl = (hdr->default_ttl > 0);
1995 42723 ok = shm_validate_layout_regions(&lo, hdr, has_arena, mapped_size, errbuf, path);
1996 32 ▶ } else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size
Branch: 2 branches
BranchTrueFalseTotal
} else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size 5 27 32
} else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size 4 1 5
1997 4 ▶ && st.st_uid == geteuid()
Branch: 1 branch
BranchTrueFalseTotal
&& st.st_uid == geteuid() 4 0 4
1998 4 ▶ && shm_file_is_zero(fd, (uint64_t)st.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
&& shm_file_is_zero(fd, (uint64_t)st.st_size)) { 3 1 4
1999 /* A creator killed before the header init left an all-zero file. */
2000 3 ▶ if (fchmod(fd, file_mode) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fchmod(fd, file_mode) < 0) { 0 3 3
2001 0 ▶ SHM_ERR("%s: fchmod: %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: fchmod: %s", path, strerror(errno)); 0 0 0
2002 0 munmap(base, (size_t)st.st_size); flock(fd, LOCK_UN); close(fd); return NULL;
2003 }
2004 3 ▶ if (shm_reserve(fd, lo.total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (shm_reserve(fd, lo.total_size) < 0) { 0 3 3
2005 0 ▶ SHM_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)lo.total_size, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)lo.total_size, strerror(errno)); 0 0 0
2006 0 munmap(base, (size_t)st.st_size);
2007 0 if (ftruncate(fd, 0) < 0) { /* best effort */ }
2008 0 flock(fd, LOCK_UN); close(fd); return NULL;
2009 }
2010 3 shm_init_header(hdr, base, &lo, max_tcap, node_size, variant_id,
2011 has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);
2012 3 ok = created = 1;
2013 29 ▶ } else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size
Branch: 2 branches
BranchTrueFalseTotal
} else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size 2 27 29
} else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size 1 1 2
2014 1 ▶ && st.st_uid == geteuid()) {
Branch: 1 branch
BranchTrueFalseTotal
&& st.st_uid == geteuid()) { 1 0 1
2015 /* A creator died between the field stores and the magic commit. */
2016 1 ▶ SHM_ERR("%s: incomplete map file left by an interrupted create; remove it and retry", path);
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s: incomplete map file left by an interrupted create; remove it and retry", path); 1 0 1
2017 } else {
2018 28 shm_format_header_error(errbuf, path, hdr, variant_id, (uint64_t)st.st_size);
2019 }
2020 42755 ▶ if (!ok) {
Branch: 1 branch
BranchTrueFalseTotal
if (!ok) { 31 42724 42755
2021 31 munmap(base, (size_t)st.st_size);
2022 31 flock(fd, LOCK_UN); close(fd);
2023 31 return NULL;
2024 }
2025 42724 ▶ if (hdr->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sealed) { 2 42722 42724
2026 2 ▶ SHM_ERR("%s is frozen (read-only); open it with new_readonly", path);
Branch: 1 branch
BranchTrueFalseTotal
SHM_ERR("%s is frozen (read-only); open it with new_readonly", path); 2 0 2
2027 2 munmap(base, (size_t)st.st_size);
2028 2 flock(fd, LOCK_UN); close(fd);
2029 2 return NULL;
2030 }
2031 }
2032  
2033 #undef SHM_ERR
2034 45800 ShmHandle *nh = shm_alloc_handle(base, mapped_size, has_arena, has_lru, has_ttl,
2035 &lo, path, -1, map_id, errbuf);
2036 45800 ▶ if (nh) { nh->file_dev = st.st_dev; nh->file_ino = st.st_ino; }
Branch: 1 branch
BranchTrueFalseTotal
if (nh) { nh->file_dev = st.st_dev; nh->file_ino = st.st_ino; } 45800 0 45800
2037 45800 ▶ if (fd >= 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { 45605 195 45800
2038 45605 ▶ if (nh && hold_fd && created) *hold_fd = fd; /* still locked: the caller releases it */
Branch: 3 branches
BranchTrueFalseTotal
if (nh && hold_fd && created) *hold_fd = fd; /* still locked: the caller releases it */ 45605 0 45605
if (nh && hold_fd && created) *hold_fd = fd; /* still locked: the caller releases it */ 25 45580 45605
if (nh && hold_fd && created) *hold_fd = fd; /* still locked: the caller releases it */ 25 0 25
2039 45580 else { flock(fd, LOCK_UN); close(fd); }
2040 }
2041 45800 return nh;
2042 }
2043  
2044 134 static ShmHandle *shm_create_memfd(const char *name, uint32_t max_entries,
2045 uint32_t node_size, uint32_t variant_id,
2046 int has_arena, uint32_t max_size,
2047 uint32_t default_ttl, uint32_t lru_skip,
2048 uint64_t arena_cap_override, char *errbuf) {
2049 134 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 134 0 134
2050  
2051 134 uint32_t max_tcap = shm_max_tcap_from_entries(max_entries);
2052 134 int has_lru = (max_size > 0);
2053 134 int has_ttl = (default_ttl > 0);
2054  
2055 ShmLayout lo;
2056 134 shm_compute_layout(&lo, max_tcap, node_size, has_lru, has_ttl, has_arena, max_entries, arena_cap_override);
2057  
2058 134 ▶ int fd = shm_memfd_create(name ? name : "hashmap", MFD_CLOEXEC | MFD_ALLOW_SEALING);
Branch: 1 branch
BranchTrueFalseTotal
int fd = shm_memfd_create(name ? name : "hashmap", MFD_CLOEXEC | MFD_ALLOW_SEALING); 133 1 134
2059 134 ▶ if (fd < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) { 0 134 134
2060 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "memfd_create: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "memfd_create: %s", strerror(errno)); 0 0 0
2061 0 return NULL;
2062 }
2063 134 ▶ if (ftruncate(fd, (off_t)lo.total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)lo.total_size) < 0) { 0 134 134
2064 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "ftruncate: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "ftruncate: %s", strerror(errno)); 0 0 0
2065 0 close(fd); return NULL;
2066 }
2067 134 ▶ if (shm_reserve(fd, lo.total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (shm_reserve(fd, lo.total_size) < 0) { 0 134 134
2068 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "memfd: cannot reserve %llu bytes: %s", (unsigned long long)lo.total_size, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "memfd: cannot reserve %llu bytes: %s", (unsigned long long)lo.total_size, strerror(errno)); 0 0 0
2069 0 close(fd); return NULL;
2070 }
2071 134 (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);
2072 134 void *base = mmap(NULL, (size_t)lo.total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2073 134 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 134 134
2074 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno)); 0 0 0
2075 0 close(fd); return NULL;
2076 }
2077  
2078 134 shm_init_header((ShmHeader *)base, base, &lo, max_tcap, node_size, variant_id,
2079 has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);
2080  
2081 struct stat mst;
2082 268 uint64_t map_id = fstat(fd, &mst) == 0 ? shm_map_id_of(mst.st_dev, mst.st_ino, base)
2083 134 ▶ : shm_map_id_of(0, 0, base);
Branch: 1 branch
BranchTrueFalseTotal
: shm_map_id_of(0, 0, base); 134 0 134
2084 134 return shm_alloc_handle(base, lo.total_size, has_arena, has_lru, has_ttl,
2085 &lo, NULL, fd, map_id, errbuf);
2086 }
2087  
2088 130 static ShmHandle *shm_open_fd_map(int fd, uint32_t variant_id, uint32_t node_size,
2089 char *errbuf) {
2090 130 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 130 0 130
2091 struct stat st;
2092 130 ▶ if (fstat(fd, &st) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { 0 130 130
2093 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat: %s", strerror(errno)); 0 0 0
2094 0 return NULL;
2095 }
2096 130 ▶ if ((uint64_t)st.st_size < sizeof(ShmHeader)) {
Branch: 1 branch
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(ShmHeader)) { 1 129 130
2097 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: file too small for header");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: file too small for header"); 1 0 1
2098 1 return NULL;
2099 }
2100 /* before mmap: a creator may truncate the file until it has set magic */
2101 uint32_t magic;
2102 129 ssize_t got = pread(fd, &magic, sizeof magic, offsetof(ShmHeader, magic));
2103 129 ▶ if (got < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (got < 0) { 1 128 129
2104 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: read: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: read: %s", strerror(errno)); 1 0 1
2105 1 return NULL;
2106 }
2107 128 ▶ if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) { 128 0 128
if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) { 0 128 128
2108 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: bad magic (not a HashMap::Shared file)");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: bad magic (not a HashMap::Shared file)"); 0 0 0
2109 0 return NULL;
2110 }
2111 128 size_t ms = (size_t)st.st_size;
2112 128 void *base = mmap(NULL, ms, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
2113 128 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 128 128
2114 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno)); 0 0 0
2115 0 return NULL;
2116 }
2117  
2118 128 ShmHeader *hdr = (ShmHeader *)base;
2119 128 uint64_t map_id = shm_map_id_of(st.st_dev, st.st_ino, base);
2120 255 if (hdr->total_size != (uint64_t)st.st_size ||
2121 127 !shm_validate_header_live(hdr, variant_id, node_size, map_id)) {
2122 1 shm_format_header_error(errbuf, "fd", hdr, variant_id, (uint64_t)st.st_size);
2123 1 munmap(base, ms);
2124 1 return NULL;
2125 }
2126 127 ▶ if (hdr->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sealed) { 1 126 127
2127 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 1 0 1
2128 "this map is frozen (read-only); open a file with new_readonly; "
2129 "a frozen memfd cannot be reopened");
2130 1 munmap(base, ms);
2131 1 return NULL;
2132 }
2133  
2134 126 int has_arena = (hdr->arena_off != 0);
2135 126 int has_lru = (hdr->max_size > 0);
2136 126 int has_ttl = (hdr->default_ttl > 0);
2137 ShmLayout lo;
2138 126 ▶ if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, "fd")) {
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, "fd")) { 0 126 126
2139 0 munmap(base, ms);
2140 0 return NULL;
2141 }
2142  
2143 126 int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
2144 126 ▶ if (myfd < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (myfd < 0) { 0 126 126
2145 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fcntl: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fcntl: %s", strerror(errno)); 0 0 0
2146 0 munmap(base, ms);
2147 0 return NULL;
2148 }
2149 126 return shm_alloc_handle(base, hdr->total_size, has_arena, has_lru, has_ttl,
2150 &lo, NULL, myfd, map_id, errbuf);
2151 }
2152  
2153 /* A non-frozen file is refused: lock-free readers would race a live writer. */
2154 39 static ShmHandle *shm_open_readonly_map(const char *path, uint32_t variant_id,
2155 uint32_t node_size, char *errbuf) {
2156 39 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 39 0 39
2157 39 ▶ if (strlen(path) >= PATH_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (strlen(path) >= PATH_MAX) { 1 38 39
2158 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%.64s...: %s", path, strerror(ENAMETOOLONG));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%.64s...: %s", path, strerror(ENAMETOOLONG)); 1 0 1
2159 1 return NULL;
2160 }
2161 38 int fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
2162 38 ▶ if (fd < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) { 5 33 38
2163 5 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, shm_open_reason(path, errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, shm_open_reason(path, errno)); 5 0 5
2164 5 return NULL;
2165 }
2166 struct stat st;
2167 33 ▶ if (fstat(fd, &st) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { 0 33 33
2168 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat(%s): %s", path, strerror(errno)); 0 0 0
2169 0 close(fd); return NULL;
2170 }
2171 33 ▶ if ((uint64_t)st.st_size < sizeof(ShmHeader)) {
Branch: 1 branch
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(ShmHeader)) { 1 32 33
2172 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: file too small for header", path);
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: file too small for header", path); 1 0 1
2173 1 close(fd); return NULL;
2174 }
2175 /* before mmap: a creator may truncate the file until it has set magic */
2176 uint32_t magic;
2177 32 ssize_t got = pread(fd, &magic, sizeof magic, offsetof(ShmHeader, magic));
2178 32 ▶ if (got < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (got < 0) { 0 32 32
2179 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: read: %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: read: %s", path, strerror(errno)); 0 0 0
2180 0 close(fd); return NULL;
2181 }
2182 32 ▶ if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) { 32 0 32
if (got != (ssize_t)sizeof magic || magic != SHM_MAGIC) { 0 32 32
2183 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: bad magic (not a HashMap::Shared file)", path);
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: bad magic (not a HashMap::Shared file)", path); 0 0 0
2184 0 close(fd); return NULL;
2185 }
2186 32 size_t ms = (size_t)st.st_size;
2187 32 void *base = mmap(NULL, ms, PROT_READ, MAP_SHARED, fd, 0);
2188 32 int mmap_errno = errno;
2189 32 close(fd);
2190 32 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 32 32
2191 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap(%s): %s", path, strerror(mmap_errno));
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap(%s): %s", path, strerror(mmap_errno)); 0 0 0
2192 0 return NULL;
2193 }
2194 32 ShmHeader *hdr = (ShmHeader *)base;
2195 32 uint64_t map_id = shm_map_id_of(st.st_dev, st.st_ino, base);
2196 63 if (hdr->total_size != (uint64_t)st.st_size ||
2197 31 !shm_validate_header_live(hdr, variant_id, node_size, map_id)) {
2198 2 shm_format_header_error(errbuf, path, hdr, variant_id, (uint64_t)st.st_size);
2199 2 munmap(base, ms);
2200 2 return NULL;
2201 }
2202 30 ▶ if (!hdr->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (!hdr->sealed) { 2 28 30
2203 2 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 2 0 2
2204 "%s is not frozen: call ->freeze on the producer before opening read-only", path);
2205 2 munmap(base, ms);
2206 2 return NULL;
2207 }
2208  
2209 /* Unrepairable: a read-write attach refuses a sealed file. freeze seals
2210 * under the lock, so a live holder will still finish. */
2211 28 ▶ if ((hdr->seq & 1) || hdr->wlock != 0) {
Branch: 2 branches
BranchTrueFalseTotal
if ((hdr->seq & 1) || hdr->wlock != 0) { 27 1 28
if ((hdr->seq & 1) || hdr->wlock != 0) { 3 24 27
2212 4 uint32_t holder = hdr->wlock & SHM_RWLOCK_PID_MASK;
2213 4 ▶ if (errbuf) {
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) { 4 0 4
2214 4 ▶ if (holder && shm_pid_alive(holder) &&
Branch: 2 branches
BranchTrueFalseTotal
if (holder && shm_pid_alive(holder) && 3 1 4
if (holder && shm_pid_alive(holder) && 2 1 3
2215 2 ▶ (holder != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id)))
Branch: 2 branches
BranchTrueFalseTotal
(holder != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id))) 1 1 2
(holder != (uint32_t)getpid() || shm_wrlock_held_by_id(map_id))) 0 1 1
2216 1 snprintf(errbuf, SHM_ERR_BUFLEN,
2217 "%s: a write is still in flight on this frozen file (pid %u); retry",
2218 path, holder);
2219 else
2220 3 snprintf(errbuf, SHM_ERR_BUFLEN,
2221 "%s: frozen file was left mid-update by a crashed writer; recreate it", path);
2222 }
2223 4 munmap(base, ms);
2224 4 return NULL;
2225 }
2226 24 int has_arena = (hdr->arena_off != 0);
2227 24 int has_lru = (hdr->max_size > 0);
2228 24 int has_ttl = (hdr->default_ttl > 0);
2229 ShmLayout lo;
2230 24 ▶ if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, path)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, path)) { 0 24 24
2231 0 munmap(base, ms);
2232 0 return NULL;
2233 }
2234 24 ShmHandle *h = shm_alloc_handle(base, hdr->total_size, has_arena, has_lru, has_ttl,
2235 &lo, path, -1, map_id,
2236 errbuf); /* munmaps + frees on OOM */
2237 24 ▶ if (!h) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return NULL; 0 24 24
2238 24 h->file_dev = st.st_dev;
2239 24 h->file_ino = st.st_ino;
2240 24 h->readonly = 1;
2241 24 return h;
2242 }
2243  
2244 43 static inline int shm_msync(ShmHandle *h) {
2245 43 ▶ if (!h) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return 0; 0 43 43
2246 43 ▶ if (h->readonly) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (h->readonly) return 0; 1 42 43
2247 42 ▶ if (h->shard_handles) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles) { 2 40 42
2248 10 ▶ for (uint32_t i = 0; i < h->num_shards; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) { 8 2 10
2249 int rc;
2250 8 do { rc = msync(h->shard_handles[i]->hdr,
2251 8 h->shard_handles[i]->mmap_size, MS_SYNC); }
2252 8 ▶ while (rc != 0 && errno == EINTR);
Branch: 2 branches
BranchTrueFalseTotal
while (rc != 0 && errno == EINTR); 0 8 8
while (rc != 0 && errno == EINTR); 0 0 0
2253 8 ▶ if (rc != 0) return rc;
Branch: 1 branch
BranchTrueFalseTotal
if (rc != 0) return rc; 0 8 8
2254 }
2255 2 return 0;
2256 }
2257 40 ▶ if (!h->hdr) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!h->hdr) return 0; 0 40 40
2258 int rc;
2259 40 do { rc = msync(h->hdr, h->mmap_size, MS_SYNC); }
2260 40 ▶ while (rc != 0 && errno == EINTR);
Branch: 2 branches
BranchTrueFalseTotal
while (rc != 0 && errno == EINTR); 0 40 40
while (rc != 0 && errno == EINTR); 0 0 0
2261 40 return rc;
2262 }
2263  
2264 40 static void shm_seal(ShmHandle *h) {
2265 40 shm_rwlock_wrlock(h);
2266 40 __atomic_store_n(&h->hdr->sealed, 1, __ATOMIC_RELEASE);
2267 40 shm_rwlock_wrunlock(h);
2268 40 }
2269  
2270 /* Seal every shard before flushing any, so a failed flush leaves the set sealed.
2271 * shm_msync skips a read-only handle, so shm_mark_readonly comes after this. */
2272 34 static int shm_freeze(ShmHandle *h) {
2273 34 ▶ if (!h) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return 0; 0 34 34
2274 34 ▶ uint32_t n = h->shard_handles ? h->num_shards : 1;
Branch: 1 branch
BranchTrueFalseTotal
uint32_t n = h->shard_handles ? h->num_shards : 1; 2 32 34
2275 34 ▶ ShmHandle **hs = h->shard_handles ? h->shard_handles : &h;
Branch: 1 branch
BranchTrueFalseTotal
ShmHandle **hs = h->shard_handles ? h->shard_handles : &h; 2 32 34
2276 74 ▶ for (uint32_t i = 0; i < n; i++) shm_seal(hs[i]);
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < n; i++) shm_seal(hs[i]); 40 34 74
2277 34 ▶ if (!h->path && h->backing_fd < 0) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (!h->path && h->backing_fd < 0) return 0; 0 34 34
if (!h->path && h->backing_fd < 0) return 0; 0 0 0
2278 34 return shm_msync(h);
2279 }
2280  
2281 34 static void shm_mark_readonly(ShmHandle *h) {
2282 34 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 34 34
2283 34 h->readonly = 1;
2284 34 ▶ if (h->shard_handles)
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles) 2 32 34
2285 10 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 8 2 10
2286 8 ▶ if (h->shard_handles[i]) h->shard_handles[i]->readonly = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles[i]) h->shard_handles[i]->readonly = 1; 8 0 8
2287 }
2288  
2289 /* The shared header, not h->readonly, which misses a freeze through another
2290 * handle. freeze seals shard 0 first. */
2291 404734 static inline int shm_is_sealed(const ShmHandle *h) {
2292 404734 ▶ if (!h) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return 0; 0 404734 404734
2293 404734 ▶ if (h->shard_handles) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles) { 49120 355614 404734
2294 49120 ▶ const ShmHandle *s = h->num_shards ? h->shard_handles[0] : NULL;
Branch: 1 branch
BranchTrueFalseTotal
const ShmHandle *s = h->num_shards ? h->shard_handles[0] : NULL; 49120 0 49120
2295 49120 ▶ return s && s->hdr && __atomic_load_n(&s->hdr->sealed, __ATOMIC_RELAXED);
Branch: 3 branches
BranchTrueFalseTotal
return s && s->hdr && __atomic_load_n(&s->hdr->sealed, __ATOMIC_RELAXED); 49120 0 49120
return s && s->hdr && __atomic_load_n(&s->hdr->sealed, __ATOMIC_RELAXED); 49120 0 49120
return s && s->hdr && __atomic_load_n(&s->hdr->sealed, __ATOMIC_RELAXED); 3 49117 49120
2296 }
2297 355614 ▶ return h->hdr && __atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED);
Branch: 2 branches
BranchTrueFalseTotal
return h->hdr && __atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED); 355614 0 355614
return h->hdr && __atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED); 297 355317 355614
2298 }
2299  
2300 /* A freezer killed between shard seals leaves a partly sealed set that only
2301 * another freeze can finish. */
2302 45 static inline int shm_is_fully_sealed(const ShmHandle *h) {
2303 45 ▶ if (!h->shard_handles) return shm_is_sealed(h);
Branch: 1 branch
BranchTrueFalseTotal
if (!h->shard_handles) return shm_is_sealed(h); 42 3 45
2304 8 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 7 1 8
2305 7 ▶ if (!__atomic_load_n(&h->shard_handles[i]->hdr->sealed, __ATOMIC_RELAXED)) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_load_n(&h->shard_handles[i]->hdr->sealed, __ATOMIC_RELAXED)) return 0; 2 5 7
2306 1 return 1;
2307 }
2308  
2309 static void shm_close_map_now(ShmHandle *h);
2310  
2311 /* Defer the free to the last unlock: the save-stack cleanup still uses the
2312 * handle. No atomics: CLONE_SKIP keeps a handle in one thread. */
2313 46306 static void shm_close_map(ShmHandle *h) {
2314 46306 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 46306 46306
2315 46306 ▶ if (h->lock_depth) { h->pending_close = 1; return; }
Branch: 1 branch
BranchTrueFalseTotal
if (h->lock_depth) { h->pending_close = 1; return; } 0 46306 46306
2316 46306 shm_close_map_now(h);
2317 }
2318  
2319 46306 static void shm_close_map_now(ShmHandle *h) {
2320 46306 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 46306 46306
2321 46306 ▶ if (h->shard_handles) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles) { 222 46084 46306
2322 2291 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 2069 222 2291
2323 2069 shm_close_map(h->shard_handles[i]);
2324 222 free(h->shard_handles);
2325 222 free(h->path);
2326 222 free(h);
2327 222 return;
2328 }
2329 46084 ▶ if (h->hdr && !h->readonly &&
Branch: 2 branches
BranchTrueFalseTotal
if (h->hdr && !h->readonly && 46084 0 46084
if (h->hdr && !h->readonly && 46020 64 46084
2330 46020 ▶ h->cached_fork_gen == __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED))
Branch: 1 branch
BranchTrueFalseTotal
h->cached_fork_gen == __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED)) 2255 43765 46020
2331 2255 shm_rdlock_drop_unwound(h);
2332 /* Not after a fork: a child must not clear the parent's slot. A slot with
2333 * a read lock still held must survive for recovery. */
2334 46084 ▶ if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid &&
Branch: 3 branches
BranchTrueFalseTotal
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 46084 0 46084
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 2295 43789 46084
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 2295 0 2295
2335 2295 ▶ h->cached_fork_gen == __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED) &&
Branch: 1 branch
BranchTrueFalseTotal
h->cached_fork_gen == __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED) && 2295 0 2295
2336 2295 ▶ __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
__atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) { 2295 0 2295
2337 /* Occ bit first, while we still own the pid. */
2338 2295 shm_occ_clear(h, h->my_slot_idx);
2339 2295 uint32_t expected = h->cached_pid;
2340 /* Leave rdepth alone: a store after the CAS could clobber a new
2341 * claimant's increments. The next claim zeroes it. */
2342 2295 __atomic_compare_exchange_n(&h->reader_slots[h->my_slot_idx].pid,
2343 &expected, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
2344 }
2345 46084 ▶ if (h->hdr) munmap(h->hdr, h->mmap_size);
Branch: 1 branch
BranchTrueFalseTotal
if (h->hdr) munmap(h->hdr, h->mmap_size); 46084 0 46084
2346 46084 ▶ if (h->backing_fd >= 0) close(h->backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (h->backing_fd >= 0) close(h->backing_fd); 260 45824 46084
2347 46084 free(h->copy_buf);
2348 46084 free(h->path);
2349 46084 free(h);
2350 }
2351  
2352 /* held[j] >= 0: shard j was created here and is still locked, so nobody has
2353 * attached it; truncate it back to an abandoned create. */
2354 9 static ShmHandle *shm_sharded_fail(ShmHandle *h, uint32_t created, int *held) {
2355 46 ▶ for (uint32_t j = 0; j < created; j++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t j = 0; j < created; j++) { 37 9 46
2356 37 shm_close_map(h->shard_handles[j]);
2357 37 ▶ if (held && held[j] >= 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (held && held[j] >= 0) { 21 16 37
if (held && held[j] >= 0) { 21 0 21
2358 21 if (ftruncate(held[j], 0) < 0) { /* best effort */ }
2359 21 flock(held[j], LOCK_UN);
2360 21 close(held[j]);
2361 }
2362 }
2363 9 free(held);
2364 9 free(h->shard_handles);
2365 9 free(h->path);
2366 9 free(h);
2367 9 return NULL;
2368 }
2369  
2370 234 static ShmHandle *shm_create_sharded(const char *path_prefix, uint32_t num_shards,
2371 uint32_t max_entries, uint32_t node_size,
2372 uint32_t variant_id, int has_arena,
2373 uint32_t max_size, uint32_t default_ttl,
2374 uint32_t lru_skip, uint64_t arena_cap_override,
2375 mode_t file_mode, char *errbuf) {
2376 234 ▶ if (!path_prefix) {
Branch: 1 branch
BranchTrueFalseTotal
if (!path_prefix) { 1 233 234
2377 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "new_sharded requires a path_prefix");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "new_sharded requires a path_prefix"); 1 0 1
2378 1 return NULL;
2379 }
2380 233 ▶ if (num_shards > SHM_MAX_SHARDS) {
Branch: 1 branch
BranchTrueFalseTotal
if (num_shards > SHM_MAX_SHARDS) { 2 231 233
2381 2 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 2 0 2
2382 "num_shards %u exceeds the maximum of %u",
2383 num_shards, (unsigned)SHM_MAX_SHARDS);
2384 2 return NULL;
2385 }
2386 231 uint32_t ns = 1;
2387 231 uint8_t want_log2 = 1;
2388 691 ▶ while (ns < num_shards) { ns <<= 1; want_log2++; }
Branch: 1 branch
BranchTrueFalseTotal
while (ns < num_shards) { ns <<= 1; want_log2++; } 460 231 691
2389 231 num_shards = ns;
2390  
2391 231 int *held = NULL; /* reserving: the still-locked fd of each shard this call creates */
2392 231 ShmHandle *h = (ShmHandle *)calloc(1, sizeof(ShmHandle));
2393 231 ▶ if (!h) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (!h) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return NULL; } 0 231 231
if (!h) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return NULL; } 0 0 0
2394  
2395 231 h->shard_handles = (ShmHandle **)calloc(num_shards, sizeof(ShmHandle *));
2396 231 ▶ if (!h->shard_handles) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return shm_sharded_fail(h, 0, held); }
Branch: 2 branches
BranchTrueFalseTotal
if (!h->shard_handles) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return shm_sharded_fail(h, 0, held); } 0 231 231
if (!h->shard_handles) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return shm_sharded_fail(h, 0, held); } 0 0 0
2397  
2398 231 h->num_shards = num_shards;
2399 231 h->shard_mask = num_shards - 1;
2400 231 h->backing_fd = -1; /* calloc left 0 = stdin */
2401 231 h->sig_count = &shm_sig_none;
2402 231 h->path = strdup(path_prefix);
2403 231 ▶ if (!h->path) {
Branch: 1 branch
BranchTrueFalseTotal
if (!h->path) { 0 231 231
2404 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory"); 0 0 0
2405 0 return shm_sharded_fail(h, 0, held);
2406 }
2407  
2408 231 ▶ if (shm_reserving()) {
Branch: 1 branch
BranchTrueFalseTotal
if (shm_reserving()) { 2 229 231
2409 2 held = (int *)malloc(num_shards * sizeof(int));
2410 2 ▶ if (!held) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "malloc: out of memory"); return shm_sharded_fail(h, 0, held); }
Branch: 2 branches
BranchTrueFalseTotal
if (!held) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "malloc: out of memory"); return shm_sharded_fail(h, 0, held); } 0 2 2
if (!held) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "malloc: out of memory"); return shm_sharded_fail(h, 0, held); } 0 0 0
2411 70 ▶ for (uint32_t i = 0; i < num_shards; i++) held[i] = -1;
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < num_shards; i++) held[i] = -1; 68 2 70
2412 }
2413 2331 ▶ for (uint32_t i = 0; i < num_shards; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < num_shards; i++) { 2109 222 2331
2414 char shard_path[4096];
2415 2109 int sn = snprintf(shard_path, sizeof(shard_path), "%s.%u", path_prefix, i);
2416 2109 ▶ if (sn < 0 || sn >= (int)sizeof(shard_path)) {
Branch: 2 branches
BranchTrueFalseTotal
if (sn < 0 || sn >= (int)sizeof(shard_path)) { 2109 0 2109
if (sn < 0 || sn >= (int)sizeof(shard_path)) { 1 2108 2109
2417 1 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "shard path too long");
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "shard path too long"); 1 0 1
2418 9 return shm_sharded_fail(h, i, held);
2419 }
2420 2134 ▶ h->shard_handles[i] = shm_create_map(shard_path, max_entries, node_size,
Branch: 1 branch
BranchTrueFalseTotal
h->shard_handles[i] = shm_create_map(shard_path, max_entries, node_size, 26 2082 2108
2421 variant_id, has_arena, max_size,
2422 default_ttl, lru_skip, arena_cap_override, file_mode, errbuf,
2423 26 held ? &held[i] : NULL);
2424 2108 ▶ if (!h->shard_handles[i]) return shm_sharded_fail(h, i, held);
Branch: 1 branch
BranchTrueFalseTotal
if (!h->shard_handles[i]) return shm_sharded_fail(h, i, held); 2 2106 2108
2425 /* A wrong count hides every key that routes elsewhere; 0 = never stamped. */
2426 2106 const ShmHeader *sh = h->shard_handles[i]->hdr;
2427 2106 ▶ if (sh->routing != SHM_ROUTING_LEGACY && sh->shard_log2 &&
Branch: 2 branches
BranchTrueFalseTotal
if (sh->routing != SHM_ROUTING_LEGACY && sh->shard_log2 && 2070 36 2106
if (sh->routing != SHM_ROUTING_LEGACY && sh->shard_log2 && 51 2019 2070
2428 51 ▶ sh->shard_log2 != want_log2) {
Branch: 1 branch
BranchTrueFalseTotal
sh->shard_log2 != want_log2) { 4 47 51
2429 4 unsigned recorded = 1u << ((sh->shard_log2 - 1) & 31);
2430 4 ▶ if (errbuf && i == 0)
Branch: 2 branches
BranchTrueFalseTotal
if (errbuf && i == 0) 4 0 4
if (errbuf && i == 0) 3 1 4
2431 3 snprintf(errbuf, SHM_ERR_BUFLEN,
2432 "%s was created with %u shards, not %u",
2433 path_prefix, recorded, num_shards);
2434 1 ▶ else if (errbuf)
Branch: 1 branch
BranchTrueFalseTotal
else if (errbuf) 1 0 1
2435 1 snprintf(errbuf, SHM_ERR_BUFLEN,
2436 "%s.%u records %u shards, not %u",
2437 path_prefix, i, recorded, num_shards);
2438 4 return shm_sharded_fail(h, i + 1, held);
2439 }
2440 2102 ▶ if (i > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (i > 0) { 1876 226 2102
2441 /* Else a stray file from an earlier run joins the set. */
2442 1876 const ShmHeader *a = h->shard_handles[0]->hdr;
2443 1876 const ShmHeader *b = sh;
2444 1876 const char *what =
2445 3752 ▶ b->max_table_cap != a->max_table_cap ? "max_entries" :
Branch: 1 branch
BranchTrueFalseTotal
b->max_table_cap != a->max_table_cap ? "max_entries" : 1876 0 1876
2446 1876 ▶ b->max_size != a->max_size ? "max_size" :
Branch: 1 branch
BranchTrueFalseTotal
b->max_size != a->max_size ? "max_size" : 1875 1 1876
2447 1875 ▶ b->default_ttl != a->default_ttl ? "ttl" :
Branch: 1 branch
BranchTrueFalseTotal
b->default_ttl != a->default_ttl ? "ttl" : 1875 0 1875
2448 1875 ▶ b->lru_skip != a->lru_skip ? "lru_skip" :
Branch: 1 branch
BranchTrueFalseTotal
b->lru_skip != a->lru_skip ? "lru_skip" : 1875 0 1875
2449 1875 ▶ b->arena_cap != a->arena_cap ? "arena_cap" :
Branch: 1 branch
BranchTrueFalseTotal
b->arena_cap != a->arena_cap ? "arena_cap" : 1875 0 1875
2450 1875 ▶ b->routing != a->routing ? "routing" : NULL;
Branch: 1 branch
BranchTrueFalseTotal
b->routing != a->routing ? "routing" : NULL; 1 1874 1875
2451 1876 ▶ if (what) {
Branch: 1 branch
BranchTrueFalseTotal
if (what) { 2 1874 1876
2452 2 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 2 0 2
2453 "%s.%u disagrees with shard 0 on %s; the shards were not created "
2454 "together", path_prefix, i, what);
2455 2 return shm_sharded_fail(h, i + 1, held);
2456 }
2457 }
2458 }
2459 /* Stamp only once every shard agreed, so a refused open stamps nothing.
2460 * Shard 0's CAS serialises creators: one with another count is refused. */
2461 222 ShmHeader *s0 = h->shard_handles[0]->hdr;
2462 222 ▶ if (s0->routing != SHM_ROUTING_LEGACY) {
Branch: 1 branch
BranchTrueFalseTotal
if (s0->routing != SHM_ROUTING_LEGACY) { 218 4 222
2463 218 uint8_t seen = 0;
2464 218 ▶ if (!__atomic_compare_exchange_n(&s0->shard_log2, &seen, want_log2, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&s0->shard_log2, &seen, want_log2, 0, 7 211 218
2465 7 __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE) &&
2466 7 ▶ seen != want_log2) {
Branch: 1 branch
BranchTrueFalseTotal
seen != want_log2) { 0 7 7
2467 0 ▶ if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, 0 0 0
2468 "%s was created with %u shards, not %u",
2469 0 path_prefix, 1u << ((seen - 1) & 31), num_shards);
2470 0 return shm_sharded_fail(h, num_shards, held);
2471 }
2472 2041 ▶ for (uint32_t i = 1; i < num_shards; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 1; i < num_shards; i++) { 1823 218 2041
2473 1823 ShmHeader *s = h->shard_handles[i]->hdr;
2474 1823 uint8_t unset = 0;
2475 1823 __atomic_compare_exchange_n(&s->shard_log2, &unset, want_log2, 0,
2476 __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
2477 }
2478 }
2479  
2480 222 h->route_high = (s0->routing != SHM_ROUTING_LEGACY);
2481 222 ▶ if (held) {
Branch: 1 branch
BranchTrueFalseTotal
if (held) { 1 221 222
2482 5 ▶ for (uint32_t i = 0; i < num_shards; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < num_shards; i++) 4 1 5
2483 4 ▶ if (held[i] >= 0) { flock(held[i], LOCK_UN); close(held[i]); }
Branch: 1 branch
BranchTrueFalseTotal
if (held[i] >= 0) { flock(held[i], LOCK_UN); close(held[i]); } 4 0 4
2484 1 free(held);
2485 }
2486 222 return h;
2487 }
2488  
2489 7 static int shm_unlink_path(const char *path) {
2490 7 return (unlink(path) == 0) ? 1 : 0;
2491 }
2492  
2493 /* After a chdir or a rename the path may name another file: leave it alone. */
2494 5 static int shm_unlink_own(const ShmHandle *h) {
2495 struct stat st;
2496 5 ▶ if (!h->path || lstat(h->path, &st) != 0 ||
Branch: 2 branches
BranchTrueFalseTotal
if (!h->path || lstat(h->path, &st) != 0 || 4 1 5
if (!h->path || lstat(h->path, &st) != 0 || 4 0 4
2497 4 ▶ st.st_dev != h->file_dev || st.st_ino != h->file_ino)
Branch: 2 branches
BranchTrueFalseTotal
st.st_dev != h->file_dev || st.st_ino != h->file_ino) 4 0 4
st.st_dev != h->file_dev || st.st_ino != h->file_ino) 2 2 4
2498 3 return 0;
2499 2 return shm_unlink_path(h->path);
2500 }
2501  
2502 5 static int shm_unlink_sharded(ShmHandle *h) {
2503 5 ▶ if (!h->shard_handles) return shm_unlink_own(h);
Branch: 1 branch
BranchTrueFalseTotal
if (!h->shard_handles) return shm_unlink_own(h); 5 0 5
2504 0 int ok = 1;
2505 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
2506 0 ▶ if (h->shard_handles[i] && h->shard_handles[i]->path &&
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles[i] && h->shard_handles[i]->path && 0 0 0
2507 0 !shm_unlink_own(h->shard_handles[i]))
2508 0 ok = 0;
2509 0 return ok;
2510 }
2511  
2512 36 static inline ShmCursor *shm_cursor_create(ShmHandle *h) {
2513 36 ShmCursor *c = (ShmCursor *)calloc(1, sizeof(ShmCursor));
2514 36 ▶ if (!c) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (!c) return NULL; 0 36 36
2515 36 c->handle = h;
2516 36 ▶ if (h->shard_handles) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->shard_handles) { 3 33 36
2517 3 c->shard_count = h->num_shards;
2518 3 c->shard_idx = 0;
2519 3 c->current = h->shard_handles[0];
2520 } else {
2521 33 c->shard_count = 1;
2522 33 c->shard_idx = 0;
2523 33 c->current = h;
2524 }
2525 36 c->gen = c->current->hdr->table_gen;
2526 36 c->current->iterating++;
2527 36 return c;
2528 }
2529  
2530 3432452 static inline uint32_t shm_shard_index(const ShmHandle *h, uint64_t hash64) {
2531 3432452 ▶ uint32_t bits = h->route_high ? (uint32_t)(hash64 >> 32) : (uint32_t)hash64;
Branch: 1 branch
BranchTrueFalseTotal
uint32_t bits = h->route_high ? (uint32_t)(hash64 >> 32) : (uint32_t)hash64; 3430052 2400 3432452
2532 3432452 return bits & h->shard_mask;
2533 }
2534  
2535 36 static inline void shm_cursor_destroy(ShmCursor *c) {
2536 36 ▶ if (!c) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!c) return; 0 36 36
2537 36 ShmHandle *cur = c->current;
2538 36 ▶ if (cur && cur->iterating > 0)
Branch: 2 branches
BranchTrueFalseTotal
if (cur && cur->iterating > 0) 19 17 36
if (cur && cur->iterating > 0) 19 0 19
2539 19 cur->iterating--;
2540 36 free(c->copy_buf);
2541 36 free(c);
2542 }
2543  
2544 #endif /* SHM_DEFS_H */
2545  
2546  
2547 /* ---- Part 2: template (included once per variant) ---- */
2548  
2549 #define SHM_PASTE2(a, b) a##_##b
2550 #define SHM_PASTE(a, b) SHM_PASTE2(a, b)
2551 #define SHM_FN(name) SHM_PASTE(SHM_PREFIX, name)
2552  
2553 typedef struct {
2554 #ifdef SHM_KEY_IS_INT
2555 SHM_KEY_INT_TYPE key;
2556 #else
2557 uint32_t key_off;
2558 uint32_t key_len;
2559 #endif
2560 #ifdef SHM_VAL_IS_STR
2561 uint32_t val_off;
2562 uint32_t val_len;
2563 #else
2564 SHM_VAL_INT_TYPE value;
2565 #endif
2566 } SHM_NODE_TYPE;
2567  
2568 #undef SHM_SHARD_DISPATCH
2569 #ifdef SHM_KEY_IS_INT
2570 #define SHM_SHARD_DISPATCH(h, key_arg) \
2571 if ((h)->shard_handles) { \
2572 h = (h)->shard_handles[shm_shard_index((h), SHM_HASH_KEY64(key_arg))]; \
2573 }
2574 #else
2575 #define SHM_SHARD_DISPATCH(h, key_str_arg, key_len_arg) \
2576 if ((h)->shard_handles) { \
2577 h = (h)->shard_handles[shm_shard_index((h), SHM_HASH_KEY_STR64(key_str_arg, key_len_arg))]; \
2578 }
2579 #endif
2580  
2581 #ifdef SHM_KEY_IS_INT
2582 #define SHM_HASH_KEY(k) ((uint32_t)(shm_hash_int64((int64_t)(k))))
2583 #define SHM_HASH_KEY64(k) (shm_hash_int64((int64_t)(k)))
2584 #define SHM_KEY_EQ(node_ptr, k) ((node_ptr)->key == (k))
2585 #else
2586 #define SHM_HASH_KEY_STR(str, len) ((uint32_t)shm_hash_string((str), (len)))
2587 #define SHM_HASH_KEY_STR64(str, len) (shm_hash_string((str), (len)))
2588 /* Bytes only: the UTF8 flag is metadata, or an upgraded ASCII key would miss. */
2589 2176349 static inline int SHM_PASTE(SHM_PREFIX, _key_eq_str)(
2590 const SHM_NODE_TYPE *np, const char *arena, uint64_t arena_cap,
2591 const char *str, uint32_t len, bool utf8) {
2592 (void)utf8;
2593 2176349 uint32_t kl_packed = np->key_len;
2594 2176349 ▶ if (SHM_IS_INLINE(kl_packed)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_IS_INLINE(kl_packed)) { 2174570 1003 2175573
if (SHM_IS_INLINE(kl_packed)) { 299 362 661
if (SHM_IS_INLINE(kl_packed)) { 56 0 56
if (SHM_IS_INLINE(kl_packed)) { 59 0 59
2595 2174984 ▶ if (shm_inline_len(kl_packed) != len) return 0;
Branch: 4 branches
BranchTrueFalseTotal
if (shm_inline_len(kl_packed) != len) return 0; 330 2174240 2174570
if (shm_inline_len(kl_packed) != len) return 0; 102 197 299
if (shm_inline_len(kl_packed) != len) return 0; 0 56 56
if (shm_inline_len(kl_packed) != len) return 0; 0 59 59
2596 char buf[SHM_INLINE_MAX];
2597 2174552 shm_inline_read(np->key_off, kl_packed, buf);
2598 2174552 return memcmp(buf, str, len) == 0;
2599 }
2600 0 ▶ if (SHM_UNPACK_LEN(kl_packed) != len) return 0;
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_UNPACK_LEN(kl_packed) != len) return 0; 31 972 1003
if (SHM_UNPACK_LEN(kl_packed) != len) return 0; 82 280 362
if (SHM_UNPACK_LEN(kl_packed) != len) return 0; 0 0 0
if (SHM_UNPACK_LEN(kl_packed) != len) return 0; 0 0 0
2601 /* A corrupt key_off is a miss, so the probe continues. */
2602 0 ▶ if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0;
Branch: 8 branches
BranchTrueFalseTotal
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 972 0 972
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 972 972
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 280 0 280
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 280 280
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 0 0
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 0 0
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 0 0
if (np->key_off < SHM_ARENA_MIN_ALLOC || (uint64_t)np->key_off + len > arena_cap) return 0; 0 0 0
2603 0 return memcmp(arena + np->key_off, str, len) == 0;
2604 }
2605 #define SHM_KEY_EQ_STR(node_ptr, arena, arena_cap, str, len, utf8) \
2606 SHM_PASTE(SHM_PREFIX, _key_eq_str)((node_ptr), (arena), (arena_cap), \
2607 (str), (len), (utf8))
2608 #endif
2609  
2610 #if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
2611 #define SHM_HAS_ARENA 1
2612 #else
2613 #define SHM_HAS_ARENA 0
2614 #endif
2615  
2616 static void SHM_FN(rz_run)(ShmHandle *h);
2617  
2618 /* Give the handle, and each shard of a set, this variant's resize resume. */
2619 44230 static ShmHandle *SHM_FN(bind)(ShmHandle *h) {
2620 44230 ▶ if (!h) return NULL;
Branch: 10 branches
BranchTrueFalseTotal
if (!h) return NULL; 2 683 685
if (!h) return NULL; 0 85 85
if (!h) return NULL; 0 33 33
if (!h) return NULL; 0 36 36
if (!h) return NULL; 0 45 45
if (!h) return NULL; 0 39 39
if (!h) return NULL; 0 39 39
if (!h) return NULL; 52 43143 43195
if (!h) return NULL; 0 36 36
if (!h) return NULL; 0 37 37
2621 44176 h->rz_resume = SHM_FN(rz_run);
2622 46245 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 591 683 1274
for (uint32_t i = 0; i < h->num_shards; i++) 29 85 114
for (uint32_t i = 0; i < h->num_shards; i++) 11 33 44
for (uint32_t i = 0; i < h->num_shards; i++) 11 36 47
for (uint32_t i = 0; i < h->num_shards; i++) 13 45 58
for (uint32_t i = 0; i < h->num_shards; i++) 15 39 54
for (uint32_t i = 0; i < h->num_shards; i++) 15 39 54
for (uint32_t i = 0; i < h->num_shards; i++) 1354 43143 44497
for (uint32_t i = 0; i < h->num_shards; i++) 15 36 51
for (uint32_t i = 0; i < h->num_shards; i++) 15 37 52
2623 2069 h->shard_handles[i]->rz_resume = SHM_FN(rz_run);
2624 44176 return h;
2625 }
2626  
2627 43732 static ShmHandle *SHM_FN(create)(const char *path, uint32_t max_entries,
2628 uint32_t max_size, uint32_t default_ttl,
2629 uint32_t lru_skip, uint64_t arena_cap_override,
2630 mode_t file_mode, char *errbuf) {
2631 43732 return SHM_FN(bind)(shm_create_map(path, max_entries,
2632 (uint32_t)sizeof(SHM_NODE_TYPE),
2633 SHM_VARIANT_ID, SHM_HAS_ARENA,
2634 max_size, default_ttl, lru_skip, arena_cap_override, file_mode, errbuf, NULL));
2635 }
2636  
2637 234 static ShmHandle *SHM_FN(create_sharded)(const char *path_prefix,
2638 uint32_t num_shards,
2639 uint32_t max_entries,
2640 uint32_t max_size, uint32_t default_ttl,
2641 uint32_t lru_skip, uint64_t arena_cap_override,
2642 mode_t file_mode, char *errbuf) {
2643 234 return SHM_FN(bind)(shm_create_sharded(path_prefix, num_shards, max_entries,
2644 (uint32_t)sizeof(SHM_NODE_TYPE),
2645 SHM_VARIANT_ID, SHM_HAS_ARENA,
2646 max_size, default_ttl, lru_skip, arena_cap_override, file_mode, errbuf));
2647 }
2648  
2649 0 static ShmHandle *SHM_FN(create_memfd)(const char *name, uint32_t max_entries,
2650 uint32_t max_size, uint32_t default_ttl,
2651 uint32_t lru_skip, uint64_t arena_cap_override,
2652 char *errbuf) {
2653 0 return SHM_FN(bind)(shm_create_memfd(name, max_entries,
2654 (uint32_t)sizeof(SHM_NODE_TYPE),
2655 SHM_VARIANT_ID, SHM_HAS_ARENA,
2656 max_size, default_ttl, lru_skip, arena_cap_override, errbuf));
2657 }
2658  
2659 0 static ShmHandle *SHM_FN(open_fd)(int fd, char *errbuf) {
2660 0 return SHM_FN(bind)(shm_open_fd_map(fd, SHM_VARIANT_ID,
2661 (uint32_t)sizeof(SHM_NODE_TYPE), errbuf));
2662 }
2663  
2664 0 static ShmHandle *SHM_FN(open_readonly)(const char *path, char *errbuf) {
2665 0 return shm_open_readonly_map(path, SHM_VARIANT_ID,
2666 (uint32_t)sizeof(SHM_NODE_TYPE), errbuf);
2667 }
2668  
2669 456137 static inline uint32_t SHM_FN(node_hash)(ShmHandle *h, const SHM_NODE_TYPE *node) {
2670 #ifdef SHM_KEY_IS_INT
2671 (void)h;
2672 276411 return SHM_HASH_KEY(node->key);
2673 #else
2674 char ibuf[SHM_INLINE_MAX];
2675 uint32_t klen;
2676 179726 const char *kptr = shm_str_ptr(node->key_off, node->key_len, h->arena, h->hdr->arena_cap, ibuf, &klen);
2677 179726 return shm_hash_string(kptr, klen);
2678 #endif
2679 }
2680  
2681 178830 static void SHM_FN(tombstone_at)(ShmHandle *h, uint32_t idx) {
2682 178830 ShmHeader *hdr = h->hdr;
2683 #if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
2684 61205 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
2685 #endif
2686 /* Retire the slot first: a freed block's first word becomes a free-list link,
2687 * which a still-live node would read if the writer died in between. */
2688 178830 h->states[idx] = SHM_TOMBSTONE;
2689 178830 hdr->size--;
2690 178830 hdr->tombstones++;
2691 #ifndef SHM_KEY_IS_INT
2692 59931 shm_str_free(hdr, h->arena, nodes[idx].key_off, nodes[idx].key_len);
2693 #endif
2694 #ifdef SHM_VAL_IS_STR
2695 47060 shm_str_free(hdr, h->arena, nodes[idx].val_off, nodes[idx].val_len);
2696 #endif
2697 178830 }
2698  
2699 178830 static inline void SHM_FN(remove_at)(ShmHandle *h, uint32_t idx) {
2700 178830 ▶ if (h->lru_prev) shm_lru_unlink(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_unlink(h, idx); 32415 13371 45786
if (h->lru_prev) shm_lru_unlink(h, idx); 9947 4047 13994
if (h->lru_prev) shm_lru_unlink(h, idx); 4 71 75
if (h->lru_prev) shm_lru_unlink(h, idx); 4 72 76
if (h->lru_prev) shm_lru_unlink(h, idx); 4 1111 1115
if (h->lru_prev) shm_lru_unlink(h, idx); 4 75 79
if (h->lru_prev) shm_lru_unlink(h, idx); 4 76 80
if (h->lru_prev) shm_lru_unlink(h, idx); 1288 115690 116978
if (h->lru_prev) shm_lru_unlink(h, idx); 4 569 573
if (h->lru_prev) shm_lru_unlink(h, idx); 4 70 74
2701 /* Tombstone first: a live entry with its TTL cleared would never expire. */
2702 178830 SHM_FN(tombstone_at)(h, idx);
2703 178830 __atomic_signal_fence(__ATOMIC_SEQ_CST);
2704 178830 ▶ if (h->expires_at) h->expires_at[idx] = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) h->expires_at[idx] = 0; 898 44888 45786
if (h->expires_at) h->expires_at[idx] = 0; 3931 10063 13994
if (h->expires_at) h->expires_at[idx] = 0; 62 13 75
if (h->expires_at) h->expires_at[idx] = 0; 62 14 76
if (h->expires_at) h->expires_at[idx] = 0; 1067 48 1115
if (h->expires_at) h->expires_at[idx] = 0; 68 11 79
if (h->expires_at) h->expires_at[idx] = 0; 68 12 80
if (h->expires_at) h->expires_at[idx] = 0; 36312 80666 116978
if (h->expires_at) h->expires_at[idx] = 0; 62 511 573
if (h->expires_at) h->expires_at[idx] = 0; 62 12 74
2705 178830 }
2706  
2707 43675 static void SHM_FN(lru_evict_at)(ShmHandle *h, uint32_t idx) {
2708 43675 ▶ int was_expired = SHM_IS_EXPIRED(h, idx, shm_now());
Branch: 30 branches
BranchTrueFalseTotal
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 32413 32413
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 9947 9947
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 1001 286 1287
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 1001 0 1001
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 1001 0 1001
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 4 4
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
int was_expired = SHM_IS_EXPIRED(h, idx, shm_now()); 0 0 0
2709 43675 SHM_FN(remove_at)(h, idx);
2710 43675 ▶ __atomic_add_fetch(was_expired ? &h->hdr->stat_expired
Branch: 10 branches
BranchTrueFalseTotal
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 32413 32413
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 9947 9947
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 1001 286 1287
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
__atomic_add_fetch(was_expired ? &h->hdr->stat_expired 0 4 4
2711 42674 : &h->hdr->stat_evictions, 1, __ATOMIC_RELAXED);
2712 43675 }
2713  
2714 /* `keep`, never the victim, is the entry an overwrite is replacing. */
2715 5398 static void SHM_FN(lru_evict_one_keep)(ShmHandle *h, uint32_t keep) {
2716 5398 ShmHeader *hdr = h->hdr;
2717 5398 ▶ uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4078 4078
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 5 5
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 1001 286 1287
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t victim = hdr->lru_tail, spared = 0, now = h->expires_at ? shm_now() : 0; 0 4 4
2718 5553 ▶ while (victim != SHM_LRU_NONE) {
Branch: 10 branches
BranchTrueFalseTotal
while (victim != SHM_LRU_NONE) { 4081 1 4082
while (victim != SHM_LRU_NONE) { 8 0 8
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 1414 0 1414
while (victim != SHM_LRU_NONE) { 7 0 7
while (victim != SHM_LRU_NONE) { 7 0 7
2719 5552 ▶ if (victim == keep) {
Branch: 10 branches
BranchTrueFalseTotal
if (victim == keep) { 1 4080 4081
if (victim == keep) { 0 8 8
if (victim == keep) { 0 7 7
if (victim == keep) { 0 7 7
if (victim == keep) { 0 7 7
if (victim == keep) { 0 7 7
if (victim == keep) { 0 7 7
if (victim == keep) { 0 1414 1414
if (victim == keep) { 0 7 7
if (victim == keep) { 0 7 7
2720 0 victim = h->lru_prev[victim];
2721 0 continue;
2722 }
2723 5551 ▶ if (SHM_IS_EXPIRED(h, victim, now)) break;
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 4080 4080
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 8 8
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 1001 413 1414
if (SHM_IS_EXPIRED(h, victim, now)) break; 1001 0 1001
if (SHM_IS_EXPIRED(h, victim, now)) break; 1001 0 1001
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 7 7
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
if (SHM_IS_EXPIRED(h, victim, now)) break; 0 0 0
2724 4550 ▶ if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX &&
Branch: 20 branches
BranchTrueFalseTotal
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 4080 0 4080
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 4080 0 4080
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 8 0 8
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 8 0 8
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 413 0 413
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 412 1 413
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
if (h->lru_accessed && spared++ < SHM_LRU_SPARE_MAX && 7 0 7
2725 4549 ▶ __atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) {
Branch: 10 branches
BranchTrueFalseTotal
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4077 4080
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 5 8
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 127 285 412
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
__atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) { 3 4 7
2726 154 __atomic_store_n(&h->lru_accessed[victim], 0, __ATOMIC_RELAXED);
2727 154 uint32_t prev = h->lru_prev[victim];
2728 154 shm_lru_unlink(h, victim);
2729 154 shm_lru_push_front(h, victim);
2730 154 victim = prev;
2731 154 ▶ if (victim == SHM_LRU_NONE) victim = hdr->lru_tail;
Branch: 10 branches
BranchTrueFalseTotal
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 127 127
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
if (victim == SHM_LRU_NONE) victim = hdr->lru_tail; 0 3 3
2732 154 continue;
2733 }
2734 4396 break;
2735 }
2736 5398 ▶ if (victim == SHM_LRU_NONE) return;
Branch: 10 branches
BranchTrueFalseTotal
if (victim == SHM_LRU_NONE) return; 1 4077 4078
if (victim == SHM_LRU_NONE) return; 0 5 5
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 1287 1287
if (victim == SHM_LRU_NONE) return; 0 4 4
if (victim == SHM_LRU_NONE) return; 0 4 4
2737 5397 SHM_FN(lru_evict_at)(h, victim);
2738 }
2739  
2740 1326 static void SHM_FN(lru_evict_one)(ShmHandle *h) {
2741 1326 SHM_FN(lru_evict_one_keep)(h, SHM_LRU_NONE);
2742 1326 }
2743  
2744 #ifdef SHM_KEY_IS_INT
2745 #define SHM_KEY_SLEN(n) 0
2746 #else
2747 #define SHM_KEY_SLEN(n) (n)
2748 #endif
2749 #ifdef SHM_VAL_IS_STR
2750 #define SHM_VAL_SLEN(n) (n)
2751 #else
2752 #define SHM_VAL_SLEN(n) 0
2753 #endif
2754  
2755 /* No eviction for strings that can never be stored: the insert is refused. */
2756 396662 static inline void SHM_FN(evict_for_insert)(ShmHandle *h, uint32_t klen, uint32_t vlen) {
2757 396662 ShmHeader *hdr = h->hdr;
2758 397995 ▶ if (hdr->max_size > 0 && hdr->size >= hdr->max_size &&
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 38011 98878 136889
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 10041 45202 55243
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 225 245
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 221 241
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 21261 21281
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 222 242
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 213 233
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 10808 169986 180794
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 1227 1247
if (hdr->max_size > 0 && hdr->size >= hdr->max_size && 20 227 247
2759 2664 ▶ shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen))
Branch: 10 branches
BranchTrueFalseTotal
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 6 5 11
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 5 0 5
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 1287 0 1287
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
shm_arena_can_hold(hdr, klen) && shm_arena_can_hold(hdr, vlen)) 4 0 4
2760 1326 SHM_FN(lru_evict_one)(h);
2761 396662 }
2762  
2763 41564 static void SHM_FN(expire_at)(ShmHandle *h, uint32_t idx) {
2764 41564 SHM_FN(remove_at)(h, idx);
2765 41564 __atomic_add_fetch(&h->hdr->stat_expired, 1, __ATOMIC_RELAXED);
2766 41564 }
2767  
2768 /* Unforced, skipped for 1, 2, 4 ... 64 s after a flush that freed under a
2769 * sixteenth of the table. A second flush in one second can free only the
2770 * entry the first one kept, left expired by a failed overwrite. */
2771 318 static uint32_t SHM_FN(reclaim_expired)(ShmHandle *h, int forced, uint32_t keep) {
2772 318 uint32_t now = shm_now(), cap = h->hdr->table_cap, freed = 0;
2773 318 ▶ if (h->reclaim_cap != cap) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->reclaim_cap != cap) { 33 14 47
if (h->reclaim_cap != cap) { 44 32 76
if (h->reclaim_cap != cap) { 5 0 5
if (h->reclaim_cap != cap) { 5 0 5
if (h->reclaim_cap != cap) { 19 12 31
if (h->reclaim_cap != cap) { 11 11 22
if (h->reclaim_cap != cap) { 11 11 22
if (h->reclaim_cap != cap) { 61 41 102
if (h->reclaim_cap != cap) { 4 0 4
if (h->reclaim_cap != cap) { 4 0 4
2774 197 h->reclaim_cap = cap;
2775 197 h->reclaim_wait = 0;
2776 197 h->reclaim_at = 0;
2777 }
2778 318 ▶ if (h->reclaim_sec == now + 1) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->reclaim_sec == now + 1) { 32 15 47
if (h->reclaim_sec == now + 1) { 47 29 76
if (h->reclaim_sec == now + 1) { 2 3 5
if (h->reclaim_sec == now + 1) { 2 3 5
if (h->reclaim_sec == now + 1) { 21 10 31
if (h->reclaim_sec == now + 1) { 14 8 22
if (h->reclaim_sec == now + 1) { 14 8 22
if (h->reclaim_sec == now + 1) { 89 13 102
if (h->reclaim_sec == now + 1) { 2 2 4
if (h->reclaim_sec == now + 1) { 2 2 4
2779 225 uint64_t kept = h->reclaim_kept;
2780 225 ▶ if (kept == UINT64_MAX) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (kept == UINT64_MAX) return 0; 20 12 32
if (kept == UINT64_MAX) return 0; 47 0 47
if (kept == UINT64_MAX) return 0; 2 0 2
if (kept == UINT64_MAX) return 0; 2 0 2
if (kept == UINT64_MAX) return 0; 9 12 21
if (kept == UINT64_MAX) return 0; 2 12 14
if (kept == UINT64_MAX) return 0; 2 12 14
if (kept == UINT64_MAX) return 0; 89 0 89
if (kept == UINT64_MAX) return 0; 2 0 2
if (kept == UINT64_MAX) return 0; 2 0 2
2781 0 ▶ if ((uint32_t)(kept >> 32) == h->hdr->table_gen) {
Branch: 10 branches
BranchTrueFalseTotal
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 11 1 12
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 11 1 12
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 11 1 12
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 11 1 12
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
if ((uint32_t)(kept >> 32) == h->hdr->table_gen) { 0 0 0
2782 0 uint32_t idx = (uint32_t)kept;
2783 0 ▶ if (idx == keep) return 0; /* the same overwrite still protects it */
Branch: 10 branches
BranchTrueFalseTotal
if (idx == keep) return 0; /* the same overwrite still protects it */ 6 5 11
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
if (idx == keep) return 0; /* the same overwrite still protects it */ 6 5 11
if (idx == keep) return 0; /* the same overwrite still protects it */ 6 5 11
if (idx == keep) return 0; /* the same overwrite still protects it */ 6 5 11
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
if (idx == keep) return 0; /* the same overwrite still protects it */ 0 0 0
2784 0 h->reclaim_kept = UINT64_MAX;
2785 0 ▶ if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) {
Branch: 40 branches
BranchTrueFalseTotal
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 5 0 5
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_LIVE(h->states[idx]) && SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
2786 shm_sig_check(h);
2787 0 SHM_FN(expire_at)(h, idx);
2788 0 return 1;
2789 }
2790 0 return 0;
2791 }
2792 0 forced = 1; /* a resize moved it */
2793 }
2794 97 ▶ if (!forced && now < h->reclaim_at) return 0;
Branch: 20 branches
BranchTrueFalseTotal
if (!forced && now < h->reclaim_at) return 0; 7 9 16
if (!forced && now < h->reclaim_at) return 0; 0 7 7
if (!forced && now < h->reclaim_at) return 0; 28 1 29
if (!forced && now < h->reclaim_at) return 0; 0 28 28
if (!forced && now < h->reclaim_at) return 0; 3 0 3
if (!forced && now < h->reclaim_at) return 0; 0 3 3
if (!forced && now < h->reclaim_at) return 0; 3 0 3
if (!forced && now < h->reclaim_at) return 0; 0 3 3
if (!forced && now < h->reclaim_at) return 0; 3 8 11
if (!forced && now < h->reclaim_at) return 0; 0 3 3
if (!forced && now < h->reclaim_at) return 0; 2 7 9
if (!forced && now < h->reclaim_at) return 0; 0 2 2
if (!forced && now < h->reclaim_at) return 0; 2 7 9
if (!forced && now < h->reclaim_at) return 0; 0 2 2
if (!forced && now < h->reclaim_at) return 0; 13 0 13
if (!forced && now < h->reclaim_at) return 0; 0 13 13
if (!forced && now < h->reclaim_at) return 0; 2 0 2
if (!forced && now < h->reclaim_at) return 0; 0 2 2
if (!forced && now < h->reclaim_at) return 0; 2 0 2
if (!forced && now < h->reclaim_at) return 0; 0 2 2
2795 97 h->reclaim_sec = now + 1;
2796 0 ▶ h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now)
Branch: 30 branches
BranchTrueFalseTotal
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 7 0 7
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 7 0 7
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 7 0 7
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 6 0 6
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
h->reclaim_kept = keep != SHM_LRU_NONE && SHM_IS_EXPIRED(h, keep, now) 0 0 0
2797 122 ▶ ? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX;
Branch: 10 branches
BranchTrueFalseTotal
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 7 9 16
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 29 29
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 3 3
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 3 3
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 6 5 11
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 6 3 9
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 6 3 9
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 13 13
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 2 2
? (uint64_t)h->hdr->table_gen << 32 | keep : UINT64_MAX; 0 2 2
2798 97 shm_sig_hold(h, cap);
2799 10321 ▶ for (uint32_t i = 0; i < cap; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap; i++) { 1488 16 1504
for (uint32_t i = 0; i < cap; i++) { 5536 29 5565
for (uint32_t i = 0; i < cap; i++) { 48 3 51
for (uint32_t i = 0; i < cap; i++) { 48 3 51
for (uint32_t i = 0; i < cap; i++) { 2320 11 2331
for (uint32_t i = 0; i < cap; i++) { 256 9 265
for (uint32_t i = 0; i < cap; i++) { 256 9 265
for (uint32_t i = 0; i < cap; i++) { 208 13 221
for (uint32_t i = 0; i < cap; i++) { 32 2 34
for (uint32_t i = 0; i < cap; i++) { 32 2 34
2800 10224 ▶ if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) {
Branch: 50 branches
BranchTrueFalseTotal
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1481 7 1488
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 923 558 1481
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 923 0 923
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 923 0 923
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 831 92 923
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 5536 0 5536
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 3670 1866 5536
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 3670 0 3670
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 3670 0 3670
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 3368 302 3670
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 48 0 48
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 9 48
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 0 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 0 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 0 39 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 48 0 48
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 9 48
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 0 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 39 0 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 0 39 39
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 2314 6 2320
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1040 1274 2314
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1040 0 1040
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1040 0 1040
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1000 40 1040
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 250 6 256
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 222 250
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 0 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 0 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1 27 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 250 6 256
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 222 250
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 0 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 28 0 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 1 27 28
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 208 0 208
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 172 36 208
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 172 0 172
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 157 15 172
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 15 142 157
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 32 0 32
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 6 32
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 0 26
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 0 26
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 0 26 26
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 32 0 32
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 6 32
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 0 26
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 26 0 26
if (i != keep && SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) { 0 26 26
2801 shm_sig_check(h);
2802 0 SHM_FN(expire_at)(h, i);
2803 0 freed++;
2804 }
2805 }
2806 97 ▶ if (freed >= cap / 16) {
Branch: 10 branches
BranchTrueFalseTotal
if (freed >= cap / 16) { 2 14 16
if (freed >= cap / 16) { 12 17 29
if (freed >= cap / 16) { 0 3 3
if (freed >= cap / 16) { 0 3 3
if (freed >= cap / 16) { 1 10 11
if (freed >= cap / 16) { 0 9 9
if (freed >= cap / 16) { 0 9 9
if (freed >= cap / 16) { 1 12 13
if (freed >= cap / 16) { 0 2 2
if (freed >= cap / 16) { 0 2 2
2807 0 h->reclaim_wait = 0;
2808 0 h->reclaim_at = 0;
2809 } else {
2810 81 ▶ h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1;
Branch: 20 branches
BranchTrueFalseTotal
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 14 14
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 17 17
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 3 3
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 3 3
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 10 10
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 9 9
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 9 9
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 1 11 12
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 1 0 1
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 2 2
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 2 2
h->reclaim_wait = h->reclaim_wait ? (h->reclaim_wait < 64 ? h->reclaim_wait * 2 : 64) : 1; 0 0 0
2811 81 h->reclaim_at = now + h->reclaim_wait;
2812 }
2813 97 return freed;
2814 }
2815  
2816 #if SHM_HAS_ARENA
2817 0 static inline int SHM_FN(block_is)(uint32_t len_field, uint32_t asize) {
2818 0 ▶ return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize;
Branch: 14 branches
BranchTrueFalseTotal
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 638383 635285 1273668
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 28336 610047 638383
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 9942 0 9942
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 9942 0 9942
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize; 0 0 0
2819 }
2820  
2821 0 static int SHM_FN(lru_evict_for)(ShmHandle *h, uint32_t slen, uint32_t keep) {
2822 0 ShmHeader *hdr = h->hdr;
2823 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
2824 0 uint32_t asize = shm_arena_round_up(slen);
2825 0 uint32_t pos = hdr->lru_tail;
2826 0 ▶ for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) {
Branch: 14 branches
BranchTrueFalseTotal
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 636842 4065 640907
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 636835 7 636842
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 9942 0 9942
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 9942 0 9942
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) { 0 0 0
2827 0 if (pos != keep && (
2828 #ifndef SHM_KEY_IS_INT
2829 0 ▶ SHM_FN(block_is)(nodes[pos].key_len, asize) ||
Branch: 1 branch
BranchTrueFalseTotal
SHM_FN(block_is)(nodes[pos].key_len, asize) || 28336 608498 636834
2830 #endif
2831 #ifdef SHM_VAL_IS_STR
2832 0 SHM_FN(block_is)(nodes[pos].val_len, asize) ||
2833 #endif
2834 0)) {
2835 0 SHM_FN(lru_evict_at)(h, pos);
2836 0 return 1;
2837 }
2838 0 pos = h->lru_prev[pos];
2839 }
2840 0 uint32_t before = hdr->size;
2841 0 SHM_FN(lru_evict_one_keep)(h, keep);
2842 0 return hdr->size < before;
2843 }
2844  
2845 351434 static int SHM_FN(store_or_evict)(ShmHandle *h, uint32_t *off, uint32_t *len,
2846 const char *str, uint32_t slen, bool utf8, uint32_t keep) {
2847 351434 shm_sig_hold_bytes(h, slen);
2848 351434 ▶ if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1;
Branch: 7 branches
BranchTrueFalseTotal
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 196230 77659 273889
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 45296 9947 55243
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 245 0 245
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 241 0 241
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 21281 20 21301
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 244 18 262
if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1; 235 18 253
2849 0 ▶ if (!shm_arena_can_hold(h->hdr, slen)) return 0;
Branch: 7 branches
BranchTrueFalseTotal
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 7 77652 77659
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 3 9944 9947
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 0 0 0
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 0 0 0
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 0 20 20
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 0 18 18
if (!shm_arena_can_hold(h->hdr, slen)) return 0; 0 18 18
2850 /* One eviction only: if its block was the wrong size, more are as blind. */
2851 0 ▶ if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) &&
Branch: 7 branches
BranchTrueFalseTotal
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 32408 45244 77652
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 9942 2 9944
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 0 0 0
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 0 0 0
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 0 20 20
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 0 18 18
if (h->lru_prev && SHM_FN(lru_evict_for)(h, slen, keep) && 0 18 18
2852 0 shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8))
2853 0 return 1;
2854 0 ▶ if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) &&
Branch: 7 branches
BranchTrueFalseTotal
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 22 49294 49316
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 2 0 2
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 0 0 0
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 0 0 0
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 20 0 20
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 18 0 18
if (h->expires_at && SHM_FN(reclaim_expired)(h, 1, keep) && 18 0 18
2855 0 shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8))
2856 0 return 1;
2857 /* Arm only after the eviction, which is how a full LRU cache normally stores. */
2858 0 h->arena_failed = 1;
2859 0 h->arena_need = shm_arena_round_up(slen);
2860 0 return 0;
2861 }
2862  
2863 /* Crash-safe: the free lists are cleared first, and a block moves only wholly
2864 * below its source, so its node points at intact bytes until one aligned 32-bit
2865 * store repoints it. A killed writer leaks the free blocks not yet relisted. */
2866 0 static uint64_t SHM_FN(arena_compact)(ShmHandle *h, int only_if_useful) {
2867 0 ShmHeader *hdr = h->hdr;
2868 0 uint64_t before = hdr->arena_bump;
2869 0 ▶ if (before <= SHM_ARENA_MIN_ALLOC) return 0;
Branch: 7 branches
BranchTrueFalseTotal
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 178 178
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 1 1
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 0 0
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 0 0
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 7 7
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 6 6
if (before <= SHM_ARENA_MIN_ALLOC) return 0; 0 6 6
2870  
2871 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
2872 0 uint8_t *states = h->states;
2873 0 uint32_t cap = hdr->table_cap;
2874 0 ▶ shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap);
Branch: 7 branches
BranchTrueFalseTotal
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 178 0 178
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 1 0 1
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 0 0 0
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 0 0 0
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 7 0 7
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 6 0 6
shm_sig_hold(h, before >= SHM_SIG_HOLD_BYTES ? SHM_SIG_HOLD_MIN : cap); 6 0 6
2875  
2876 0 uint32_t n = 0;
2877 0 uint64_t live = 0;
2878 0 ▶ for (uint32_t i = 0; i < cap; i++) {
Branch: 7 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap; i++) { 1144976 178 1145154
for (uint32_t i = 0; i < cap; i++) { 2048 1 2049
for (uint32_t i = 0; i < cap; i++) { 0 0 0
for (uint32_t i = 0; i < cap; i++) { 0 0 0
for (uint32_t i = 0; i < cap; i++) { 2144 7 2151
for (uint32_t i = 0; i < cap; i++) { 96 6 102
for (uint32_t i = 0; i < cap; i++) { 96 6 102
2879 0 ▶ if (states[i] < SHM_TAG_MIN) continue;
Branch: 7 branches
BranchTrueFalseTotal
if (states[i] < SHM_TAG_MIN) continue; 814019 330957 1144976
if (states[i] < SHM_TAG_MIN) continue; 1049 999 2048
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 1138 1006 2144
if (states[i] < SHM_TAG_MIN) continue; 89 7 96
if (states[i] < SHM_TAG_MIN) continue; 89 7 96
2880 #ifndef SHM_KEY_IS_INT
2881 0 ▶ if (!SHM_IS_INLINE(nodes[i].key_len)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_IS_INLINE(nodes[i].key_len)) { 514 330443 330957
if (!SHM_IS_INLINE(nodes[i].key_len)) { 999 0 999
if (!SHM_IS_INLINE(nodes[i].key_len)) { 0 0 0
if (!SHM_IS_INLINE(nodes[i].key_len)) { 0 0 0
2882 0 n++;
2883 0 live += shm_arena_round_up(SHM_UNPACK_LEN(nodes[i].key_len));
2884 }
2885 #endif
2886 #ifdef SHM_VAL_IS_STR
2887 331977 ▶ if (!SHM_IS_INLINE(nodes[i].val_len)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_IS_INLINE(nodes[i].val_len)) { 330856 101 330957
if (!SHM_IS_INLINE(nodes[i].val_len)) { 1005 1 1006
if (!SHM_IS_INLINE(nodes[i].val_len)) { 6 1 7
if (!SHM_IS_INLINE(nodes[i].val_len)) { 6 1 7
2888 331873 n++;
2889 331873 live += shm_arena_round_up(SHM_UNPACK_LEN(nodes[i].val_len));
2890 }
2891 #endif
2892 }
2893 0 ▶ if (n == 0) {
Branch: 7 branches
BranchTrueFalseTotal
if (n == 0) { 1 177 178
if (n == 0) { 0 1 1
if (n == 0) { 0 0 0
if (n == 0) { 0 0 0
if (n == 0) { 0 7 7
if (n == 0) { 0 6 6
if (n == 0) { 0 6 6
2894 0 shm_arena_reset(hdr);
2895 0 return before - SHM_ARENA_MIN_ALLOC;
2896 }
2897 0 ▶ if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0;
Branch: 14 branches
BranchTrueFalseTotal
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 158 19 177
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 85 73 158
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 1 0 1
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 1 0 1
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 0 0 0
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 0 0 0
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 0 0 0
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 0 0 0
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 7 0 7
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 7 0 7
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 6 0 6
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 6 0 6
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 6 0 6
if (only_if_useful && shm_arena_slide_useless(hdr, live)) return 0; 6 0 6
2898  
2899 /* Allocate first, so a failure leaves the arena as it was. */
2900 0 ShmArenaRef *refs = (ShmArenaRef *)malloc((size_t)n * sizeof *refs);
2901 0 ▶ if (!refs) return 0;
Branch: 7 branches
BranchTrueFalseTotal
if (!refs) return 0; 0 92 92
if (!refs) return 0; 0 0 0
if (!refs) return 0; 0 0 0
if (!refs) return 0; 0 0 0
if (!refs) return 0; 0 0 0
if (!refs) return 0; 0 0 0
if (!refs) return 0; 0 0 0
2902  
2903 0 uint32_t fcap = shm_arena_snapshot_free(hdr, h->arena, NULL, 0);
2904 0 ShmArenaRef *frees = NULL;
2905 0 uint32_t fn = 0;
2906 0 ▶ if (fcap) {
Branch: 7 branches
BranchTrueFalseTotal
if (fcap) { 90 2 92
if (fcap) { 0 0 0
if (fcap) { 0 0 0
if (fcap) { 0 0 0
if (fcap) { 0 0 0
if (fcap) { 0 0 0
if (fcap) { 0 0 0
2907 0 frees = (ShmArenaRef *)malloc((size_t)fcap * sizeof *frees);
2908 0 ▶ if (!frees) { free(refs); return 0; }
Branch: 7 branches
BranchTrueFalseTotal
if (!frees) { free(refs); return 0; } 0 90 90
if (!frees) { free(refs); return 0; } 0 0 0
if (!frees) { free(refs); return 0; } 0 0 0
if (!frees) { free(refs); return 0; } 0 0 0
if (!frees) { free(refs); return 0; } 0 0 0
if (!frees) { free(refs); return 0; } 0 0 0
if (!frees) { free(refs); return 0; } 0 0 0
2909 0 fn = shm_arena_snapshot_free(hdr, h->arena, frees, fcap);
2910 0 qsort(frees, fn, sizeof *frees, shm_arena_ref_cmp);
2911 }
2912  
2913 /* Sources vacated by a slide, newer than the snapshot; every destination lies
2914 * below the gap being relisted. */
2915 0 ShmArenaRef *vac = (ShmArenaRef *)malloc((size_t)n * sizeof *vac);
2916 0 ▶ if (!vac) { free(frees); free(refs); return 0; }
Branch: 7 branches
BranchTrueFalseTotal
if (!vac) { free(frees); free(refs); return 0; } 0 92 92
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
if (!vac) { free(frees); free(refs); return 0; } 0 0 0
2917  
2918 /* From here on nothing may be handed a block that is about to move. */
2919 0 memset(hdr->arena_free, 0, sizeof(hdr->arena_free));
2920 0 hdr->arena_large_free = 0;
2921  
2922 0 uint32_t m = 0;
2923 0 ▶ for (uint32_t i = 0; i < cap && m < n; i++) {
Branch: 14 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < cap && m < n; i++) { 597399 2 597401
for (uint32_t i = 0; i < cap && m < n; i++) { 597309 90 597399
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
for (uint32_t i = 0; i < cap && m < n; i++) { 0 0 0
2924 0 ▶ if (states[i] < SHM_TAG_MIN) continue;
Branch: 7 branches
BranchTrueFalseTotal
if (states[i] < SHM_TAG_MIN) continue; 423968 173341 597309
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
if (states[i] < SHM_TAG_MIN) continue; 0 0 0
2925 #ifndef SHM_KEY_IS_INT
2926 0 ▶ if (!SHM_IS_INLINE(nodes[i].key_len)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_IS_INLINE(nodes[i].key_len)) { 514 172827 173341
if (!SHM_IS_INLINE(nodes[i].key_len)) { 0 0 0
if (!SHM_IS_INLINE(nodes[i].key_len)) { 0 0 0
if (!SHM_IS_INLINE(nodes[i].key_len)) { 0 0 0
2927 0 refs[m].off = nodes[i].key_off; refs[m].ref = i << 1; m++;
2928 }
2929 #endif
2930 #ifdef SHM_VAL_IS_STR
2931 0 ▶ if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) {
Branch: 8 branches
BranchTrueFalseTotal
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 173341 0 173341
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 173341 0 173341
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
if (m < n && !SHM_IS_INLINE(nodes[i].val_len)) { 0 0 0
2932 0 refs[m].off = nodes[i].val_off; refs[m].ref = (i << 1) | 1u; m++;
2933 }
2934 #endif
2935 }
2936 0 qsort(refs, m, sizeof *refs, shm_arena_ref_cmp);
2937  
2938 0 uint32_t vn = 0, vi = 0;
2939  
2940 0 uint64_t bump = SHM_ARENA_MIN_ALLOC;
2941 0 uint32_t fi = 0;
2942 0 ▶ for (uint32_t k = 0; k < m; k++) {
Branch: 7 branches
BranchTrueFalseTotal
for (uint32_t k = 0; k < m; k++) { 173855 92 173947
for (uint32_t k = 0; k < m; k++) { 0 0 0
for (uint32_t k = 0; k < m; k++) { 0 0 0
for (uint32_t k = 0; k < m; k++) { 0 0 0
for (uint32_t k = 0; k < m; k++) { 0 0 0
for (uint32_t k = 0; k < m; k++) { 0 0 0
for (uint32_t k = 0; k < m; k++) { 0 0 0
2943 0 uint32_t idx = refs[k].ref >> 1;
2944 0 uint32_t off = refs[k].off, packed;
2945 uint32_t *offp;
2946 #if defined(SHM_VAL_IS_STR) && !defined(SHM_KEY_IS_INT)
2947 173855 int is_val = (int)(refs[k].ref & 1u);
2948 173855 ▶ offp = is_val ? &nodes[idx].val_off : &nodes[idx].key_off;
Branch: 1 branch
BranchTrueFalseTotal
offp = is_val ? &nodes[idx].val_off : &nodes[idx].key_off; 173341 514 173855
2949 173855 ▶ packed = is_val ? nodes[idx].val_len : nodes[idx].key_len;
Branch: 1 branch
BranchTrueFalseTotal
packed = is_val ? nodes[idx].val_len : nodes[idx].key_len; 173341 514 173855
2950 #elif defined(SHM_VAL_IS_STR)
2951 0 offp = &nodes[idx].val_off; packed = nodes[idx].val_len;
2952 #else
2953 0 offp = &nodes[idx].key_off; packed = nodes[idx].key_len;
2954 #endif
2955 0 uint32_t len = SHM_UNPACK_LEN(packed);
2956 0 uint32_t size = shm_arena_round_up(len);
2957 /* Leave a wild peer-written offset where it is. */
2958 0 ▶ if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap)
Branch: 14 branches
BranchTrueFalseTotal
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 173855 0 173855
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 173855 173855
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
if (off < SHM_ARENA_MIN_ALLOC || (uint64_t)off + size > hdr->arena_cap) 0 0 0
2959 0 continue;
2960 0 ▶ if (bump + size <= off) {
Branch: 7 branches
BranchTrueFalseTotal
if (bump + size <= off) { 150730 23125 173855
if (bump + size <= off) { 0 0 0
if (bump + size <= off) { 0 0 0
if (bump + size <= off) { 0 0 0
if (bump + size <= off) { 0 0 0
if (bump + size <= off) { 0 0 0
if (bump + size <= off) { 0 0 0
2961 0 memcpy(h->arena + bump, h->arena + off, len);
2962 0 __atomic_store_n(offp, (uint32_t)bump, __ATOMIC_RELEASE);
2963 0 bump += size;
2964 0 vac[vn].off = off; vac[vn].ref = size; vn++;
2965 0 ▶ } else if ((uint64_t)off + size > bump) {
Branch: 7 branches
BranchTrueFalseTotal
} else if ((uint64_t)off + size > bump) { 23125 0 23125
} else if ((uint64_t)off + size > bump) { 0 0 0
} else if ((uint64_t)off + size > bump) { 0 0 0
} else if ((uint64_t)off + size > bump) { 0 0 0
} else if ((uint64_t)off + size > bump) { 0 0 0
} else if ((uint64_t)off + size > bump) { 0 0 0
} else if ((uint64_t)off + size > bump) { 0 0 0
2966 /* Stuck: relist the free space below it, or the bump strands it. */
2967 0 ▶ if ((uint64_t)off > bump) {
Branch: 7 branches
BranchTrueFalseTotal
if ((uint64_t)off > bump) { 382 22743 23125
if ((uint64_t)off > bump) { 0 0 0
if ((uint64_t)off > bump) { 0 0 0
if ((uint64_t)off > bump) { 0 0 0
if ((uint64_t)off > bump) { 0 0 0
if ((uint64_t)off > bump) { 0 0 0
if ((uint64_t)off > bump) { 0 0 0
2968 0 shm_arena_relist_gap(hdr, h->arena, frees, fn, &fi, (uint32_t)bump, off);
2969 0 shm_arena_relist_gap(hdr, h->arena, vac, vn, &vi, (uint32_t)bump, off);
2970 }
2971 0 bump = (uint64_t)off + size;
2972 }
2973 }
2974 0 free(vac);
2975 0 free(frees);
2976 0 free(refs);
2977 0 hdr->arena_bump = bump < before ? bump : before;
2978 0 return before - hdr->arena_bump;
2979 }
2980 #endif
2981  
2982 /* Call before anything of the entry being stored is allocated: the key is stored
2983 * before size++, and a reclaim in between would hand its block out again.
2984 * A slide moves blocks: no arena pointer or offset survives the call. */
2985 #define SHM_COMPACT_BACKOFF 64u
2986 #define SHM_COMPACT_BACKOFF_MAX (1u << 20)
2987 396834 static void SHM_FN(arena_reclaim)(ShmHandle *h) {
2988 396834 ShmHeader *hdr = h->hdr;
2989 396834 ▶ if (!h->arena) return;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->arena) return; 0 137001 137001
if (!h->arena) return; 0 55243 55243
if (!h->arena) return; 0 245 245
if (!h->arena) return; 0 241 241
if (!h->arena) return; 0 21301 21301
if (!h->arena) return; 0 262 262
if (!h->arena) return; 0 253 253
if (!h->arena) return; 180794 0 180794
if (!h->arena) return; 1247 0 1247
if (!h->arena) return; 247 0 247
2990 0 ▶ if (hdr->size == 0) {
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->size == 0) { 252 136749 137001
if (hdr->size == 0) { 108 55135 55243
if (hdr->size == 0) { 33 212 245
if (hdr->size == 0) { 36 205 241
if (hdr->size == 0) { 82 21219 21301
if (hdr->size == 0) { 39 223 262
if (hdr->size == 0) { 39 214 253
if (hdr->size == 0) { 0 0 0
if (hdr->size == 0) { 0 0 0
if (hdr->size == 0) { 0 0 0
2991 /* Even if the bump reads empty: a writer killed among clear()'s plain
2992 * arena stores can leave lists naming blocks the bump hands out again. */
2993 0 h->arena_failed = 0;
2994 0 h->compact_backoff = 0;
2995 0 h->compact_period = 0;
2996 0 shm_arena_reset(hdr);
2997 0 return;
2998 }
2999 #if SHM_HAS_ARENA
3000 213957 ▶ if (!h->arena_failed) return;
Branch: 7 branches
BranchTrueFalseTotal
if (!h->arena_failed) return; 87467 49282 136749
if (!h->arena_failed) return; 55134 1 55135
if (!h->arena_failed) return; 212 0 212
if (!h->arena_failed) return; 205 0 205
if (!h->arena_failed) return; 21206 13 21219
if (!h->arena_failed) return; 211 12 223
if (!h->arena_failed) return; 202 12 214
3001 0 h->arena_failed = 0;
3002 /* A sixteenth of the entries gone since may have freed what the slide lacked. */
3003 0 ▶ if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16)
Branch: 14 branches
BranchTrueFalseTotal
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 49135 147 49282
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 11 49124 49135
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 1 1
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 0 0
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 0 0
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 0 0
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 0 0
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 0 0
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 6 7 13
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 6 6
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 6 6 12
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 6 6
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 6 6 12
if (h->compact_backoff && hdr->size < h->compact_size - h->compact_size / 16) 0 6 6
3004 0 h->compact_backoff = 0;
3005 0 ▶ if (h->compact_backoff) { h->compact_backoff--; return; }
Branch: 7 branches
BranchTrueFalseTotal
if (h->compact_backoff) { h->compact_backoff--; return; } 49124 158 49282
if (h->compact_backoff) { h->compact_backoff--; return; } 0 1 1
if (h->compact_backoff) { h->compact_backoff--; return; } 0 0 0
if (h->compact_backoff) { h->compact_backoff--; return; } 0 0 0
if (h->compact_backoff) { h->compact_backoff--; return; } 6 7 13
if (h->compact_backoff) { h->compact_backoff--; return; } 6 6 12
if (h->compact_backoff) { h->compact_backoff--; return; } 6 6 12
3006 0 uint64_t got = SHM_FN(arena_compact)(h, 1);
3007 0 ▶ if (got >= h->arena_need) {
Branch: 7 branches
BranchTrueFalseTotal
if (got >= h->arena_need) { 69 89 158
if (got >= h->arena_need) { 0 1 1
if (got >= h->arena_need) { 0 0 0
if (got >= h->arena_need) { 0 0 0
if (got >= h->arena_need) { 0 7 7
if (got >= h->arena_need) { 0 6 6
if (got >= h->arena_need) { 0 6 6
3008 0 h->compact_period = 0;
3009 0 ▶ } else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) {
Branch: 7 branches
BranchTrueFalseTotal
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 89 89
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 1 1
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 0 0
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 0 0
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 7 7
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 6 6
} else if (h->compact_period >= SHM_COMPACT_BACKOFF_MAX / 8) { 0 6 6
3010 0 h->compact_period = SHM_COMPACT_BACKOFF_MAX;
3011 } else {
3012 0 ▶ h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF;
Branch: 7 branches
BranchTrueFalseTotal
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 8 81 89
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 1 1
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 0 0
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 0 0
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 7 7
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 6 6
h->compact_period = h->compact_period ? h->compact_period * 8 : SHM_COMPACT_BACKOFF; 0 6 6
3013 }
3014 0 h->compact_size = hdr->size;
3015 0 h->compact_backoff = h->compact_period;
3016 #endif
3017 }
3018  
3019 /* No free slot on a TTL map: flush the expired, take the first freed on the path. */
3020 396903 static uint32_t SHM_FN(insert_slot)(ShmHandle *h, uint32_t pos, uint32_t mask, uint32_t insert_pos) {
3021 396903 ▶ if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE))
Branch: 30 branches
BranchTrueFalseTotal
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 3 136889 136892
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 3 3
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 10 55243 55253
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 1 9 10
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 1 0 1
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 9 245 254
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 9 9
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 9 241 250
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 9 9
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 3 21281 21284
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 3 3
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 3 242 245
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 3 3
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 3 233 236
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 3 3
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 183 180794 180977
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 17 166 183
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 17 0 17
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 9 1247 1256
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 9 9
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 9 247 256
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 9 9
if (insert_pos != UINT32_MAX || !h->expires_at || !SHM_FN(reclaim_expired)(h, 1, SHM_LRU_NONE)) 0 0 0
3022 396903 return insert_pos;
3023 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
3024 0 uint32_t idx = (pos + i) & mask;
3025 0 ▶ if (!SHM_IS_LIVE(h->states[idx])) return idx;
Branch: 10 branches
BranchTrueFalseTotal
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
if (!SHM_IS_LIVE(h->states[idx])) return idx; 0 0 0
3026 }
3027 0 return UINT32_MAX;
3028 }
3029  
3030 /* ---- Resize (elastic grow/shrink) ----
3031 * In place and restartable: the rz_* record and the table decide at every
3032 * instruction boundary how to finish; a dead writer's successor reruns rz_run. */
3033  
3034 /* Complete a committed move: repoint the LRU neighbours at dst, then free src. */
3035 static inline __attribute__((always_inline))
3036 void SHM_FN(rz_settle)(ShmHandle *h, uint32_t src, uint32_t dst) {
3037 276029 ShmHeader *hdr = h->hdr;
3038 0 ▶ if (h->lru_prev) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 0 0
3039 51295 uint32_t p = h->lru_prev[dst], n = h->lru_next[dst];
3040 51295 ▶ if (p != SHM_LRU_NONE) h->lru_next[p] = dst;
Branch: 40 branches
BranchTrueFalseTotal
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 11313 65 11378
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 82 0 82
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 69 13 82
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 3890 94 3984
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 192 0 192
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 178 14 192
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 11897 362 12259
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 11563 0 11563
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 11562 1 11563
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
if (p != SHM_LRU_NONE) h->lru_next[p] = dst; 0 0 0
3041 549 ▶ else if (hdr->lru_head == src) hdr->lru_head = dst;
Branch: 40 branches
BranchTrueFalseTotal
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 65 0 65
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 13 0 13
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 94 0 94
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 14 0 14
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 362 0 362
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 1 0 1
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
else if (hdr->lru_head == src) hdr->lru_head = dst; 0 0 0
3042 51295 ▶ if (n != SHM_LRU_NONE) h->lru_prev[n] = dst;
Branch: 40 branches
BranchTrueFalseTotal
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 11290 88 11378
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 78 4 82
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 80 2 82
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 3852 132 3984
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 182 10 192
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 184 8 192
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 12178 81 12259
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 11521 42 11563
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 11519 44 11563
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
if (n != SHM_LRU_NONE) h->lru_prev[n] = dst; 0 0 0
3043 411 ▶ else if (hdr->lru_tail == src) hdr->lru_tail = dst;
Branch: 40 branches
BranchTrueFalseTotal
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 88 0 88
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 4 0 4
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 2 0 2
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 132 0 132
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 10 0 10
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 8 0 8
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 81 0 81
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 42 0 42
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 44 0 44
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
else if (hdr->lru_tail == src) hdr->lru_tail = dst; 0 0 0
3044 }
3045 276029 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3046 276029 h->states[src] = SHM_EMPTY;
3047 276029 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3048 276029 ▶ if (h->expires_at) h->expires_at[src] = 0;
Branch: 40 branches
BranchTrueFalseTotal
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 1101 54026 55127
if (h->expires_at) h->expires_at[src] = 0; 5 308 313
if (h->expires_at) h->expires_at[src] = 0; 5 308 313
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 3165 36213 39378
if (h->expires_at) h->expires_at[src] = 0; 14 219 233
if (h->expires_at) h->expires_at[src] = 0; 14 219 233
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 80 6 86
if (h->expires_at) h->expires_at[src] = 0; 1 0 1
if (h->expires_at) h->expires_at[src] = 0; 1 0 1
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 80 0 80
if (h->expires_at) h->expires_at[src] = 0; 1 0 1
if (h->expires_at) h->expires_at[src] = 0; 1 0 1
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 1037 15791 16828
if (h->expires_at) h->expires_at[src] = 0; 16 16 32
if (h->expires_at) h->expires_at[src] = 0; 16 16 32
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 70 0 70
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 70 0 70
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 31252 106034 137286
if (h->expires_at) h->expires_at[src] = 0; 83 12145 12228
if (h->expires_at) h->expires_at[src] = 0; 83 12145 12228
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 69 1308 1377
if (h->expires_at) h->expires_at[src] = 0; 2 13 15
if (h->expires_at) h->expires_at[src] = 0; 2 13 15
if (h->expires_at) h->expires_at[src] = 0; 0 0 0
if (h->expires_at) h->expires_at[src] = 0; 69 0 69
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
if (h->expires_at) h->expires_at[src] = 0; 2 0 2
3049 276029 ▶ if (h->lru_prev) {
Branch: 40 branches
BranchTrueFalseTotal
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 11378 43749 55127
if (h->lru_prev) { 82 231 313
if (h->lru_prev) { 82 231 313
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 3984 35394 39378
if (h->lru_prev) { 192 41 233
if (h->lru_prev) { 192 41 233
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 86 86
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 80 80
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 16828 16828
if (h->lru_prev) { 0 32 32
if (h->lru_prev) { 0 32 32
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 70 70
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 70 70
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 12259 125027 137286
if (h->lru_prev) { 11563 665 12228
if (h->lru_prev) { 11563 665 12228
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 1377 1377
if (h->lru_prev) { 0 15 15
if (h->lru_prev) { 0 15 15
if (h->lru_prev) { 0 0 0
if (h->lru_prev) { 0 69 69
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
3050 51295 h->lru_prev[src] = h->lru_next[src] = SHM_LRU_NONE;
3051 51295 ▶ if (h->lru_accessed) h->lru_accessed[src] = 0;
Branch: 40 branches
BranchTrueFalseTotal
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 11378 0 11378
if (h->lru_accessed) h->lru_accessed[src] = 0; 82 0 82
if (h->lru_accessed) h->lru_accessed[src] = 0; 82 0 82
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 3984 0 3984
if (h->lru_accessed) h->lru_accessed[src] = 0; 192 0 192
if (h->lru_accessed) h->lru_accessed[src] = 0; 192 0 192
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 12259 0 12259
if (h->lru_accessed) h->lru_accessed[src] = 0; 11563 0 11563
if (h->lru_accessed) h->lru_accessed[src] = 0; 11563 0 11563
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
if (h->lru_accessed) h->lru_accessed[src] = 0; 0 0 0
3052 }
3053 276029 }
3054  
3055 /* The record names the move before the copy; states[dst] leaving SHM_EMPTY commits it. */
3056 static inline __attribute__((always_inline))
3057 void SHM_FN(rz_move)(ShmHandle *h, uint32_t src, uint32_t dst, uint8_t st) {
3058 12829 ▶ if (src != dst) {
Branch: 10 branches
BranchTrueFalseTotal
if (src != dst) { 313 0 313
if (src != dst) { 233 0 233
if (src != dst) { 1 0 1
if (src != dst) { 1 0 1
if (src != dst) { 32 0 32
if (src != dst) { 2 0 2
if (src != dst) { 2 0 2
if (src != dst) { 12228 0 12228
if (src != dst) { 15 0 15
if (src != dst) { 2 0 2
3059 276029 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3060 276029 __atomic_store_n(&h->hdr->rz_move, (uint64_t)src | (uint64_t)dst << 32, __ATOMIC_RELAXED);
3061 276029 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3062 276029 nodes[dst] = nodes[src];
3063 276029 ▶ if (h->expires_at) h->expires_at[dst] = h->expires_at[src];
Branch: 30 branches
BranchTrueFalseTotal
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1101 54026 55127
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 5 308 313
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 5 308 313
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 3165 36213 39378
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 14 219 233
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 14 219 233
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 80 6 86
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1 0 1
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1 0 1
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 80 0 80
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1 0 1
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1 0 1
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 1037 15791 16828
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 16 16 32
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 16 16 32
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 70 0 70
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 70 0 70
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 31252 106034 137286
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 83 12145 12228
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 83 12145 12228
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 69 1308 1377
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 13 15
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 13 15
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 69 0 69
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
if (h->expires_at) h->expires_at[dst] = h->expires_at[src]; 2 0 2
3064 276029 ▶ if (h->lru_prev) {
Branch: 30 branches
BranchTrueFalseTotal
if (h->lru_prev) { 11378 43749 55127
if (h->lru_prev) { 82 231 313
if (h->lru_prev) { 82 231 313
if (h->lru_prev) { 3984 35394 39378
if (h->lru_prev) { 192 41 233
if (h->lru_prev) { 192 41 233
if (h->lru_prev) { 0 86 86
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 80 80
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 1 1
if (h->lru_prev) { 0 16828 16828
if (h->lru_prev) { 0 32 32
if (h->lru_prev) { 0 32 32
if (h->lru_prev) { 0 70 70
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 70 70
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 12259 125027 137286
if (h->lru_prev) { 11563 665 12228
if (h->lru_prev) { 11563 665 12228
if (h->lru_prev) { 0 1377 1377
if (h->lru_prev) { 0 15 15
if (h->lru_prev) { 0 15 15
if (h->lru_prev) { 0 69 69
if (h->lru_prev) { 0 2 2
if (h->lru_prev) { 0 2 2
3065 51295 h->lru_prev[dst] = h->lru_prev[src];
3066 51295 h->lru_next[dst] = h->lru_next[src];
3067 51295 ▶ if (h->lru_accessed)
Branch: 30 branches
BranchTrueFalseTotal
if (h->lru_accessed) 11378 0 11378
if (h->lru_accessed) 82 0 82
if (h->lru_accessed) 82 0 82
if (h->lru_accessed) 3984 0 3984
if (h->lru_accessed) 192 0 192
if (h->lru_accessed) 192 0 192
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 12259 0 12259
if (h->lru_accessed) 11563 0 11563
if (h->lru_accessed) 11563 0 11563
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
if (h->lru_accessed) 0 0 0
3068 51295 h->lru_accessed[dst] = __atomic_load_n(&h->lru_accessed[src], __ATOMIC_RELAXED);
3069 }
3070 }
3071 468966 shm_publish_tag(h->states, dst, st);
3072 468966 if (src != dst) {
3073 276029 ▶ __atomic_signal_fence(__ATOMIC_SEQ_CST);
Branch: 30 branches
BranchTrueFalseTotal
__atomic_signal_fence(__ATOMIC_SEQ_CST); 11378 43749 55127
__atomic_signal_fence(__ATOMIC_SEQ_CST); 82 231 313
__atomic_signal_fence(__ATOMIC_SEQ_CST); 82 231 313
__atomic_signal_fence(__ATOMIC_SEQ_CST); 3984 35394 39378
__atomic_signal_fence(__ATOMIC_SEQ_CST); 192 41 233
__atomic_signal_fence(__ATOMIC_SEQ_CST); 192 41 233
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 86 86
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 1 1
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 1 1
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 80 80
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 1 1
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 1 1
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 16828 16828
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 32 32
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 32 32
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 70 70
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 70 70
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
__atomic_signal_fence(__ATOMIC_SEQ_CST); 12259 125027 137286
__atomic_signal_fence(__ATOMIC_SEQ_CST); 11563 665 12228
__atomic_signal_fence(__ATOMIC_SEQ_CST); 11563 665 12228
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 1377 1377
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 15 15
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 15 15
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 69 69
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
__atomic_signal_fence(__ATOMIC_SEQ_CST); 0 2 2
3074 SHM_FN(rz_settle)(h, src, dst);
3075 }
3076 468966 }
3077  
3078 /* A target held by an entry still to move is vacated by parking that entry in a
3079 * free slot, placed next. 0 (no free slot) only on a record resize() did not write. */
3080 443308 static int SHM_FN(rz_place)(ShmHandle *h, uint32_t s, uint32_t mask,
3081 uint32_t work, uint32_t *spare) {
3082 443308 ShmHeader *hdr = h->hdr;
3083 443308 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3084 443308 uint8_t *states = h->states;
3085 12829 for (;;) {
3086 456137 uint32_t hash = SHM_FN(node_hash)(h, &nodes[s]);
3087 456137 uint32_t t = hash & mask, n = 0;
3088 893542 ▶ while (SHM_IS_LIVE(states[t])) {
Branch: 10 branches
BranchTrueFalseTotal
while (SHM_IS_LIVE(states[t])) { 32276 111679 143955
while (SHM_IS_LIVE(states[t])) { 21990 67804 89794
while (SHM_IS_LIVE(states[t])) { 31 128 159
while (SHM_IS_LIVE(states[t])) { 29 115 144
while (SHM_IS_LIVE(states[t])) { 7952 26202 34154
while (SHM_IS_LIVE(states[t])) { 36 102 138
while (SHM_IS_LIVE(states[t])) { 36 102 138
while (SHM_IS_LIVE(states[t])) { 374219 247761 621980
while (SHM_IS_LIVE(states[t])) { 800 2143 2943
while (SHM_IS_LIVE(states[t])) { 36 101 137
3089 437405 ▶ if (n++ > mask) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (n++ > mask) return 0; 0 32276 32276
if (n++ > mask) return 0; 0 21990 21990
if (n++ > mask) return 0; 0 31 31
if (n++ > mask) return 0; 0 29 29
if (n++ > mask) return 0; 0 7952 7952
if (n++ > mask) return 0; 0 36 36
if (n++ > mask) return 0; 0 36 36
if (n++ > mask) return 0; 0 374219 374219
if (n++ > mask) return 0; 0 800 800
if (n++ > mask) return 0; 0 36 36
3090 437405 t = (t + 1) & mask;
3091 }
3092 456137 ▶ if (t == s || states[t] == SHM_EMPTY) {
Branch: 20 branches
BranchTrueFalseTotal
if (t == s || states[t] == SHM_EMPTY) { 55440 56239 111679
if (t == s || states[t] == SHM_EMPTY) { 55127 313 55440
if (t == s || states[t] == SHM_EMPTY) { 39611 28193 67804
if (t == s || states[t] == SHM_EMPTY) { 39378 233 39611
if (t == s || states[t] == SHM_EMPTY) { 87 41 128
if (t == s || states[t] == SHM_EMPTY) { 86 1 87
if (t == s || states[t] == SHM_EMPTY) { 81 34 115
if (t == s || states[t] == SHM_EMPTY) { 80 1 81
if (t == s || states[t] == SHM_EMPTY) { 16860 9342 26202
if (t == s || states[t] == SHM_EMPTY) { 16828 32 16860
if (t == s || states[t] == SHM_EMPTY) { 72 30 102
if (t == s || states[t] == SHM_EMPTY) { 70 2 72
if (t == s || states[t] == SHM_EMPTY) { 72 30 102
if (t == s || states[t] == SHM_EMPTY) { 70 2 72
if (t == s || states[t] == SHM_EMPTY) { 149514 98247 247761
if (t == s || states[t] == SHM_EMPTY) { 137286 12228 149514
if (t == s || states[t] == SHM_EMPTY) { 1392 751 2143
if (t == s || states[t] == SHM_EMPTY) { 1377 15 1392
if (t == s || states[t] == SHM_EMPTY) { 71 30 101
if (t == s || states[t] == SHM_EMPTY) { 69 2 71
3093 443308 ▶ SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash));
Branch: 10 branches
BranchTrueFalseTotal
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 55127 56239 111366
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 39378 28193 67571
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 86 41 127
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 80 34 114
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 16828 9342 26170
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 70 30 100
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 70 30 100
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 137286 98247 235533
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 1377 751 2128
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 69 30 99
3094 443308 return 1;
3095 }
3096 12829 uint32_t e = *spare;
3097 1407060 ▶ for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0)
Branch: 20 branches
BranchTrueFalseTotal
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 314 0 314
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 314 313 627
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 369 0 369
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 369 233 602
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 0 0 0
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 0 1 1
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 0 0 0
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 0 1 1
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 23 0 23
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 23 32 55
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 0 1
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 2 3
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 0 1
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 2 3
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1388093 5420 1393513
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1393513 12228 1405741
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 9 0 9
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 9 15 24
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 0 1
for (n = 0; states[e] != SHM_EMPTY; e = e + 1 < work ? e + 1 : 0) 1 2 3
3098 0 ▶ if (n++ >= work) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (n++ >= work) return 0; 0 314 314
if (n++ >= work) return 0; 0 369 369
if (n++ >= work) return 0; 0 0 0
if (n++ >= work) return 0; 0 0 0
if (n++ >= work) return 0; 0 23 23
if (n++ >= work) return 0; 0 1 1
if (n++ >= work) return 0; 0 1 1
if (n++ >= work) return 0; 0 1393513 1393513
if (n++ >= work) return 0; 0 9 9
if (n++ >= work) return 0; 0 1 1
3099 12829 *spare = e;
3100 12829 ▶ if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED);
Branch: 10 branches
BranchTrueFalseTotal
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 10 303 313
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 18 215 233
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 1 1
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 1 1
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 32 32
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 2 2
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 2 2
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 83 12145 12228
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 15 15
if (e < hdr->rz_cursor) __atomic_store_n(&hdr->rz_cursor, e, __ATOMIC_RELAXED); 0 2 2
3101 SHM_FN(rz_move)(h, t, e, SHM_MOVE);
3102 12829 ▶ SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash));
Branch: 10 branches
BranchTrueFalseTotal
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 313 0 313
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 233 0 233
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 1 0 1
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 1 0 1
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 32 0 32
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 2 0 2
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 2 0 2
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 12228 0 12228
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 15 0 15
SHM_FN(rz_move)(h, s, t, SHM_MAKE_TAG(hash)); 2 0 2
3103 12829 s = e;
3104 }
3105 }
3106  
3107 2446 static void SHM_FN(rz_run)(ShmHandle *h) {
3108 2446 ShmHeader *hdr = h->hdr;
3109 2446 uint8_t *states = h->states;
3110 2446 uint8_t phase = __atomic_load_n(&hdr->rz_phase, __ATOMIC_ACQUIRE);
3111 2446 ▶ if (phase == SHM_RZ_NONE) return;
Branch: 10 branches
BranchTrueFalseTotal
if (phase == SHM_RZ_NONE) return; 0 781 781
if (phase == SHM_RZ_NONE) return; 0 513 513
if (phase == SHM_RZ_NONE) return; 0 7 7
if (phase == SHM_RZ_NONE) return; 0 6 6
if (phase == SHM_RZ_NONE) return; 0 42 42
if (phase == SHM_RZ_NONE) return; 0 6 6
if (phase == SHM_RZ_NONE) return; 0 6 6
if (phase == SHM_RZ_NONE) return; 0 1067 1067
if (phase == SHM_RZ_NONE) return; 0 13 13
if (phase == SHM_RZ_NONE) return; 0 5 5
3112 2446 ▶ if (phase == SHM_RZ_CLEAR) {
Branch: 10 branches
BranchTrueFalseTotal
if (phase == SHM_RZ_CLEAR) { 0 781 781
if (phase == SHM_RZ_CLEAR) { 0 513 513
if (phase == SHM_RZ_CLEAR) { 0 7 7
if (phase == SHM_RZ_CLEAR) { 0 6 6
if (phase == SHM_RZ_CLEAR) { 0 42 42
if (phase == SHM_RZ_CLEAR) { 0 6 6
if (phase == SHM_RZ_CLEAR) { 0 6 6
if (phase == SHM_RZ_CLEAR) { 0 1067 1067
if (phase == SHM_RZ_CLEAR) { 0 13 13
if (phase == SHM_RZ_CLEAR) { 0 5 5
3113 0 ▶ if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED))
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
if (hdr->rz_seq == __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED)) 0 0 0
3114 0 shm_clear_run(h);
3115 else
3116 0 __atomic_store_n(&hdr->rz_phase, SHM_RZ_NONE, __ATOMIC_RELEASE);
3117 0 return;
3118 }
3119 2446 uint8_t ol = hdr->rz_old_log2, nl = hdr->rz_new_log2;
3120 2446 ▶ uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0;
Branch: 20 branches
BranchTrueFalseTotal
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 781 0 781
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 781 0 781
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 513 0 513
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 513 0 513
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 7 0 7
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 7 0 7
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 42 0 42
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 42 0 42
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 6 0 6
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 1067 0 1067
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 1067 0 1067
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 13 0 13
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 13 0 13
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 5 0 5
uint32_t old_cap = ol < 32 ? 1u << ol : 0, new_cap = nl < 32 ? 1u << nl : 0; 5 0 5
3121 2446 uint32_t work = old_cap > new_cap ? old_cap : new_cap;
3122 /* Drop a record that cannot be followed within the table, or a stale one:
3123 * only the write section that recorded it is still at its seq. table_cap
3124 * changes last. */
3125 2446 uint32_t cap = hdr->table_cap;
3126 4892 ▶ if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) ||
Branch: 10 branches
BranchTrueFalseTotal
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 781 0 781
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 513 0 513
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 7 0 7
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 6 0 6
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 42 0 42
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 6 0 6
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 6 0 6
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 1067 0 1067
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 13 0 13
if (phase > SHM_RZ_PLACE || hdr->rz_seq != __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED) || 5 0 5
3127 4892 ▶ !shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) ||
Branch: 20 branches
BranchTrueFalseTotal
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 781 0 781
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 781 781
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 513 0 513
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 513 513
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 7 0 7
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 7 7
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 6 0 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 6 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 42 0 42
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 42 42
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 6 0 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 6 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 6 0 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 6 6
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 1067 0 1067
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 1067 1067
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 13 0 13
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 13 13
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 5 0 5
!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask) || 0 5 5
3128 0 ▶ (cap != old_cap && cap != new_cap)) {
Branch: 10 branches
BranchTrueFalseTotal
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
(cap != old_cap && cap != new_cap)) { 0 0 0
3129 0 __atomic_store_n(&hdr->rz_phase, SHM_RZ_NONE, __ATOMIC_RELEASE);
3130 0 return;
3131 }
3132 #ifndef SHM_KEY_IS_INT
3133 1307 ▶ if (hdr->arena_bump >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN);
Branch: 4 branches
BranchTrueFalseTotal
if (hdr->arena_bump >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN); 2 779 781
if (hdr->arena_bump >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN); 0 513 513
if (hdr->arena_bump >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN); 0 7 7
if (hdr->arena_bump >= SHM_SIG_HOLD_BYTES) shm_sig_hold(h, SHM_SIG_HOLD_MIN); 0 6 6
3134 #endif
3135 2446 ▶ if (phase == SHM_RZ_CLEAN) {
Branch: 10 branches
BranchTrueFalseTotal
if (phase == SHM_RZ_CLEAN) { 781 0 781
if (phase == SHM_RZ_CLEAN) { 513 0 513
if (phase == SHM_RZ_CLEAN) { 7 0 7
if (phase == SHM_RZ_CLEAN) { 6 0 6
if (phase == SHM_RZ_CLEAN) { 42 0 42
if (phase == SHM_RZ_CLEAN) { 6 0 6
if (phase == SHM_RZ_CLEAN) { 6 0 6
if (phase == SHM_RZ_CLEAN) { 1067 0 1067
if (phase == SHM_RZ_CLEAN) { 13 0 13
if (phase == SHM_RZ_CLEAN) { 5 0 5
3136 2446 __atomic_store_n(&hdr->tombstones, 0, __ATOMIC_RELAXED);
3137 2446 shm_states_clean(states, old_cap);
3138 2446 ▶ if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap);
Branch: 10 branches
BranchTrueFalseTotal
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 467 314 781
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 90 423 513
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 6 1 7
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 5 1 6
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 40 2 42
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 5 1 6
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 5 1 6
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 903 164 1067
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 11 2 13
if (new_cap > old_cap) memset(states + old_cap, SHM_EMPTY, new_cap - old_cap); 4 1 5
3139 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3140 2446 __atomic_store_n(&hdr->rz_phase, phase = SHM_RZ_MARK, __ATOMIC_RELEASE);
3141 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3142 }
3143 2446 ▶ if (phase == SHM_RZ_MARK) {
Branch: 10 branches
BranchTrueFalseTotal
if (phase == SHM_RZ_MARK) { 781 0 781
if (phase == SHM_RZ_MARK) { 513 0 513
if (phase == SHM_RZ_MARK) { 7 0 7
if (phase == SHM_RZ_MARK) { 6 0 6
if (phase == SHM_RZ_MARK) { 42 0 42
if (phase == SHM_RZ_MARK) { 6 0 6
if (phase == SHM_RZ_MARK) { 6 0 6
if (phase == SHM_RZ_MARK) { 1067 0 1067
if (phase == SHM_RZ_MARK) { 13 0 13
if (phase == SHM_RZ_MARK) { 5 0 5
3144 2446 shm_states_mark(states, old_cap);
3145 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3146 2446 __atomic_store_n(&hdr->rz_phase, SHM_RZ_PLACE, __ATOMIC_RELEASE);
3147 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3148 }
3149 2446 uint64_t mv = __atomic_load_n(&hdr->rz_move, __ATOMIC_RELAXED);
3150 2446 uint32_t src = (uint32_t)mv, dst = (uint32_t)(mv >> 32);
3151 2446 ▶ if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY)
Branch: 40 branches
BranchTrueFalseTotal
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 781 781
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 513 513
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 7 7
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 6 6
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 42 42
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 6 6
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 6 6
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 1067 1067
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 13 13
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 5 5
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
if (src != dst && src < work && dst < work && states[dst] != SHM_EMPTY) 0 0 0
3152 SHM_FN(rz_settle)(h, src, dst);
3153 2446 ▶ uint32_t spare = new_cap > old_cap ? old_cap : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t spare = new_cap > old_cap ? old_cap : 0; 467 314 781
uint32_t spare = new_cap > old_cap ? old_cap : 0; 90 423 513
uint32_t spare = new_cap > old_cap ? old_cap : 0; 6 1 7
uint32_t spare = new_cap > old_cap ? old_cap : 0; 5 1 6
uint32_t spare = new_cap > old_cap ? old_cap : 0; 40 2 42
uint32_t spare = new_cap > old_cap ? old_cap : 0; 5 1 6
uint32_t spare = new_cap > old_cap ? old_cap : 0; 5 1 6
uint32_t spare = new_cap > old_cap ? old_cap : 0; 903 164 1067
uint32_t spare = new_cap > old_cap ? old_cap : 0; 11 2 13
uint32_t spare = new_cap > old_cap ? old_cap : 0; 4 1 5
3154 2446 uint32_t i = hdr->rz_cursor;
3155 2446 ▶ if (i > work) i = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (i > work) i = 0; 0 781 781
if (i > work) i = 0; 0 513 513
if (i > work) i = 0; 0 7 7
if (i > work) i = 0; 0 6 6
if (i > work) i = 0; 0 42 42
if (i > work) i = 0; 0 6 6
if (i > work) i = 0; 0 6 6
if (i > work) i = 0; 0 1067 1067
if (i > work) i = 0; 0 13 13
if (i > work) i = 0; 0 5 5
3156 /* Only a run resumed in PLACE can find an entry to move above old_cap. */
3157 2446 ▶ uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 781 781
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 513 513
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 7 7
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 6 6
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 42 42
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 6 6
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 6 6
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 1067 1067
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 13 13
uint32_t scan = phase == SHM_RZ_PLACE ? work : old_cap; 0 5 5
3158 #ifndef SHM_KEY_IS_INT
3159 1307 const int prefetch = work >= SHM_RZ_PREFETCH_MIN;
3160 #endif
3161 7621 ▶ while (i < scan) {
Branch: 10 branches
BranchTrueFalseTotal
while (i < scan) { 2115 781 2896
while (i < scan) { 769 513 1282
while (i < scan) { 7 7 14
while (i < scan) { 6 6 12
while (i < scan) { 162 42 204
while (i < scan) { 6 6 12
while (i < scan) { 6 6 12
while (i < scan) { 2075 1067 3142
while (i < scan) { 24 13 37
while (i < scan) { 5 5 10
3162 5175 uint32_t stop = (i | SHM_SIG_CHECK_SLOTS) + 1;
3163 5175 ▶ if (stop > scan) stop = scan;
Branch: 10 branches
BranchTrueFalseTotal
if (stop > scan) stop = scan; 647 1468 2115
if (stop > scan) stop = scan; 475 294 769
if (stop > scan) stop = scan; 7 0 7
if (stop > scan) stop = scan; 6 0 6
if (stop > scan) stop = scan; 22 140 162
if (stop > scan) stop = scan; 6 0 6
if (stop > scan) stop = scan; 6 0 6
if (stop > scan) stop = scan; 767 1308 2075
if (stop > scan) stop = scan; 9 15 24
if (stop > scan) stop = scan; 5 0 5
3164 927063 ▶ for (; i < stop; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (; i < stop; i++) { 400896 2115 403011
for (; i < stop; i++) { 104880 769 105649
for (; i < stop; i++) { 256 7 263
for (; i < stop; i++) { 240 6 246
for (; i < stop; i++) { 37040 162 37202
for (; i < stop; i++) { 240 6 246
for (; i < stop; i++) { 240 6 246
for (; i < stop; i++) { 373568 2075 375643
for (; i < stop; i++) { 4304 24 4328
for (; i < stop; i++) { 224 5 229
3165 #ifndef SHM_KEY_IS_INT
3166 506272 uint32_t j = i + SHM_RZ_PREFETCH_AHEAD;
3167 506272 ▶ if (prefetch && j < scan && states[j] == SHM_MOVE) {
Branch: 12 branches
BranchTrueFalseTotal
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 400896 400896
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 104880 104880
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 256 256
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 240 240
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
if (prefetch && j < scan && states[j] == SHM_MOVE) { 0 0 0
3168 0 const SHM_NODE_TYPE *nd = (const SHM_NODE_TYPE *)h->nodes + j;
3169 0 ▶ if (!SHM_IS_INLINE(nd->key_len)) __builtin_prefetch(h->arena + nd->key_off);
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_IS_INLINE(nd->key_len)) __builtin_prefetch(h->arena + nd->key_off); 0 0 0
if (!SHM_IS_INLINE(nd->key_len)) __builtin_prefetch(h->arena + nd->key_off); 0 0 0
if (!SHM_IS_INLINE(nd->key_len)) __builtin_prefetch(h->arena + nd->key_off); 0 0 0
if (!SHM_IS_INLINE(nd->key_len)) __builtin_prefetch(h->arena + nd->key_off); 0 0 0
3170 }
3171 #endif
3172 921888 ▶ if (states[i] != SHM_MOVE) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (states[i] != SHM_MOVE) continue; 289530 111366 400896
if (states[i] != SHM_MOVE) continue; 37309 67571 104880
if (states[i] != SHM_MOVE) continue; 129 127 256
if (states[i] != SHM_MOVE) continue; 126 114 240
if (states[i] != SHM_MOVE) continue; 10870 26170 37040
if (states[i] != SHM_MOVE) continue; 140 100 240
if (states[i] != SHM_MOVE) continue; 140 100 240
if (states[i] != SHM_MOVE) continue; 138035 235533 373568
if (states[i] != SHM_MOVE) continue; 2176 2128 4304
if (states[i] != SHM_MOVE) continue; 125 99 224
3173 443308 ▶ if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) {
Branch: 10 branches
BranchTrueFalseTotal
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 111366 111366
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 67571 67571
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 127 127
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 114 114
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 26170 26170
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 100 100
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 100 100
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 235533 235533
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 2128 2128
if (!SHM_FN(rz_place)(h, i, new_cap - 1, work, &spare)) { 0 99 99
3174 0 __atomic_store_n(&hdr->rz_phase, SHM_RZ_NONE, __ATOMIC_RELEASE);
3175 0 return;
3176 }
3177 443308 __atomic_store_n(&hdr->rz_cursor, i + 1, __ATOMIC_RELAXED);
3178 }
3179 shm_sig_check(h);
3180 }
3181 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3182 2446 __atomic_store_n(&hdr->table_cap, new_cap, __ATOMIC_RELEASE);
3183 /* Rehashing can move an expired entry behind a partial flush's cursor. */
3184 2446 ▶ if (h->expires_at) hdr->flush_cursor = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) hdr->flush_cursor = 0; 27 754 781
if (h->expires_at) hdr->flush_cursor = 0; 36 477 513
if (h->expires_at) hdr->flush_cursor = 0; 6 1 7
if (h->expires_at) hdr->flush_cursor = 0; 6 0 6
if (h->expires_at) hdr->flush_cursor = 0; 14 28 42
if (h->expires_at) hdr->flush_cursor = 0; 6 0 6
if (h->expires_at) hdr->flush_cursor = 0; 6 0 6
if (h->expires_at) hdr->flush_cursor = 0; 67 1000 1067
if (h->expires_at) hdr->flush_cursor = 0; 5 8 13
if (h->expires_at) hdr->flush_cursor = 0; 5 0 5
3185 2446 hdr->table_gen++;
3186 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3187 2446 __atomic_store_n(&hdr->rz_phase, SHM_RZ_NONE, __ATOMIC_RELEASE);
3188 }
3189  
3190 2575 static int SHM_FN(resize)(ShmHandle *h, uint32_t new_cap) {
3191 2575 ShmHeader *hdr = h->hdr;
3192 2575 uint32_t old_cap = hdr->table_cap, live;
3193 2575 ▶ if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0;
Branch: 20 branches
BranchTrueFalseTotal
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 795 0 795
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 795 795
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 528 0 528
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 528 528
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 13 0 13
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 13 13
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 12 0 12
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 12 12
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 83 0 83
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 83 83
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 11 0 11
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 11 11
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 11 0 11
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 11 11
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 1094 0 1094
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 1094 1094
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 18 0 18
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 18 18
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 10 0 10
if (!shm_rz_cap_ok(old_cap, h->max_mask) || !shm_rz_cap_ok(new_cap, h->max_mask)) return 0; 0 10 10
3194 2575 shm_sig_hold(h, old_cap > new_cap ? old_cap : new_cap);
3195 2575 live = shm_count_live(h->states, old_cap);
3196 /* Placement needs a free slot to park in. */
3197 2575 ▶ if (live >= new_cap) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (live >= new_cap) return 0; 0 795 795
if (live >= new_cap) return 0; 0 528 528
if (live >= new_cap) return 0; 0 13 13
if (live >= new_cap) return 0; 0 12 12
if (live >= new_cap) return 0; 0 83 83
if (live >= new_cap) return 0; 0 11 11
if (live >= new_cap) return 0; 0 11 11
if (live >= new_cap) return 0; 0 1094 1094
if (live >= new_cap) return 0; 0 18 18
if (live >= new_cap) return 0; 0 10 10
3198 2575 ▶ if (live == 0) {
Branch: 10 branches
BranchTrueFalseTotal
if (live == 0) { 14 781 795
if (live == 0) { 15 513 528
if (live == 0) { 6 7 13
if (live == 0) { 6 6 12
if (live == 0) { 41 42 83
if (live == 0) { 5 6 11
if (live == 0) { 5 6 11
if (live == 0) { 27 1067 1094
if (live == 0) { 5 13 18
if (live == 0) { 5 5 10
3199 /* No record needed: the table is a whole empty one after each store. */
3200 129 memset(h->states, SHM_EMPTY, old_cap > new_cap ? old_cap : new_cap);
3201 129 __atomic_store_n(&hdr->tombstones, 0, __ATOMIC_RELAXED);
3202 129 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3203 129 __atomic_store_n(&hdr->table_cap, new_cap, __ATOMIC_RELEASE);
3204 129 ▶ if (h->expires_at) hdr->flush_cursor = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) hdr->flush_cursor = 0; 3 11 14
if (h->expires_at) hdr->flush_cursor = 0; 12 3 15
if (h->expires_at) hdr->flush_cursor = 0; 3 3 6
if (h->expires_at) hdr->flush_cursor = 0; 3 3 6
if (h->expires_at) hdr->flush_cursor = 0; 3 38 41
if (h->expires_at) hdr->flush_cursor = 0; 3 2 5
if (h->expires_at) hdr->flush_cursor = 0; 3 2 5
if (h->expires_at) hdr->flush_cursor = 0; 13 14 27
if (h->expires_at) hdr->flush_cursor = 0; 3 2 5
if (h->expires_at) hdr->flush_cursor = 0; 3 2 5
3205 129 hdr->table_gen++;
3206 129 return 1;
3207 }
3208 2446 hdr->rz_old_log2 = (uint8_t)__builtin_ctz(old_cap);
3209 2446 hdr->rz_new_log2 = (uint8_t)__builtin_ctz(new_cap);
3210 2446 hdr->rz_seq = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
3211 2446 hdr->rz_move = 0;
3212 2446 hdr->rz_cursor = 0;
3213 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3214 2446 __atomic_store_n(&hdr->rz_phase, SHM_RZ_CLEAN, __ATOMIC_RELEASE);
3215 2446 __atomic_signal_fence(__ATOMIC_SEQ_CST);
3216 2446 SHM_FN(rz_run)(h);
3217 2446 return 1;
3218 }
3219  
3220 3456 static __attribute__((noinline)) void SHM_FN(grow_or_compact)(ShmHandle *h) {
3221 3456 ShmHeader *hdr = h->hdr;
3222 3456 uint32_t size = hdr->size, tomb = hdr->tombstones, cap = hdr->table_cap;
3223 3456 ▶ if (shm_over_load(size, tomb, cap)) {
Branch: 10 branches
BranchTrueFalseTotal
if (shm_over_load(size, tomb, cap)) { 477 302 779
if (shm_over_load(size, tomb, cap)) { 147 422 569
if (shm_over_load(size, tomb, cap)) { 21 3 24
if (shm_over_load(size, tomb, cap)) { 20 3 23
if (shm_over_load(size, tomb, cap)) { 45 39 84
if (shm_over_load(size, tomb, cap)) { 10 2 12
if (shm_over_load(size, tomb, cap)) { 10 2 12
if (shm_over_load(size, tomb, cap)) { 1897 7 1904
if (shm_over_load(size, tomb, cap)) { 26 2 28
if (shm_over_load(size, tomb, cap)) { 19 2 21
3224 2672 ▶ if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) {
Branch: 20 branches
BranchTrueFalseTotal
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 25 452 477
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 25 25
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 73 74 147
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 11 62 73
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 5 16 21
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 5 5
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 5 15 20
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 5 5
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 11 34 45
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 11 11
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 4 6 10
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 4 4
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 4 6 10
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 4 4
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 85 1812 1897
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 1 84 85
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 4 22 26
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 4 4
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 4 15 19
if (h->expires_at && SHM_FN(reclaim_expired)(h, 0, SHM_LRU_NONE) != 0) { 0 4 4
3225 0 size = hdr->size;
3226 0 tomb = hdr->tombstones;
3227 }
3228 /* Grow even while iterating (iterators reset on the table_gen bump), or an
3229 * abandoned each() wedges the table; compaction and shrink stay deferred. */
3230 2672 ▶ if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0)
Branch: 40 branches
BranchTrueFalseTotal
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 25 452 477
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 24 25
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 0 1
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 0 1
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 73 74 147
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 11 62 73
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 11 0 11
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 11 0 11
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 5 16 21
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 5 5
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 5 15 20
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 5 5
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 11 34 45
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 11 11
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 4 6 10
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 4 4
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 4 6 10
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 4 4
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 85 1812 1897
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 84 85
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 0 1
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 1 0 1
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 4 22 26
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 4 4
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 4 15 19
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 4 4
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
if (h->expires_at && tomb >= cap / 16 && !shm_over_load(size, 0, cap) && h->iterating == 0) 0 0 0
3231 0 SHM_FN(resize)(h, cap);
3232 2659 ▶ else if (cap < hdr->max_table_cap)
Branch: 10 branches
BranchTrueFalseTotal
else if (cap < hdr->max_table_cap) 466 10 476
else if (cap < hdr->max_table_cap) 90 46 136
else if (cap < hdr->max_table_cap) 6 15 21
else if (cap < hdr->max_table_cap) 5 15 20
else if (cap < hdr->max_table_cap) 39 6 45
else if (cap < hdr->max_table_cap) 4 6 10
else if (cap < hdr->max_table_cap) 4 6 10
else if (cap < hdr->max_table_cap) 900 996 1896
else if (cap < hdr->max_table_cap) 11 15 26
else if (cap < hdr->max_table_cap) 4 15 19
3233 1529 SHM_FN(resize)(h, cap * 2);
3234 1130 ▶ else if (shm_needs_compaction(size, tomb, cap))
Branch: 10 branches
BranchTrueFalseTotal
else if (shm_needs_compaction(size, tomb, cap)) 1 9 10
else if (shm_needs_compaction(size, tomb, cap)) 0 46 46
else if (shm_needs_compaction(size, tomb, cap)) 0 15 15
else if (shm_needs_compaction(size, tomb, cap)) 0 15 15
else if (shm_needs_compaction(size, tomb, cap)) 0 6 6
else if (shm_needs_compaction(size, tomb, cap)) 0 6 6
else if (shm_needs_compaction(size, tomb, cap)) 0 6 6
else if (shm_needs_compaction(size, tomb, cap)) 146 850 996
else if (shm_needs_compaction(size, tomb, cap)) 0 15 15
else if (shm_needs_compaction(size, tomb, cap)) 0 15 15
3235 0 SHM_FN(resize)(h, cap);
3236 784 ▶ } else if (shm_needs_compaction(size, tomb, cap)) {
Branch: 10 branches
BranchTrueFalseTotal
} else if (shm_needs_compaction(size, tomb, cap)) { 302 0 302
} else if (shm_needs_compaction(size, tomb, cap)) { 422 0 422
} else if (shm_needs_compaction(size, tomb, cap)) { 3 0 3
} else if (shm_needs_compaction(size, tomb, cap)) { 3 0 3
} else if (shm_needs_compaction(size, tomb, cap)) { 39 0 39
} else if (shm_needs_compaction(size, tomb, cap)) { 2 0 2
} else if (shm_needs_compaction(size, tomb, cap)) { 2 0 2
} else if (shm_needs_compaction(size, tomb, cap)) { 7 0 7
} else if (shm_needs_compaction(size, tomb, cap)) { 2 0 2
} else if (shm_needs_compaction(size, tomb, cap)) { 2 0 2
3237 784 ▶ if (h->iterating > 0) { h->deferred = 1; return; }
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) { h->deferred = 1; return; } 0 302 302
if (h->iterating > 0) { h->deferred = 1; return; } 0 422 422
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 39 39
if (h->iterating > 0) { h->deferred = 1; return; } 0 2 2
if (h->iterating > 0) { h->deferred = 1; return; } 0 2 2
if (h->iterating > 0) { h->deferred = 1; return; } 0 7 7
if (h->iterating > 0) { h->deferred = 1; return; } 0 2 2
if (h->iterating > 0) { h->deferred = 1; return; } 0 2 2
3238 784 SHM_FN(resize)(h, cap);
3239 }
3240 }
3241  
3242 static inline __attribute__((always_inline)) void SHM_FN(maybe_grow)(ShmHandle *h) {
3243 400696 const ShmHeader *hdr = h->hdr;
3244 400696 uint32_t size = hdr->size, tomb = hdr->tombstones, cap = hdr->table_cap;
3245 400696 ▶ if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0))
Branch: 84 branches
BranchTrueFalseTotal
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 58 1 59
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 58 58
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 57 0 57
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 57 57
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 61 0 61
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 302 136036 136338
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 0 0
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 67 3 70
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 67 67
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 53 4 57
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 53 53
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 1 3
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 5 1 6
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 422 54566 54988
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 0 0
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 2 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 3 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 0 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 4 0 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 3 225 228
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 1 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 2 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 3 5
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 0 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 4 0 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 3 221 224
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 1 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 3 0 3
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 3 3
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 5 0 5
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 39 21206 21245
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 1 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 0 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 5 0 5
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 240 242
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 1 3
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 0 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 5 0 5
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 230 232
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 31 5 36
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 31 31
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1251 5 1256
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1251 1251
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1246 10 1256
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1246 1246
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 0 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 6 0 6
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 7 180161 180168
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 1 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 2 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 3 5
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 0 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 4 0 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 1222 1224
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 1 1 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 1 1
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 2 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 3 3 6
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 3 3
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 0 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 0 2 2
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 4 0 4
if (__builtin_expect(shm_over_load(size, tomb, cap) || shm_needs_compaction(size, tomb, cap), 0)) 2 228 230
3246 3456 SHM_FN(grow_or_compact)(h);
3247 400696 }
3248  
3249 53332 static inline void SHM_FN(maybe_shrink)(ShmHandle *h) {
3250 53332 ShmHeader *hdr = h->hdr;
3251 53332 ▶ if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) {
Branch: 10 branches
BranchTrueFalseTotal
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 20 12449 12469
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 4 115 119
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 9 12
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 10 13
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 44 47
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 7 10
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 8 11
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 10105 30025 40130
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 4 506 510
if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) { 3 8 11
3252 10151 ▶ if (h->iterating > 0) { h->deferred = 1; return; }
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) { h->deferred = 1; return; } 0 20 20
if (h->iterating > 0) { h->deferred = 1; return; } 0 4 4
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
if (h->iterating > 0) { h->deferred = 1; return; } 10082 23 10105
if (h->iterating > 0) { h->deferred = 1; return; } 0 4 4
if (h->iterating > 0) { h->deferred = 1; return; } 0 3 3
3253 69 SHM_FN(resize)(h, shm_shrink_target(hdr->size, hdr->table_cap));
3254 }
3255 }
3256  
3257 0 static inline void SHM_FN(flush_deferred)(ShmHandle *h) {
3258 0 ▶ if (!h || !h->deferred || h->iterating > 0) return;
Branch: 30 branches
BranchTrueFalseTotal
if (!h || !h->deferred || h->iterating > 0) return; 15 0 15
if (!h || !h->deferred || h->iterating > 0) return; 0 15 15
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 22 0 22
if (!h || !h->deferred || h->iterating > 0) return; 0 22 22
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 1 0 1
if (!h || !h->deferred || h->iterating > 0) return; 0 1 1
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 43 2 45
if (!h || !h->deferred || h->iterating > 0) return; 6 37 43
if (!h || !h->deferred || h->iterating > 0) return; 0 6 6
if (!h || !h->deferred || h->iterating > 0) return; 1 0 1
if (!h || !h->deferred || h->iterating > 0) return; 0 1 1
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
if (!h || !h->deferred || h->iterating > 0) return; 9 0 9
if (!h || !h->deferred || h->iterating > 0) return; 0 9 9
if (!h || !h->deferred || h->iterating > 0) return; 0 0 0
3259 0 h->deferred = 0;
3260 /* A deferred resize outlives freeze(), and no caller here has a frozen guard. */
3261 0 ▶ if (h->readonly || shm_is_sealed(h)) return;
Branch: 20 branches
BranchTrueFalseTotal
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 4 2 6
if (h->readonly || shm_is_sealed(h)) return; 0 4 4
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
if (h->readonly || shm_is_sealed(h)) return; 0 0 0
3262 0 ShmHeader *hdr = h->hdr;
3263 0 shm_rwlock_wrlock(h);
3264 /* Re-test under the lock: freeze() seals under it. */
3265 0 ▶ if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; }
Branch: 10 branches
BranchTrueFalseTotal
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 4 4
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; } 0 0 0
3266 0 shm_seqlock_write_begin(&hdr->seq);
3267 0 uint32_t size = hdr->size, tomb = hdr->tombstones, cap = hdr->table_cap;
3268 0 ▶ if (shm_over_load(size, tomb, cap)) {
Branch: 10 branches
BranchTrueFalseTotal
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 4 4
if (shm_over_load(size, tomb, cap)) { 0 0 0
if (shm_over_load(size, tomb, cap)) { 0 0 0
3269 0 ▶ if (cap < hdr->max_table_cap)
Branch: 10 branches
BranchTrueFalseTotal
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
if (cap < hdr->max_table_cap) 0 0 0
3270 0 SHM_FN(resize)(h, cap * 2);
3271 0 ▶ else if (shm_needs_compaction(size, tomb, cap))
Branch: 10 branches
BranchTrueFalseTotal
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
else if (shm_needs_compaction(size, tomb, cap)) 0 0 0
3272 0 SHM_FN(resize)(h, cap);
3273 0 ▶ } else if (shm_under_load(size, cap)) {
Branch: 10 branches
BranchTrueFalseTotal
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 4 0 4
} else if (shm_under_load(size, cap)) { 0 0 0
} else if (shm_under_load(size, cap)) { 0 0 0
3274 0 SHM_FN(resize)(h, shm_shrink_target(size, cap));
3275 0 ▶ } else if (shm_needs_compaction(size, tomb, cap)) {
Branch: 10 branches
BranchTrueFalseTotal
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
} else if (shm_needs_compaction(size, tomb, cap)) { 0 0 0
3276 0 SHM_FN(resize)(h, cap);
3277 }
3278 0 shm_seqlock_write_end(&hdr->seq);
3279 0 shm_rwlock_wrunlock(h);
3280 }
3281  
3282 /* Caller holds wrlock + seqlock. 0 when the arena or table is full. */
3283 397732 static int SHM_FN(put_inner)(ShmHandle *h,
3284 #ifdef SHM_KEY_IS_INT
3285 SHM_KEY_INT_TYPE key,
3286 #else
3287 const char *key_str, uint32_t key_len, bool key_utf8,
3288 #endif
3289 #ifdef SHM_VAL_IS_STR
3290 const char *val_str, uint32_t val_len, bool val_utf8,
3291 #else
3292 SHM_VAL_INT_TYPE value,
3293 #endif
3294 uint32_t ttl_sec
3295 ) {
3296 397732 ShmHeader *hdr = h->hdr;
3297 397732 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3298 397732 uint8_t *states = h->states;
3299  
3300 SHM_FN(maybe_grow)(h);
3301 397732 uint32_t mask = hdr->table_cap - 1;
3302 #ifdef SHM_KEY_IS_INT
3303 205313 uint32_t hash = SHM_HASH_KEY(key);
3304 #else
3305 192419 shm_sig_hold_bytes(h, key_len);
3306 192419 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3307 #endif
3308 397732 uint32_t pos = hash & mask;
3309 397732 uint32_t insert_pos = UINT32_MAX;
3310  
3311 397732 uint32_t exp_ts = 0;
3312 397732 ▶ if (h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) { 1600 135214 136814
if (h->expires_at) { 4188 50936 55124
if (h->expires_at) { 159 84 243
if (h->expires_at) { 161 77 238
if (h->expires_at) { 1188 20101 21289
if (h->expires_at) { 180 71 251
if (h->expires_at) { 179 62 241
if (h->expires_at) { 39717 142328 182045
if (h->expires_at) { 159 1085 1244
if (h->expires_at) { 161 82 243
3313 47692 ▶ uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1554 46 1600
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1811 2377 4188
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 121 38 159
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 123 38 161
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1143 45 1188
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 136 44 180
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 135 44 179
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 21415 18302 39717
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 121 38 159
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 123 38 161
3314 47692 ▶ if (ttl > 0)
Branch: 10 branches
BranchTrueFalseTotal
if (ttl > 0) 1597 3 1600
if (ttl > 0) 4185 3 4188
if (ttl > 0) 156 3 159
if (ttl > 0) 158 3 161
if (ttl > 0) 1185 3 1188
if (ttl > 0) 177 3 180
if (ttl > 0) 176 3 179
if (ttl > 0) 37649 2068 39717
if (ttl > 0) 156 3 159
if (ttl > 0) 158 3 161
3315 45597 exp_ts = shm_expiry_ts(ttl);
3316 }
3317  
3318 397732 uint8_t tag = SHM_MAKE_TAG(hash);
3319 1351789 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 370223 2 370225
for (uint32_t i = 0; i <= mask; i++) { 180890 4 180894
for (uint32_t i = 0; i <= mask; i++) { 434 3 437
for (uint32_t i = 0; i <= mask; i++) { 425 3 428
for (uint32_t i = 0; i <= mask; i++) { 69866 2 69868
for (uint32_t i = 0; i <= mask; i++) { 500 2 502
for (uint32_t i = 0; i <= mask; i++) { 482 2 484
for (uint32_t i = 0; i <= mask; i++) { 724385 171 724556
for (uint32_t i = 0; i <= mask; i++) { 3870 3 3873
for (uint32_t i = 0; i <= mask; i++) { 519 3 522
3320 1351594 uint32_t idx = (pos + i) & mask;
3321 1351594 uint8_t st = states[idx];
3322 1351594 __builtin_prefetch(&nodes[idx], 0, 1);
3323 1351594 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
3324  
3325 1351594 ▶ if (st == SHM_EMPTY) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 136719 233504 370223
if (st == SHM_EMPTY) { 55120 125770 180890
if (st == SHM_EMPTY) { 240 194 434
if (st == SHM_EMPTY) { 235 190 425
if (st == SHM_EMPTY) { 21275 48591 69866
if (st == SHM_EMPTY) { 237 263 500
if (st == SHM_EMPTY) { 227 255 482
if (st == SHM_EMPTY) { 178286 546099 724385
if (st == SHM_EMPTY) { 1241 2629 3870
if (st == SHM_EMPTY) { 240 279 519
3326 393820 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 128285 8434 136719
if (insert_pos == UINT32_MAX) insert_pos = idx; 52316 2804 55120
if (insert_pos == UINT32_MAX) insert_pos = idx; 240 0 240
if (insert_pos == UINT32_MAX) insert_pos = idx; 234 1 235
if (insert_pos == UINT32_MAX) insert_pos = idx; 21275 0 21275
if (insert_pos == UINT32_MAX) insert_pos = idx; 237 0 237
if (insert_pos == UINT32_MAX) insert_pos = idx; 227 0 227
if (insert_pos == UINT32_MAX) insert_pos = idx; 173885 4401 178286
if (insert_pos == UINT32_MAX) insert_pos = idx; 1241 0 1241
if (insert_pos == UINT32_MAX) insert_pos = idx; 240 0 240
3327 393820 break;
3328 }
3329 957774 ▶ if (st == SHM_TOMBSTONE) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 12552 220952 233504
if (st == SHM_TOMBSTONE) { 5268 120502 125770
if (st == SHM_TOMBSTONE) { 0 194 194
if (st == SHM_TOMBSTONE) { 1 189 190
if (st == SHM_TOMBSTONE) { 0 48591 48591
if (st == SHM_TOMBSTONE) { 0 263 263
if (st == SHM_TOMBSTONE) { 0 255 255
if (st == SHM_TOMBSTONE) { 5462 540637 546099
if (st == SHM_TOMBSTONE) { 0 2629 2629
if (st == SHM_TOMBSTONE) { 0 279 279
3330 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 8434 4118 12552
if (insert_pos == UINT32_MAX) insert_pos = idx; 2804 2464 5268
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 4618 844 5462
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
3331 0 continue;
3332 }
3333 934491 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 219953 999 220952
if (st != tag) continue; 120014 488 120502
if (st != tag) continue; 193 1 194
if (st != tag) continue; 188 1 189
if (st != tag) continue; 48386 205 48591
if (st != tag) continue; 250 13 263
if (st != tag) continue; 242 13 255
if (st != tag) continue; 534738 5899 540637
if (st != tag) continue; 2621 8 2629
if (st != tag) continue; 278 1 279
3334 #ifdef SHM_KEY_IS_INT
3335 6139 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 12 193 205
if (SHM_KEY_EQ(&nodes[idx], key)) { 12 1 13
if (SHM_KEY_EQ(&nodes[idx], key)) { 12 1 13
if (SHM_KEY_EQ(&nodes[idx], key)) { 3588 2311 5899
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 8 8
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 1 1
3336 #else
3337 1489 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 93 906 999
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 488 488
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 1 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 1 1
3338 #endif
3339 #ifdef SHM_VAL_IS_STR
3340 {
3341 /* Before old_off is read: a slide may move the old block. */
3342 129 SHM_FN(arena_reclaim)(h);
3343 129 uint32_t old_off = nodes[idx].val_off;
3344 129 uint32_t old_lf = nodes[idx].val_len;
3345 129 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx))
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) 14 79 93
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) 12 0 12
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) 12 0 12
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) 12 0 12
3346 50 return 0;
3347 0 shm_str_free(hdr, h->arena, old_off, old_lf);
3348 }
3349 #else
3350 0 __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
3351 #endif
3352 0 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 1 78 79
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 16 3572 3588
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
3353 0 ▶ if (h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) { 2 77 79
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 3588 3588
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
3354 0 ▶ if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0)
Branch: 20 branches
BranchTrueFalseTotal
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 2 0 2
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 2 0 2
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0) 0 0 0
3355 0 h->expires_at[idx] = exp_ts;
3356 }
3357 0 return 1;
3358 }
3359 }
3360  
3361 394015 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX)
Branch: 10 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 2 136719 136721
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 4 55120 55124
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 3 240 243
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 3 235 238
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 2 21275 21277
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 2 237 239
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 2 227 229
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 171 178286 178457
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 3 1241 1244
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) 3 240 243
3362 195 return 0;
3363  
3364 393820 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), SHM_VAL_SLEN(val_len));
3365  
3366 393820 int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
3367 393820 SHM_FN(arena_reclaim)(h);
3368  
3369 #ifdef SHM_KEY_IS_INT
3370 201506 nodes[insert_pos].key = key;
3371 #else
3372 192314 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE))
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) 1 136718 136719
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) 2 55118 55120
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) 0 240 240
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) 0 235 235
3373 0 return 0;
3374 #endif
3375  
3376 #ifdef SHM_VAL_IS_STR
3377 158457 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 49293 87425 136718
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 1 21274 21275
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 237 237
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 227 227
3378 #ifndef SHM_KEY_IS_INT
3379 49293 shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
3380 #endif
3381 0 return 0;
3382 }
3383 #else
3384 235360 __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
3385 #endif
3386  
3387 344523 ▶ if (h->expires_at) h->expires_at[insert_pos] = exp_ts;
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 1584 85841 87425
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 4186 50932 55118
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 159 81 240
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 161 74 235
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 1175 20099 21274
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 168 69 237
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 167 60 227
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 39700 138586 178286
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 159 1082 1241
if (h->expires_at) h->expires_at[insert_pos] = exp_ts; 161 79 240
3388 344523 shm_publish_tag(states, insert_pos, SHM_MAKE_TAG(hash));
3389 /* tombstones-- before size++, or a saturated map briefly reads as
3390 * size + tombstones > table_cap, which an attach refuses. */
3391 344523 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 10 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 7811 79614 87425
if (was_tombstone) hdr->tombstones--; 2804 52314 55118
if (was_tombstone) hdr->tombstones--; 0 240 240
if (was_tombstone) hdr->tombstones--; 1 234 235
if (was_tombstone) hdr->tombstones--; 0 21274 21274
if (was_tombstone) hdr->tombstones--; 0 237 237
if (was_tombstone) hdr->tombstones--; 0 227 227
if (was_tombstone) hdr->tombstones--; 4401 173885 178286
if (was_tombstone) hdr->tombstones--; 0 1241 1241
if (was_tombstone) hdr->tombstones--; 0 240 240
3392 344523 hdr->size++;
3393  
3394 344523 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 33931 53494 87425
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 10032 45086 55118
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 220 240
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 215 235
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 21254 21274
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 217 237
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 207 227
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 10808 167478 178286
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 1221 1241
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 20 220 240
3395  
3396 344523 return 1;
3397 }
3398  
3399 367700 static int SHM_FN(put_impl)(ShmHandle *h,
3400 #ifdef SHM_KEY_IS_INT
3401 SHM_KEY_INT_TYPE key,
3402 #else
3403 const char *key_str, uint32_t key_len, bool key_utf8,
3404 #endif
3405 #ifdef SHM_VAL_IS_STR
3406 const char *val_str, uint32_t val_len, bool val_utf8,
3407 #else
3408 SHM_VAL_INT_TYPE value,
3409 #endif
3410 uint32_t ttl_sec
3411 ) {
3412 #ifdef SHM_KEY_IS_INT
3413 195288 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 10054 1234 11288
SHM_SHARD_DISPATCH(h, key); 55 195 250
SHM_SHARD_DISPATCH(h, key); 55 185 240
SHM_SHARD_DISPATCH(h, key); 33466 148557 182023
SHM_SHARD_DISPATCH(h, key); 55 1189 1244
SHM_SHARD_DISPATCH(h, key); 55 188 243
3414 #else
3415 172412 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 15903 110907 126810
SHM_SHARD_DISPATCH(h, key_str, key_len); 10254 34869 45123
SHM_SHARD_DISPATCH(h, key_str, key_len); 54 188 242
SHM_SHARD_DISPATCH(h, key_str, key_len); 54 183 237
3416 #endif
3417 367700 shm_rwlock_wrlock(h);
3418 367700 shm_seqlock_write_begin(&h->hdr->seq);
3419 367700 int rc = SHM_FN(put_inner)(h,
3420 #ifdef SHM_KEY_IS_INT
3421 key,
3422 #else
3423 key_str, key_len, key_utf8,
3424 #endif
3425 #ifdef SHM_VAL_IS_STR
3426 val_str, val_len, val_utf8,
3427 #else
3428 value,
3429 #endif
3430 ttl_sec);
3431 367700 shm_seqlock_write_end(&h->hdr->seq);
3432 367700 shm_rwlock_wrunlock(h);
3433 367700 return rc;
3434 }
3435  
3436 346690 static inline int SHM_FN(put)(ShmHandle *h,
3437 #ifdef SHM_KEY_IS_INT
3438 SHM_KEY_INT_TYPE key,
3439 #else
3440 const char *key_str, uint32_t key_len, bool key_utf8,
3441 #endif
3442 #ifdef SHM_VAL_IS_STR
3443 const char *val_str, uint32_t val_len, bool val_utf8
3444 #else
3445 SHM_VAL_INT_TYPE value
3446 #endif
3447 ) {
3448 346690 return SHM_FN(put_impl)(h,
3449 #ifdef SHM_KEY_IS_INT
3450 key,
3451 #else
3452 key_str, key_len, key_utf8,
3453 #endif
3454 #ifdef SHM_VAL_IS_STR
3455 val_str, val_len, val_utf8,
3456 #else
3457 value,
3458 #endif
3459 SHM_TTL_USE_DEFAULT);
3460 }
3461  
3462 21010 static inline int SHM_FN(put_ttl)(ShmHandle *h,
3463 #ifdef SHM_KEY_IS_INT
3464 SHM_KEY_INT_TYPE key,
3465 #else
3466 const char *key_str, uint32_t key_len, bool key_utf8,
3467 #endif
3468 #ifdef SHM_VAL_IS_STR
3469 const char *val_str, uint32_t val_len, bool val_utf8,
3470 #else
3471 SHM_VAL_INT_TYPE value,
3472 #endif
3473 uint32_t ttl_sec
3474 ) {
3475 21010 ▶ if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
Branch: 10 branches
BranchTrueFalseTotal
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 46 46
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2377 2377
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 38 38
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 38 38
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 45 45
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 44 44
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 44 44
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 1 18301 18302
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 38 38
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 38 38
3476 21010 return SHM_FN(put_impl)(h,
3477 #ifdef SHM_KEY_IS_INT
3478 key,
3479 #else
3480 key_str, key_len, key_utf8,
3481 #endif
3482 #ifdef SHM_VAL_IS_STR
3483 val_str, val_len, val_utf8,
3484 #else
3485 value,
3486 #endif
3487 ttl_sec);
3488 }
3489  
3490 /* ---- Get (seqlock -- lock-free read path) ---- */
3491  
3492 #ifndef SHM_KEY_IS_INT
3493 /* -1: arena offset out of bounds (a torn record, or corruption). */
3494 1948504 static inline int SHM_FN(key_at)(const ShmHandle *h, const SHM_NODE_TYPE *n,
3495 const char *key_str, uint32_t key_len, uint64_t arena_cap) {
3496 1948504 uint32_t kl_packed = n->key_len;
3497 1948504 ▶ uint32_t kl = SHM_STR_LEN(kl_packed);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(kl_packed); 1931537 16839 1948376
uint32_t kl = SHM_STR_LEN(kl_packed); 58 9 67
uint32_t kl = SHM_STR_LEN(kl_packed); 28 0 28
uint32_t kl = SHM_STR_LEN(kl_packed); 33 0 33
3498 1948504 ▶ if (kl != key_len) return 0;
Branch: 4 branches
BranchTrueFalseTotal
if (kl != key_len) return 0; 14 1948362 1948376
if (kl != key_len) return 0; 0 67 67
if (kl != key_len) return 0; 0 28 28
if (kl != key_len) return 0; 0 33 33
3499 1948490 ▶ if (SHM_IS_INLINE(kl_packed)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_IS_INLINE(kl_packed)) { 1931524 16838 1948362
if (SHM_IS_INLINE(kl_packed)) { 58 9 67
if (SHM_IS_INLINE(kl_packed)) { 28 0 28
if (SHM_IS_INLINE(kl_packed)) { 33 0 33
3500 char ibuf[SHM_INLINE_MAX];
3501 1931643 shm_inline_read(n->key_off, kl_packed, ibuf);
3502 1931643 return memcmp(ibuf, key_str, kl) == 0;
3503 }
3504 0 uint32_t koff = n->key_off;
3505 0 ▶ if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1;
Branch: 8 branches
BranchTrueFalseTotal
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 16838 0 16838
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 16838 16838
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 9 0 9
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 9 9
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 0 0
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 0 0
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 0 0
if (koff < SHM_ARENA_MIN_ALLOC || (uint64_t)koff + kl > arena_cap) return -1; 0 0 0
3506 0 return memcmp(h->arena + koff, key_str, kl) == 0;
3507 }
3508 #endif
3509  
3510 3666089 static inline uint32_t SHM_FN(probe_lockfree)(const ShmHandle *h, uint32_t mask, uint32_t hash,
3511 #ifdef SHM_KEY_IS_INT
3512 SHM_KEY_INT_TYPE key
3513 #else
3514 const char *key_str, uint32_t key_len, uint64_t arena_cap
3515 #endif
3516 ) {
3517 3666089 const SHM_NODE_TYPE *nodes = (const SHM_NODE_TYPE *)h->nodes;
3518 3666089 const uint8_t *states = h->states;
3519 3666089 uint32_t pos = hash & mask;
3520 3666089 uint8_t tag = SHM_MAKE_TAG(hash);
3521 3666089 uint32_t i = 0;
3522 3666089 __builtin_prefetch(&nodes[pos], 0, 1);
3523 #ifdef SHM_KEY_IS_INT
3524 #define SHM_PROBE_MATCH(idx) \
3525 if (SHM_KEY_EQ(&nodes[idx], key)) return (idx)
3526 #else
3527 #define SHM_PROBE_MATCH(idx) do { \
3528 int r_ = SHM_FN(key_at)(h, &nodes[idx], key_str, key_len, arena_cap); \
3529 if (r_) return r_ > 0 ? (idx) : UINT32_MAX; \
3530 } while (0)
3531 #endif
3532 #ifdef __SSE2__
3533 /* Bound by the clamped mask, never a re-read of the peer-writable table_cap. */
3534 3666089 ▶ if (pos + 16 <= mask + 1) {
Branch: 10 branches
BranchTrueFalseTotal
if (pos + 16 <= mask + 1) { 322058 1626396 1948454
if (pos + 16 <= mask + 1) { 4 70 74
if (pos + 16 <= mask + 1) { 1 31 32
if (pos + 16 <= mask + 1) { 0 38 38
if (pos + 16 <= mask + 1) { 2 55 57
if (pos + 16 <= mask + 1) { 2 76 78
if (pos + 16 <= mask + 1) { 2 81 83
if (pos + 16 <= mask + 1) { 46221 1670925 1717146
if (pos + 16 <= mask + 1) { 1 60 61
if (pos + 16 <= mask + 1) { 4 62 66
3535 uint16_t mmask, emask;
3536 0 shm_probe_group(states, pos, tag, &mmask, &emask);
3537 0 ▶ uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF;
Branch: 10 branches
BranchTrueFalseTotal
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 322041 17 322058
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 4 0 4
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 1 0 1
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 0 0 0
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 2 0 2
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 2 0 2
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 2 0 2
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 44390 1831 46221
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 1 0 1
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 4 0 4
3538 0 ▶ for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 322010 74 322084
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 4 0 4
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 1 0 1
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 0 0 0
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 2 0 2
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 2 0 2
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 2 0 2
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 42595 3770 46365
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 1 0 1
for (uint16_t relevant = mmask & cutoff; relevant; relevant &= relevant - 1) { 3 1 4
3539 0 uint32_t idx = pos + (uint32_t)__builtin_ctz(relevant);
3540 0 ▶ SHM_PROBE_MATCH(idx);
Branch: 14 branches
BranchTrueFalseTotal
SHM_PROBE_MATCH(idx); 321984 26 322010
SHM_PROBE_MATCH(idx); 321984 0 321984
SHM_PROBE_MATCH(idx); 4 0 4
SHM_PROBE_MATCH(idx); 4 0 4
SHM_PROBE_MATCH(idx); 1 0 1
SHM_PROBE_MATCH(idx); 1 0 1
SHM_PROBE_MATCH(idx); 0 0 0
SHM_PROBE_MATCH(idx); 0 0 0
SHM_PROBE_MATCH(idx); 2 0 2
SHM_PROBE_MATCH(idx); 2 0 2
SHM_PROBE_MATCH(idx); 2 0 2
SHM_PROBE_MATCH(idx); 42451 144 42595
SHM_PROBE_MATCH(idx); 1 0 1
SHM_PROBE_MATCH(idx); 3 0 3
3541 }
3542 0 ▶ if (emask) return UINT32_MAX;
Branch: 10 branches
BranchTrueFalseTotal
if (emask) return UINT32_MAX; 72 2 74
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 3663 107 3770
if (emask) return UINT32_MAX; 0 0 0
if (emask) return UINT32_MAX; 1 0 1
3543 0 i = 16;
3544 }
3545 #endif
3546 4549036 ▶ for (; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (; i <= mask; i++) { 2042475 1 2042476
for (; i <= mask; i++) { 95 1 96
for (; i <= mask; i++) { 49 1 50
for (; i <= mask; i++) { 57 1 58
for (; i <= mask; i++) { 73 1 74
for (; i <= mask; i++) { 95 1 96
for (; i <= mask; i++) { 100 1 101
for (; i <= mask; i++) { 2505925 2 2505927
for (; i <= mask; i++) { 77 1 78
for (; i <= mask; i++) { 79 1 80
3547 4549025 uint32_t idx = (pos + i) & mask;
3548 4549025 uint8_t st = states[idx];
3549 4549025 __builtin_prefetch(&nodes[idx], 0, 1);
3550 4549025 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
3551 4549025 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 31 2042444 2042475
if (st == SHM_EMPTY) break; 6 89 95
if (st == SHM_EMPTY) break; 3 46 49
if (st == SHM_EMPTY) break; 4 53 57
if (st == SHM_EMPTY) break; 1 72 73
if (st == SHM_EMPTY) break; 1 94 95
if (st == SHM_EMPTY) break; 3 97 100
if (st == SHM_EMPTY) break; 1023 2504902 2505925
if (st == SHM_EMPTY) break; 2 75 77
if (st == SHM_EMPTY) break; 3 76 79
3552 4547948 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 416078 1626366 2042444
if (st != tag) continue; 26 63 89
if (st != tag) continue; 19 27 46
if (st != tag) continue; 20 33 53
if (st != tag) continue; 19 53 72
if (st != tag) continue; 20 74 94
if (st != tag) continue; 20 77 97
if (st != tag) continue; 834886 1670016 2504902
if (st != tag) continue; 18 57 75
if (st != tag) continue; 18 58 76
3553 3296824 ▶ SHM_PROBE_MATCH(idx);
Branch: 14 branches
BranchTrueFalseTotal
SHM_PROBE_MATCH(idx); 1626366 0 1626366
SHM_PROBE_MATCH(idx); 1626366 0 1626366
SHM_PROBE_MATCH(idx); 63 0 63
SHM_PROBE_MATCH(idx); 63 0 63
SHM_PROBE_MATCH(idx); 27 0 27
SHM_PROBE_MATCH(idx); 27 0 27
SHM_PROBE_MATCH(idx); 33 0 33
SHM_PROBE_MATCH(idx); 33 0 33
SHM_PROBE_MATCH(idx); 53 0 53
SHM_PROBE_MATCH(idx); 74 0 74
SHM_PROBE_MATCH(idx); 77 0 77
SHM_PROBE_MATCH(idx); 1670007 9 1670016
SHM_PROBE_MATCH(idx); 57 0 57
SHM_PROBE_MATCH(idx); 58 0 58
3554 }
3555 #undef SHM_PROBE_MATCH
3556 1088 return UINT32_MAX;
3557 }
3558  
3559 3655855 static int SHM_FN(get)(ShmHandle *h,
3560 #ifdef SHM_KEY_IS_INT
3561 SHM_KEY_INT_TYPE key,
3562 #else
3563 const char *key_str, uint32_t key_len, bool key_utf8,
3564 #endif
3565 #ifdef SHM_VAL_IS_STR
3566 const char **out_str, uint32_t *out_len, bool *out_utf8
3567 #else
3568 SHM_VAL_INT_TYPE *out_value
3569 #endif
3570 ) {
3571 #ifdef SHM_KEY_IS_INT
3572 1715731 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 10 34 44
SHM_SHARD_DISPATCH(h, key); 30 37 67
SHM_SHARD_DISPATCH(h, key); 30 41 71
SHM_SHARD_DISPATCH(h, key); 1690286 25138 1715424
SHM_SHARD_DISPATCH(h, key); 30 31 61
SHM_SHARD_DISPATCH(h, key); 30 34 64
3573 #else
3574 1940124 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 1664044 275963 1940007
SHM_SHARD_DISPATCH(h, key_str, key_len); 10 44 54
SHM_SHARD_DISPATCH(h, key_str, key_len); 10 19 29
SHM_SHARD_DISPATCH(h, key_str, key_len); 10 24 34
3575 #endif
3576 #ifndef SHM_KEY_IS_INT
3577 (void)key_utf8;
3578 #endif
3579 3655855 ShmHeader *hdr = h->hdr;
3580 3655855 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3581 3655855 uint32_t max_mask = h->max_mask;
3582 #if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
3583 1940306 uint64_t arena_cap = hdr->arena_cap;
3584 #endif
3585 #ifndef SHM_KEY_IS_INT
3586 1940124 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3587 #else
3588 1715731 uint32_t hash = SHM_HASH_KEY(key);
3589 #endif
3590  
3591 3655855 int locked = 0;
3592 #define SHM_GET_RETURN(v) do { if (locked) shm_rwlock_rdunlock(h); return (v); } while (0)
3593 3664298 for (int pass = 0; ; pass++) {
3594 3664298 ▶ if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) {
Branch: 20 branches
BranchTrueFalseTotal
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 1948433 1948433
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 54 54
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 29 29
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 34 34
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 51 51
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 72 72
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 76 76
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 1715424 1715424
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 61 61
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 64 64
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
3595 0 shm_rwlock_rdlock(h);
3596 0 locked = 1;
3597 }
3598 3664298 uint32_t seq = shm_seqlock_read_begin(h);
3599  
3600 3664298 uint32_t mask = (hdr->table_cap - 1) & max_mask;
3601 3664298 ▶ if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1);
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 71 1948362 1948433
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 13 41 54
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 10 19 29
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 10 24 34
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 22 29 51
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 21 51 72
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 21 55 76
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 1107 1714317 1715424
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 10 51 61
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 10 54 64
3602 3664298 ▶ if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 79 1948354 1948433
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 8 46 54
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 23 29
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 28 34
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 45 51
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 66 72
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 70 76
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 2217 1713207 1715424
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 55 61
if (h->lru_accessed) __builtin_prefetch(&h->lru_accessed[hash & mask], 0, 1); 6 58 64
3603 #ifdef SHM_KEY_IS_INT
3604 1715748 uint32_t local_idx = SHM_FN(probe_lockfree)(h, mask, hash, key);
3605 #else
3606 1948550 uint32_t local_idx = SHM_FN(probe_lockfree)(h, mask, hash, key_str, key_len, arena_cap);
3607 #endif
3608 3664298 int found = local_idx != UINT32_MAX;
3609 3664298 ▶ if (found && h->expires_at) {
Branch: 20 branches
BranchTrueFalseTotal
if (found && h->expires_at) { 1948335 98 1948433
if (found && h->expires_at) { 71 1948264 1948335
if (found && h->expires_at) { 51 3 54
if (found && h->expires_at) { 13 38 51
if (found && h->expires_at) { 27 2 29
if (found && h->expires_at) { 10 17 27
if (found && h->expires_at) { 31 3 34
if (found && h->expires_at) { 10 21 31
if (found && h->expires_at) { 49 2 51
if (found && h->expires_at) { 22 27 49
if (found && h->expires_at) { 70 2 72
if (found && h->expires_at) { 21 49 70
if (found && h->expires_at) { 72 4 76
if (found && h->expires_at) { 21 51 72
if (found && h->expires_at) { 1711146 4278 1715424
if (found && h->expires_at) { 1107 1710039 1711146
if (found && h->expires_at) { 58 3 61
if (found && h->expires_at) { 10 48 58
if (found && h->expires_at) { 60 4 64
if (found && h->expires_at) { 10 50 60
3610 1295 uint32_t exp = h->expires_at[local_idx];
3611 1295 ▶ if (exp != 0 && shm_now() >= exp) found = 0;
Branch: 20 branches
BranchTrueFalseTotal
if (exp != 0 && shm_now() >= exp) found = 0; 65 6 71
if (exp != 0 && shm_now() >= exp) found = 0; 34 31 65
if (exp != 0 && shm_now() >= exp) found = 0; 7 6 13
if (exp != 0 && shm_now() >= exp) found = 0; 3 4 7
if (exp != 0 && shm_now() >= exp) found = 0; 4 6 10
if (exp != 0 && shm_now() >= exp) found = 0; 3 1 4
if (exp != 0 && shm_now() >= exp) found = 0; 4 6 10
if (exp != 0 && shm_now() >= exp) found = 0; 3 1 4
if (exp != 0 && shm_now() >= exp) found = 0; 16 6 22
if (exp != 0 && shm_now() >= exp) found = 0; 3 13 16
if (exp != 0 && shm_now() >= exp) found = 0; 15 6 21
if (exp != 0 && shm_now() >= exp) found = 0; 3 12 15
if (exp != 0 && shm_now() >= exp) found = 0; 15 6 21
if (exp != 0 && shm_now() >= exp) found = 0; 3 12 15
if (exp != 0 && shm_now() >= exp) found = 0; 1098 9 1107
if (exp != 0 && shm_now() >= exp) found = 0; 66 1032 1098
if (exp != 0 && shm_now() >= exp) found = 0; 4 6 10
if (exp != 0 && shm_now() >= exp) found = 0; 3 1 4
if (exp != 0 && shm_now() >= exp) found = 0; 4 6 10
if (exp != 0 && shm_now() >= exp) found = 0; 3 1 4
3612 }
3613  
3614 3664298 ▶ if (found) {
Branch: 10 branches
BranchTrueFalseTotal
if (found) { 1948301 132 1948433
if (found) { 48 6 54
if (found) { 24 5 29
if (found) { 28 6 34
if (found) { 46 5 51
if (found) { 67 5 72
if (found) { 69 7 76
if (found) { 1711080 4344 1715424
if (found) { 55 6 61
if (found) { 57 7 64
3615 #ifdef SHM_VAL_IS_STR
3616 1948483 uint32_t local_vlen_packed = nodes[local_idx].val_len;
3617 1948483 ▶ uint32_t local_vl = SHM_STR_LEN(local_vlen_packed);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t local_vl = SHM_STR_LEN(local_vlen_packed); 233 1948068 1948301
uint32_t local_vl = SHM_STR_LEN(local_vlen_packed); 29 17 46
uint32_t local_vl = SHM_STR_LEN(local_vlen_packed); 51 16 67
uint32_t local_vl = SHM_STR_LEN(local_vlen_packed); 56 13 69
3618 1948483 uint32_t local_voff = nodes[local_idx].val_off;
3619 1948483 ▶ if (SHM_IS_INLINE(local_vlen_packed)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_IS_INLINE(local_vlen_packed)) { 233 1948068 1948301
if (SHM_IS_INLINE(local_vlen_packed)) { 29 17 46
if (SHM_IS_INLINE(local_vlen_packed)) { 51 16 67
if (SHM_IS_INLINE(local_vlen_packed)) { 56 13 69
3620 369 ▶ if (local_vl > h->copy_buf_size || !h->copy_buf) {
Branch: 8 branches
BranchTrueFalseTotal
if (local_vl > h->copy_buf_size || !h->copy_buf) { 233 0 233
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 233 233
if (local_vl > h->copy_buf_size || !h->copy_buf) { 29 0 29
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 29 29
if (local_vl > h->copy_buf_size || !h->copy_buf) { 51 0 51
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 51 51
if (local_vl > h->copy_buf_size || !h->copy_buf) { 56 0 56
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 56 56
3621 0 ▶ if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0);
Branch: 12 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
3622 0 continue;
3623 }
3624 369 shm_inline_read(local_voff, local_vlen_packed, h->copy_buf);
3625 } else {
3626 1948114 ▶ if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) {
Branch: 8 branches
BranchTrueFalseTotal
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 1948068 0 1948068
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 0 1948068 1948068
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 17 0 17
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 0 17 17
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 16 0 16
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 0 16 16
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 13 0 13
if (local_voff < SHM_ARENA_MIN_ALLOC || (uint64_t)local_voff + local_vl > arena_cap) { 0 13 13
3627 /* Retry only if seq moved (torn); stable corruption reads as zeros. */
3628 0 ▶ if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
Branch: 4 branches
BranchTrueFalseTotal
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 0 0
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 0 0
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 0 0
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 0 0
3629 0 ▶ if (local_vl > h->copy_buf_size || !h->copy_buf) {
Branch: 8 branches
BranchTrueFalseTotal
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 0 0
3630 0 ▶ if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0);
Branch: 12 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
3631 0 continue;
3632 }
3633 0 memset(h->copy_buf, 0, local_vl);
3634 } else {
3635 1948114 ▶ if (local_vl > h->copy_buf_size || !h->copy_buf) {
Branch: 8 branches
BranchTrueFalseTotal
if (local_vl > h->copy_buf_size || !h->copy_buf) { 1948043 25 1948068
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 1948043 1948043
if (local_vl > h->copy_buf_size || !h->copy_buf) { 10 7 17
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 10 10
if (local_vl > h->copy_buf_size || !h->copy_buf) { 11 5 16
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 11 11
if (local_vl > h->copy_buf_size || !h->copy_buf) { 8 5 13
if (local_vl > h->copy_buf_size || !h->copy_buf) { 0 8 8
3636 42 ▶ if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0);
Branch: 12 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 25 0 25
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 25 25
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 7 0 7
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 7 7
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 5 0 5
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 5 5
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 5 0 5
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 5 5
if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) SHM_GET_RETURN(0); 0 0 0
3637 42 continue;
3638 }
3639 1948072 memcpy(h->copy_buf, h->arena + local_voff, local_vl);
3640 }
3641 }
3642 #else
3643 1711292 SHM_VAL_INT_TYPE local_value = __atomic_load_n(&nodes[local_idx].value, __ATOMIC_RELAXED);
3644 #endif
3645 3659733 ▶ if (shm_seqlock_read_retry(&hdr->seq, seq)) {
Branch: 10 branches
BranchTrueFalseTotal
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 8401 1939875 1948276
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 48 48
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 24 24
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 28 28
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 39 39
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 62 62
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 64 64
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 1711080 1711080
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 55 55
if (shm_seqlock_read_retry(&hdr->seq, seq)) { 0 57 57
3646 #ifdef SHM_VAL_IS_STR
3647 0 ▶ if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1)
Branch: 8 branches
BranchTrueFalseTotal
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 8401 8401
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
if (local_vl >= SHM_READ_TORN_BIG && pass < SHM_READ_TRIES_UNLOCKED - 1) 0 0 0
3648 0 pass = SHM_READ_TRIES_UNLOCKED - 1;
3649 #endif
3650 0 continue;
3651 }
3652  
3653 3651332 shm_lru_mark(h, local_idx);
3654 #ifdef SHM_VAL_IS_STR
3655 1940040 *out_str = h->copy_buf;
3656 1940040 *out_len = local_vl;
3657 1940040 *out_utf8 = SHM_UNPACK_UTF8(local_vlen_packed);
3658 #else
3659 1711292 *out_value = local_value;
3660 #endif
3661 3651332 ▶ SHM_GET_RETURN(1);
Branch: 10 branches
BranchTrueFalseTotal
SHM_GET_RETURN(1); 0 1939875 1939875
SHM_GET_RETURN(1); 0 48 48
SHM_GET_RETURN(1); 0 24 24
SHM_GET_RETURN(1); 0 28 28
SHM_GET_RETURN(1); 0 39 39
SHM_GET_RETURN(1); 0 62 62
SHM_GET_RETURN(1); 0 64 64
SHM_GET_RETURN(1); 0 1711080 1711080
SHM_GET_RETURN(1); 0 55 55
SHM_GET_RETURN(1); 0 57 57
3662 }
3663  
3664 4523 ▶ if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 132 132
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 6 6
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 5 5
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 6 6
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 5 5
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 5 5
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 7 7
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 4344 4344
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 6 6
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 7 7
3665 4523 ▶ SHM_GET_RETURN(0);
Branch: 10 branches
BranchTrueFalseTotal
SHM_GET_RETURN(0); 0 132 132
SHM_GET_RETURN(0); 0 6 6
SHM_GET_RETURN(0); 0 5 5
SHM_GET_RETURN(0); 0 6 6
SHM_GET_RETURN(0); 0 5 5
SHM_GET_RETURN(0); 0 5 5
SHM_GET_RETURN(0); 0 7 7
SHM_GET_RETURN(0); 0 4344 4344
SHM_GET_RETURN(0); 0 6 6
SHM_GET_RETURN(0); 0 7 7
3666 }
3667 #undef SHM_GET_RETURN
3668 }
3669  
3670 0 static int SHM_FN(exists)(ShmHandle *h,
3671 #ifdef SHM_KEY_IS_INT
3672 SHM_KEY_INT_TYPE key
3673 #else
3674 const char *key_str, uint32_t key_len, bool key_utf8
3675 #endif
3676 ) {
3677 #ifdef SHM_KEY_IS_INT
3678 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 6 6
SHM_SHARD_DISPATCH(h, key); 0 6 6
SHM_SHARD_DISPATCH(h, key); 0 7 7
SHM_SHARD_DISPATCH(h, key); 0 1722 1722
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 2 2
3679 #else
3680 48 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 21 21
SHM_SHARD_DISPATCH(h, key_str, key_len); 1 19 20
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 3 3
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 4 4
3681 #endif
3682 #ifndef SHM_KEY_IS_INT
3683 (void)key_utf8;
3684 #endif
3685 0 ShmHeader *hdr = h->hdr;
3686 0 uint32_t max_mask = h->max_mask;
3687 #ifndef SHM_KEY_IS_INT
3688 48 uint64_t arena_cap = hdr->arena_cap;
3689 48 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3690 #else
3691 0 uint32_t hash = SHM_HASH_KEY(key);
3692 #endif
3693  
3694 0 int locked = 0;
3695 0 for (int pass = 0; ; pass++) {
3696 0 ▶ if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) {
Branch: 20 branches
BranchTrueFalseTotal
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 21 21
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 20 20
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 3 3
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 4 4
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 6 6
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 6 6
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 7 7
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 1722 1722
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 2 2
if (pass == SHM_READ_TRIES_UNLOCKED && !h->readonly) { 0 0 0
3697 0 shm_rwlock_rdlock(h);
3698 0 locked = 1;
3699 }
3700 0 uint32_t seq = shm_seqlock_read_begin(h);
3701  
3702 0 uint32_t mask = (hdr->table_cap - 1) & max_mask;
3703 0 ▶ if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1);
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 7 14 21
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 7 13 20
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 0 3 3
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 0 4 4
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 6 0 6
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 6 0 6
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 6 1 7
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 1046 676 1722
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 0 0 0
if (h->expires_at) __builtin_prefetch(&h->expires_at[hash & mask], 0, 1); 0 2 2
3704 #ifdef SHM_KEY_IS_INT
3705 0 uint32_t idx = SHM_FN(probe_lockfree)(h, mask, hash, key);
3706 #else
3707 48 uint32_t idx = SHM_FN(probe_lockfree)(h, mask, hash, key_str, key_len, arena_cap);
3708 #endif
3709 0 int found = idx != UINT32_MAX;
3710 0 ▶ if (found && h->expires_at) {
Branch: 20 branches
BranchTrueFalseTotal
if (found && h->expires_at) { 15 6 21
if (found && h->expires_at) { 7 8 15
if (found && h->expires_at) { 16 4 20
if (found && h->expires_at) { 7 9 16
if (found && h->expires_at) { 1 2 3
if (found && h->expires_at) { 0 1 1
if (found && h->expires_at) { 2 2 4
if (found && h->expires_at) { 0 2 2
if (found && h->expires_at) { 6 0 6
if (found && h->expires_at) { 6 0 6
if (found && h->expires_at) { 6 0 6
if (found && h->expires_at) { 6 0 6
if (found && h->expires_at) { 7 0 7
if (found && h->expires_at) { 6 1 7
if (found && h->expires_at) { 1312 410 1722
if (found && h->expires_at) { 1044 268 1312
if (found && h->expires_at) { 0 0 0
if (found && h->expires_at) { 0 0 0
if (found && h->expires_at) { 1 1 2
if (found && h->expires_at) { 0 1 1
3711 0 uint32_t exp = h->expires_at[idx];
3712 0 ▶ if (exp != 0 && shm_now() >= exp) found = 0;
Branch: 20 branches
BranchTrueFalseTotal
if (exp != 0 && shm_now() >= exp) found = 0; 7 0 7
if (exp != 0 && shm_now() >= exp) found = 0; 6 1 7
if (exp != 0 && shm_now() >= exp) found = 0; 7 0 7
if (exp != 0 && shm_now() >= exp) found = 0; 0 7 7
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 6 0 6
if (exp != 0 && shm_now() >= exp) found = 0; 1044 0 1044
if (exp != 0 && shm_now() >= exp) found = 0; 31 1013 1044
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
if (exp != 0 && shm_now() >= exp) found = 0; 0 0 0
3713 }
3714  
3715 0 ▶ if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 21 21
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 20 20
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 3 3
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 4 4
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 6 6
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 6 6
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 7 7
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 1722 1722
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 0 0
if (shm_seqlock_read_retry(&hdr->seq, seq)) continue; 0 2 2
3716 0 ▶ if (locked) shm_rwlock_rdunlock(h);
Branch: 10 branches
BranchTrueFalseTotal
if (locked) shm_rwlock_rdunlock(h); 0 21 21
if (locked) shm_rwlock_rdunlock(h); 0 20 20
if (locked) shm_rwlock_rdunlock(h); 0 3 3
if (locked) shm_rwlock_rdunlock(h); 0 4 4
if (locked) shm_rwlock_rdunlock(h); 0 6 6
if (locked) shm_rwlock_rdunlock(h); 0 6 6
if (locked) shm_rwlock_rdunlock(h); 0 7 7
if (locked) shm_rwlock_rdunlock(h); 0 1722 1722
if (locked) shm_rwlock_rdunlock(h); 0 0 0
if (locked) shm_rwlock_rdunlock(h); 0 2 2
3717 0 return found;
3718 }
3719 }
3720  
3721 /* Caller holds wrlock + seqlock on the dispatched shard. */
3722 73863 static int SHM_FN(remove_inner)(ShmHandle *h,
3723 #ifdef SHM_KEY_IS_INT
3724 SHM_KEY_INT_TYPE key
3725 #else
3726 const char *key_str, uint32_t key_len, bool key_utf8
3727 #endif
3728 ) {
3729 73863 ShmHeader *hdr = h->hdr;
3730 73863 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3731 73863 uint8_t *states = h->states;
3732 73863 uint32_t mask = hdr->table_cap - 1;
3733 #ifdef SHM_KEY_IS_INT
3734 61321 uint32_t hash = SHM_HASH_KEY(key);
3735 #else
3736 12542 shm_sig_hold_bytes(h, key_len);
3737 12542 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3738 #endif
3739 73863 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 1 12430 12431
uint32_t now = h->expires_at ? shm_now() : 0; 1 103 104
uint32_t now = h->expires_at ? shm_now() : 0; 1 2 3
uint32_t now = h->expires_at ? shm_now() : 0; 1 3 4
uint32_t now = h->expires_at ? shm_now() : 0; 1 36 37
uint32_t now = h->expires_at ? shm_now() : 0; 1 0 1
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 18 60760 60778
uint32_t now = h->expires_at ? shm_now() : 0; 1 500 501
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
3740 73863 uint32_t pos = hash & mask;
3741 73863 uint8_t tag = SHM_MAKE_TAG(hash);
3742 123216 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 17925 0 17925
for (uint32_t i = 0; i <= mask; i++) { 106 0 106
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 38 0 38
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
for (uint32_t i = 0; i <= mask; i++) { 104540 0 104540
for (uint32_t i = 0; i <= mask; i++) { 592 0 592
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
3743 123216 uint32_t idx = (pos + i) & mask;
3744 123216 uint8_t st = states[idx];
3745 123216 __builtin_prefetch(&nodes[idx], 0, 1);
3746 123216 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
3747 123216 ▶ if (st == SHM_EMPTY) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) return 0; 1 17924 17925
if (st == SHM_EMPTY) return 0; 0 106 106
if (st == SHM_EMPTY) return 0; 0 3 3
if (st == SHM_EMPTY) return 0; 0 4 4
if (st == SHM_EMPTY) return 0; 0 38 38
if (st == SHM_EMPTY) return 0; 0 2 2
if (st == SHM_EMPTY) return 0; 0 3 3
if (st == SHM_EMPTY) return 0; 1249 103291 104540
if (st == SHM_EMPTY) return 0; 0 592 592
if (st == SHM_EMPTY) return 0; 0 3 3
3748 121966 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 5493 12431 17924
if (st != tag) continue; 2 104 106
if (st != tag) continue; 0 3 3
if (st != tag) continue; 0 4 4
if (st != tag) continue; 1 37 38
if (st != tag) continue; 1 1 2
if (st != tag) continue; 1 2 3
if (st != tag) continue; 43738 59553 103291
if (st != tag) continue; 91 501 592
if (st != tag) continue; 1 2 3
3749 #ifdef SHM_KEY_IS_INT
3750 60096 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 37 0 37
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 59529 24 59553
if (SHM_KEY_EQ(&nodes[idx], key)) { 501 0 501
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
3751 #else
3752 12542 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 12430 1 12431
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 104 0 104
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 3 0 3
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 4 0 4
3753 #endif
3754 /* Expired is absent, as every other operation reports it. */
3755 72613 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 12429 12430
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 103 104
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 2 3
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 3 4
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 36 37
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 18 59511 59529
if (SHM_IS_EXPIRED(h, idx, now)) { 0 18 18
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 500 501
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
3756 0 SHM_FN(expire_at)(h, idx);
3757 0 return 0;
3758 }
3759 72613 SHM_FN(remove_at)(h, idx);
3760 72613 return 1;
3761 }
3762 }
3763 0 return 0;
3764 }
3765  
3766 53853 static int SHM_FN(remove)(ShmHandle *h,
3767 #ifdef SHM_KEY_IS_INT
3768 SHM_KEY_INT_TYPE key
3769 #else
3770 const char *key_str, uint32_t key_len, bool key_utf8
3771 #endif
3772 ) {
3773 #ifdef SHM_KEY_IS_INT
3774 41318 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 37 37
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 7900 32875 40775
SHM_SHARD_DISPATCH(h, key); 0 501 501
SHM_SHARD_DISPATCH(h, key); 0 2 2
3775 #else
3776 12535 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 8 12419 12427
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 103 103
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 3 3
3777 #endif
3778 53853 ShmHeader *hdr = h->hdr;
3779 53853 shm_rwlock_wrlock(h);
3780 53853 shm_seqlock_write_begin(&hdr->seq);
3781 53853 int rc = SHM_FN(remove_inner)(h,
3782 #ifdef SHM_KEY_IS_INT
3783 key
3784 #else
3785 key_str, key_len, key_utf8
3786 #endif
3787 );
3788 53853 ▶ if (rc) SHM_FN(maybe_shrink)(h);
Branch: 10 branches
BranchTrueFalseTotal
if (rc) SHM_FN(maybe_shrink)(h); 12427 0 12427
if (rc) SHM_FN(maybe_shrink)(h); 103 0 103
if (rc) SHM_FN(maybe_shrink)(h); 2 0 2
if (rc) SHM_FN(maybe_shrink)(h); 3 0 3
if (rc) SHM_FN(maybe_shrink)(h); 37 0 37
if (rc) SHM_FN(maybe_shrink)(h); 1 0 1
if (rc) SHM_FN(maybe_shrink)(h); 2 0 2
if (rc) SHM_FN(maybe_shrink)(h); 39527 1248 40775
if (rc) SHM_FN(maybe_shrink)(h); 501 0 501
if (rc) SHM_FN(maybe_shrink)(h); 2 0 2
3789 53853 shm_seqlock_write_end(&hdr->seq);
3790 53853 shm_rwlock_wrunlock(h);
3791 53853 return rc;
3792 }
3793  
3794 /* ---- Add: 1 inserted, 0 present or full ---- */
3795  
3796 104 static int SHM_FN(add_impl)(ShmHandle *h,
3797 #ifdef SHM_KEY_IS_INT
3798 SHM_KEY_INT_TYPE key,
3799 #else
3800 const char *key_str, uint32_t key_len, bool key_utf8,
3801 #endif
3802 #ifdef SHM_VAL_IS_STR
3803 const char *val_str, uint32_t val_len, bool val_utf8,
3804 #else
3805 SHM_VAL_INT_TYPE value,
3806 #endif
3807 uint32_t ttl_sec
3808 ) {
3809 #ifdef SHM_KEY_IS_INT
3810 29 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 5 5
SHM_SHARD_DISPATCH(h, key); 0 5 5
SHM_SHARD_DISPATCH(h, key); 0 5 5
SHM_SHARD_DISPATCH(h, key); 0 6 6
SHM_SHARD_DISPATCH(h, key); 0 4 4
SHM_SHARD_DISPATCH(h, key); 0 4 4
3811 #else
3812 75 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 61 61
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 6 6
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 4 4
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 4 4
3813 #endif
3814 104 ShmHeader *hdr = h->hdr;
3815 104 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3816 104 uint8_t *states = h->states;
3817 104 shm_rwlock_wrlock(h);
3818 104 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 5 56 61
uint32_t now = h->expires_at ? shm_now() : 0; 3 3 6
uint32_t now = h->expires_at ? shm_now() : 0; 2 2 4
uint32_t now = h->expires_at ? shm_now() : 0; 2 2 4
uint32_t now = h->expires_at ? shm_now() : 0; 3 2 5
uint32_t now = h->expires_at ? shm_now() : 0; 3 2 5
uint32_t now = h->expires_at ? shm_now() : 0; 3 2 5
uint32_t now = h->expires_at ? shm_now() : 0; 6 0 6
uint32_t now = h->expires_at ? shm_now() : 0; 2 2 4
uint32_t now = h->expires_at ? shm_now() : 0; 2 2 4
3819 104 shm_seqlock_write_begin(&hdr->seq);
3820  
3821 SHM_FN(maybe_grow)(h);
3822 104 uint32_t mask = hdr->table_cap - 1;
3823 #ifdef SHM_KEY_IS_INT
3824 29 uint32_t hash = SHM_HASH_KEY(key);
3825 #else
3826 75 shm_sig_hold_bytes(h, key_len);
3827 75 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3828 #endif
3829 104 uint32_t pos = hash & mask;
3830 104 uint8_t tag = SHM_MAKE_TAG(hash);
3831 104 uint32_t insert_pos = UINT32_MAX;
3832  
3833 274 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 231 0 231
for (uint32_t i = 0; i <= mask; i++) { 6 0 6
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 5 0 5
for (uint32_t i = 0; i <= mask; i++) { 5 0 5
for (uint32_t i = 0; i <= mask; i++) { 5 0 5
for (uint32_t i = 0; i <= mask; i++) { 6 0 6
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
3834 274 uint32_t idx = (pos + i) & mask;
3835 274 uint8_t st = states[idx];
3836 274 __builtin_prefetch(&nodes[idx], 0, 1);
3837 274 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
3838 274 ▶ if (st == SHM_EMPTY) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 59 172 231
if (st == SHM_EMPTY) { 3 3 6
if (st == SHM_EMPTY) { 2 2 4
if (st == SHM_EMPTY) { 2 2 4
if (st == SHM_EMPTY) { 3 2 5
if (st == SHM_EMPTY) { 3 2 5
if (st == SHM_EMPTY) { 3 2 5
if (st == SHM_EMPTY) { 4 2 6
if (st == SHM_EMPTY) { 2 2 4
if (st == SHM_EMPTY) { 2 2 4
3839 83 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 9 50 59
if (insert_pos == UINT32_MAX) insert_pos = idx; 3 0 3
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = idx; 3 0 3
if (insert_pos == UINT32_MAX) insert_pos = idx; 3 0 3
if (insert_pos == UINT32_MAX) insert_pos = idx; 3 0 3
if (insert_pos == UINT32_MAX) insert_pos = idx; 4 0 4
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
3840 83 break;
3841 }
3842 191 ▶ if (st == SHM_TOMBSTONE) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 60 112 172
if (st == SHM_TOMBSTONE) { 0 3 3
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 2 2
3843 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 50 10 60
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
3844 0 continue;
3845 }
3846 131 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 110 2 112
if (st != tag) continue; 0 3 3
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
3847 #ifdef SHM_KEY_IS_INT
3848 12 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
3849 #else
3850 9 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 3 0 3
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
3851 #endif
3852 21 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 2 3
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
3853 0 SHM_FN(expire_at)(h, idx);
3854 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
3855 0 break;
3856 }
3857 20 shm_seqlock_write_end(&hdr->seq);
3858 20 shm_rwlock_wrunlock(h);
3859 20 return 0;
3860 }
3861 }
3862  
3863 84 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) {
Branch: 10 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 59 59
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 3 3
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 2 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 2 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 3 3
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 3 3
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 3 3
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 5 5
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 2 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 2 2
3864 0 shm_seqlock_write_end(&hdr->seq);
3865 0 shm_rwlock_wrunlock(h);
3866 0 return 0;
3867 }
3868  
3869 84 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), SHM_VAL_SLEN(val_len));
3870  
3871 84 int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
3872 84 SHM_FN(arena_reclaim)(h);
3873 #ifdef SHM_KEY_IS_INT
3874 18 nodes[insert_pos].key = key;
3875 #else
3876 66 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 59 59
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 3 3
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 2 2
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 2 2
3877 0 shm_seqlock_write_end(&hdr->seq);
3878 0 shm_rwlock_wrunlock(h);
3879 0 return 0;
3880 }
3881 #endif
3882 #ifdef SHM_VAL_IS_STR
3883 68 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 2 57 59
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 3 3
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 3 3
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 3 3
3884 #ifndef SHM_KEY_IS_INT
3885 2 shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
3886 #endif
3887 0 shm_seqlock_write_end(&hdr->seq);
3888 0 shm_rwlock_wrunlock(h);
3889 0 return 0;
3890 }
3891 #else
3892 16 __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
3893 #endif
3894 82 ▶ if (h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) { 4 53 57
if (h->expires_at) { 2 1 3
if (h->expires_at) { 1 1 2
if (h->expires_at) { 1 1 2
if (h->expires_at) { 2 1 3
if (h->expires_at) { 2 1 3
if (h->expires_at) { 2 1 3
if (h->expires_at) { 5 0 5
if (h->expires_at) { 1 1 2
if (h->expires_at) { 1 1 2
3895 21 ▶ uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 2 2 4
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1 1 2
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 0 1 1
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 0 1 1
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1 1 2
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1 1 2
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 1 1 2
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 0 5 5
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 0 1 1
uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec; 0 1 1
3896 21 ▶ h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0;
Branch: 10 branches
BranchTrueFalseTotal
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 4 0 4
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 2 0 2
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 2 0 2
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 2 0 2
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 2 0 2
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 4 1 5
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
3897 }
3898 82 shm_publish_tag(states, insert_pos, SHM_MAKE_TAG(hash));
3899 /* tombstones-- before size++: see put_inner */
3900 82 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 10 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 50 7 57
if (was_tombstone) hdr->tombstones--; 0 3 3
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 3 3
if (was_tombstone) hdr->tombstones--; 0 3 3
if (was_tombstone) hdr->tombstones--; 0 3 3
if (was_tombstone) hdr->tombstones--; 1 4 5
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 2 2
3901 82 hdr->size++;
3902  
3903 82 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 1 56 57
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 3 3
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 3 3
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 3 3
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 3 3
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 5 5
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
3904  
3905 82 shm_seqlock_write_end(&hdr->seq);
3906 82 shm_rwlock_wrunlock(h);
3907 82 return 1;
3908 }
3909  
3910 0 static inline int SHM_FN(add)(ShmHandle *h,
3911 #ifdef SHM_KEY_IS_INT
3912 SHM_KEY_INT_TYPE key,
3913 #else
3914 const char *key_str, uint32_t key_len, bool key_utf8,
3915 #endif
3916 #ifdef SHM_VAL_IS_STR
3917 const char *val_str, uint32_t val_len, bool val_utf8
3918 #else
3919 SHM_VAL_INT_TYPE value
3920 #endif
3921 ) {
3922 0 return SHM_FN(add_impl)(h,
3923 #ifdef SHM_KEY_IS_INT
3924 key,
3925 #else
3926 key_str, key_len, key_utf8,
3927 #endif
3928 #ifdef SHM_VAL_IS_STR
3929 val_str, val_len, val_utf8,
3930 #else
3931 value,
3932 #endif
3933 SHM_TTL_USE_DEFAULT);
3934 }
3935  
3936 25 static inline int SHM_FN(add_ttl)(ShmHandle *h,
3937 #ifdef SHM_KEY_IS_INT
3938 SHM_KEY_INT_TYPE key,
3939 #else
3940 const char *key_str, uint32_t key_len, bool key_utf8,
3941 #endif
3942 #ifdef SHM_VAL_IS_STR
3943 const char *val_str, uint32_t val_len, bool val_utf8,
3944 #else
3945 SHM_VAL_INT_TYPE value,
3946 #endif
3947 uint32_t ttl_sec
3948 ) {
3949 25 ▶ if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
Branch: 10 branches
BranchTrueFalseTotal
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 3 3
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 1 5 6
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 2 2
3950 25 return SHM_FN(add_impl)(h,
3951 #ifdef SHM_KEY_IS_INT
3952 key,
3953 #else
3954 key_str, key_len, key_utf8,
3955 #endif
3956 #ifdef SHM_VAL_IS_STR
3957 val_str, val_len, val_utf8,
3958 #else
3959 value,
3960 #endif
3961 ttl_sec);
3962 }
3963  
3964 /* ---- Update: 1 updated, 0 absent or full ---- */
3965  
3966 26 static int SHM_FN(update_impl)(ShmHandle *h,
3967 #ifdef SHM_KEY_IS_INT
3968 SHM_KEY_INT_TYPE key,
3969 #else
3970 const char *key_str, uint32_t key_len, bool key_utf8,
3971 #endif
3972 #ifdef SHM_VAL_IS_STR
3973 const char *val_str, uint32_t val_len, bool val_utf8,
3974 #else
3975 SHM_VAL_INT_TYPE value,
3976 #endif
3977 uint32_t ttl_sec
3978 ) {
3979 #ifdef SHM_KEY_IS_INT
3980 13 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 4 4
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 2 2
3981 #else
3982 13 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 9 9
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
3983 #endif
3984 26 ShmHeader *hdr = h->hdr;
3985 26 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
3986 26 uint8_t *states = h->states;
3987 26 shm_rwlock_wrlock(h);
3988 26 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 5 4 9
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 4 0 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
3989 26 shm_seqlock_write_begin(&hdr->seq);
3990  
3991 26 uint32_t mask = hdr->table_cap - 1;
3992 #ifdef SHM_KEY_IS_INT
3993 13 uint32_t hash = SHM_HASH_KEY(key);
3994 #else
3995 13 shm_sig_hold_bytes(h, key_len);
3996 13 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
3997 #endif
3998 26 uint32_t pos = hash & mask;
3999 26 uint8_t tag = SHM_MAKE_TAG(hash);
4000  
4001 26 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 9 0 9
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
4002 26 uint32_t idx = (pos + i) & mask;
4003 26 uint8_t st = states[idx];
4004 26 __builtin_prefetch(&nodes[idx], 0, 1);
4005 26 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
4006 26 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 2 7 9
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 1 3 4
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 1 1 2
4007 22 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 7 7
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 3 3
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
4008 #ifdef SHM_KEY_IS_INT
4009 11 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 3 0 3
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
4010 #else
4011 11 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 7 0 7
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
4012 #endif
4013 22 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 4 3 7
if (SHM_IS_EXPIRED(h, idx, now)) { 4 0 4
if (SHM_IS_EXPIRED(h, idx, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, idx, now)) { 0 2 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 3 0 3
if (SHM_IS_EXPIRED(h, idx, now)) { 3 0 3
if (SHM_IS_EXPIRED(h, idx, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
4014 0 SHM_FN(expire_at)(h, idx);
4015 0 SHM_FN(maybe_shrink)(h);
4016 0 shm_seqlock_write_end(&hdr->seq);
4017 0 shm_rwlock_wrunlock(h);
4018 0 return 0;
4019 }
4020 #ifdef SHM_VAL_IS_STR
4021 {
4022 13 SHM_FN(arena_reclaim)(h);
4023 13 uint32_t old_off = nodes[idx].val_off;
4024 13 uint32_t old_lf = nodes[idx].val_len;
4025 13 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 1 6 7
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 2 2
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 2 2
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 2 2
4026 0 shm_seqlock_write_end(&hdr->seq);
4027 0 shm_rwlock_wrunlock(h);
4028 0 return 0;
4029 }
4030 12 shm_str_free(hdr, h->arena, old_off, old_lf);
4031 }
4032 #else
4033 9 __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
4034 #endif
4035 21 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 1 5 6
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 3 3
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
4036 21 ▶ if (h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) { 4 2 6
if (h->expires_at) { 0 2 2
if (h->expires_at) { 0 1 1
if (h->expires_at) { 0 1 1
if (h->expires_at) { 1 1 2
if (h->expires_at) { 1 1 2
if (h->expires_at) { 1 1 2
if (h->expires_at) { 3 0 3
if (h->expires_at) { 0 1 1
if (h->expires_at) { 0 1 1
4037 0 ▶ if (ttl_sec == SHM_TTL_USE_DEFAULT) {
Branch: 10 branches
BranchTrueFalseTotal
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 2 2 4
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 0 0
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 0 0
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 0 0
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 1 0 1
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 1 0 1
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 1 0 1
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 3 3
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 0 0
if (ttl_sec == SHM_TTL_USE_DEFAULT) { 0 0 0
4038 0 ▶ if (hdr->default_ttl > 0 && h->expires_at[idx] != 0)
Branch: 20 branches
BranchTrueFalseTotal
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 2 0 2
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 2 0 2
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
4039 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
4040 } else {
4041 0 ▶ h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0;
Branch: 10 branches
BranchTrueFalseTotal
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 2 0 2
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 2 1 3
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0; 0 0 0
4042 }
4043 }
4044  
4045 21 shm_seqlock_write_end(&hdr->seq);
4046 21 shm_rwlock_wrunlock(h);
4047 21 return 1;
4048 }
4049 }
4050  
4051 0 shm_seqlock_write_end(&hdr->seq);
4052 0 shm_rwlock_wrunlock(h);
4053 0 return 0;
4054 }
4055  
4056 0 static inline int SHM_FN(update)(ShmHandle *h,
4057 #ifdef SHM_KEY_IS_INT
4058 SHM_KEY_INT_TYPE key,
4059 #else
4060 const char *key_str, uint32_t key_len, bool key_utf8,
4061 #endif
4062 #ifdef SHM_VAL_IS_STR
4063 const char *val_str, uint32_t val_len, bool val_utf8
4064 #else
4065 SHM_VAL_INT_TYPE value
4066 #endif
4067 ) {
4068 0 return SHM_FN(update_impl)(h,
4069 #ifdef SHM_KEY_IS_INT
4070 key,
4071 #else
4072 key_str, key_len, key_utf8,
4073 #endif
4074 #ifdef SHM_VAL_IS_STR
4075 val_str, val_len, val_utf8,
4076 #else
4077 value,
4078 #endif
4079 SHM_TTL_USE_DEFAULT);
4080 }
4081  
4082 0 static inline int SHM_FN(update_ttl)(ShmHandle *h,
4083 #ifdef SHM_KEY_IS_INT
4084 SHM_KEY_INT_TYPE key,
4085 #else
4086 const char *key_str, uint32_t key_len, bool key_utf8,
4087 #endif
4088 #ifdef SHM_VAL_IS_STR
4089 const char *val_str, uint32_t val_len, bool val_utf8,
4090 #else
4091 SHM_VAL_INT_TYPE value,
4092 #endif
4093 uint32_t ttl_sec
4094 ) {
4095 0 ▶ if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
Branch: 10 branches
BranchTrueFalseTotal
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 3 3
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 1 3 4
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2; 0 0 0
4096 0 return SHM_FN(update_impl)(h,
4097 #ifdef SHM_KEY_IS_INT
4098 key,
4099 #else
4100 key_str, key_len, key_utf8,
4101 #endif
4102 #ifdef SHM_VAL_IS_STR
4103 val_str, val_len, val_utf8,
4104 #else
4105 value,
4106 #endif
4107 ttl_sec);
4108 }
4109  
4110 /* ---- Swap: 1 replaced (old value out), 2 inserted, 0 full ---- */
4111  
4112 74 static int SHM_FN(swap)(ShmHandle *h,
4113 #ifdef SHM_KEY_IS_INT
4114 SHM_KEY_INT_TYPE key,
4115 #else
4116 const char *key_str, uint32_t key_len, bool key_utf8,
4117 #endif
4118 #ifdef SHM_VAL_IS_STR
4119 const char *val_str, uint32_t val_len, bool val_utf8,
4120 const char **out_str, uint32_t *out_len, bool *out_utf8
4121 #else
4122 SHM_VAL_INT_TYPE value,
4123 SHM_VAL_INT_TYPE *out_value
4124 #endif
4125 ) {
4126 #ifdef SHM_KEY_IS_INT
4127 12 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 3 3
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 2 2
4128 #else
4129 62 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 57 57
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 3 3
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
4130 #endif
4131 74 ShmHeader *hdr = h->hdr;
4132 74 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4133 74 uint8_t *states = h->states;
4134 74 shm_rwlock_wrlock(h);
4135 74 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 2 55 57
uint32_t now = h->expires_at ? shm_now() : 0; 1 2 3
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 1 2 3
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 2 0 2
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
4136 74 shm_seqlock_write_begin(&hdr->seq);
4137  
4138 SHM_FN(maybe_grow)(h);
4139 74 uint32_t mask = hdr->table_cap - 1;
4140 #ifdef SHM_KEY_IS_INT
4141 12 uint32_t hash = SHM_HASH_KEY(key);
4142 #else
4143 62 shm_sig_hold_bytes(h, key_len);
4144 62 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
4145 #endif
4146 74 uint32_t pos = hash & mask;
4147 74 uint8_t tag = SHM_MAKE_TAG(hash);
4148 74 uint32_t insert_pos = UINT32_MAX;
4149  
4150 244 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 227 0 227
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
4151 244 uint32_t idx = (pos + i) & mask;
4152 244 uint8_t st = states[idx];
4153 244 __builtin_prefetch(&nodes[idx], 0, 1);
4154 244 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
4155 244 ▶ if (st == SHM_EMPTY) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 54 173 227
if (st == SHM_EMPTY) { 1 2 3
if (st == SHM_EMPTY) { 0 1 1
if (st == SHM_EMPTY) { 0 1 1
if (st == SHM_EMPTY) { 2 1 3
if (st == SHM_EMPTY) { 1 1 2
if (st == SHM_EMPTY) { 1 1 2
if (st == SHM_EMPTY) { 0 2 2
if (st == SHM_EMPTY) { 0 1 1
if (st == SHM_EMPTY) { 1 1 2
4156 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 4 50 54
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
4157 0 break;
4158 }
4159 184 ▶ if (st == SHM_TOMBSTONE) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 60 113 173
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 2 2
if (st == SHM_TOMBSTONE) { 0 1 1
if (st == SHM_TOMBSTONE) { 0 1 1
4160 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 50 10 60
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
4161 0 continue;
4162 }
4163 124 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 110 3 113
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
4164 #ifdef SHM_KEY_IS_INT
4165 7 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
4166 #else
4167 7 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 3 0 3
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
4168 #endif
4169 14 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 2 3
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 2 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
4170 0 SHM_FN(expire_at)(h, idx);
4171 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
4172 0 break;
4173 }
4174 #ifdef SHM_VAL_IS_STR
4175 {
4176 6 ▶ uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len); 2 1 3
uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len); 1 0 1
uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len); 1 0 1
uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len); 1 0 1
4177 6 shm_sig_hold_bytes(h, old_vl);
4178 6 ▶ if (!shm_ensure_copy_buf(h, old_vl)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, old_vl)) { 0 3 3
if (!shm_ensure_copy_buf(h, old_vl)) { 0 1 1
if (!shm_ensure_copy_buf(h, old_vl)) { 0 1 1
if (!shm_ensure_copy_buf(h, old_vl)) { 0 1 1
4179 0 shm_seqlock_write_end(&hdr->seq);
4180 0 shm_rwlock_wrunlock(h);
4181 0 return 0;
4182 }
4183 6 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, old_vl);
4184 6 *out_str = h->copy_buf;
4185 6 *out_len = old_vl;
4186 6 *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4187  
4188 {
4189 6 SHM_FN(arena_reclaim)(h); /* the old value is already in copy_buf */
4190 6 uint32_t old_off = nodes[idx].val_off;
4191 6 uint32_t old_lf = nodes[idx].val_len;
4192 6 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 3 3
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) { 0 1 1
4193 0 shm_seqlock_write_end(&hdr->seq);
4194 0 shm_rwlock_wrunlock(h);
4195 0 return 0;
4196 }
4197 6 shm_str_free(hdr, h->arena, old_off, old_lf);
4198 }
4199 }
4200 #else
4201 8 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4202 8 __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
4203 #endif
4204 14 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 1 2 3
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 2 2
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
4205 14 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
Branch: 30 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 2 3
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 2 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 2 0 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 2 0 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 1 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
4206 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
4207  
4208 14 shm_seqlock_write_end(&hdr->seq);
4209 14 shm_rwlock_wrunlock(h);
4210 14 return 1;
4211 }
4212 }
4213  
4214 0 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) {
Branch: 10 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 54 54
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 1 1
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 0 0
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 0 0
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 2 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 1 1
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 1 1
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 0 0
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 0 0
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 0 1 1
4215 0 shm_seqlock_write_end(&hdr->seq);
4216 0 shm_rwlock_wrunlock(h);
4217 0 return 0;
4218 }
4219  
4220 0 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), SHM_VAL_SLEN(val_len));
4221  
4222 0 int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
4223 0 SHM_FN(arena_reclaim)(h);
4224 #ifdef SHM_KEY_IS_INT
4225 0 nodes[insert_pos].key = key;
4226 #else
4227 0 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 54 54
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 0 0
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 0 0
4228 0 shm_seqlock_write_end(&hdr->seq);
4229 0 shm_rwlock_wrunlock(h);
4230 0 return 0;
4231 }
4232 #endif
4233 #ifdef SHM_VAL_IS_STR
4234 58 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 2 52 54
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 2 2
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) { 0 1 1
4235 #ifndef SHM_KEY_IS_INT
4236 2 shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
4237 #endif
4238 0 shm_seqlock_write_end(&hdr->seq);
4239 0 shm_rwlock_wrunlock(h);
4240 0 return 0;
4241 }
4242 #else
4243 0 __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
4244 #endif
4245 0 ▶ if (h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) { 1 51 52
if (h->expires_at) { 1 0 1
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 1 1 2
if (h->expires_at) { 1 0 1
if (h->expires_at) { 1 0 1
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 0 0
if (h->expires_at) { 0 1 1
4246 0 uint32_t ttl = hdr->default_ttl;
4247 0 ▶ h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0;
Branch: 10 branches
BranchTrueFalseTotal
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 0 0 0
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 0 0 0
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 1 0 1
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 0 0 0
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 0 0 0
h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0; 0 0 0
4248 }
4249 0 shm_publish_tag(states, insert_pos, SHM_MAKE_TAG(hash));
4250 /* tombstones-- before size++: see put_inner */
4251 0 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 10 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 50 2 52
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 0 0
if (was_tombstone) hdr->tombstones--; 0 0 0
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 0 0
if (was_tombstone) hdr->tombstones--; 0 0 0
if (was_tombstone) hdr->tombstones--; 0 1 1
4252 0 hdr->size++;
4253  
4254 0 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 1 51 52
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 0 0
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 0 0
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 0 0
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 0 0
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
4255  
4256 0 shm_seqlock_write_end(&hdr->seq);
4257 0 shm_rwlock_wrunlock(h);
4258 0 return 2;
4259 }
4260  
4261 0 static int SHM_FN(take)(ShmHandle *h,
4262 #ifdef SHM_KEY_IS_INT
4263 SHM_KEY_INT_TYPE key,
4264 #else
4265 const char *key_str, uint32_t key_len, bool key_utf8,
4266 #endif
4267 #ifdef SHM_VAL_IS_STR
4268 const char **out_str, uint32_t *out_len, bool *out_utf8
4269 #else
4270 SHM_VAL_INT_TYPE *out_value
4271 #endif
4272 ) {
4273 #ifdef SHM_KEY_IS_INT
4274 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
4275 #else
4276 0 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
4277 #endif
4278 0 ShmHeader *hdr = h->hdr;
4279 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4280 0 uint8_t *states = h->states;
4281 0 shm_rwlock_wrlock(h);
4282 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
4283 0 shm_seqlock_write_begin(&hdr->seq);
4284  
4285 0 uint32_t mask = hdr->table_cap - 1;
4286 #ifdef SHM_KEY_IS_INT
4287 0 uint32_t hash = SHM_HASH_KEY(key);
4288 #else
4289 0 shm_sig_hold_bytes(h, key_len);
4290 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
4291 #endif
4292 0 uint32_t pos = hash & mask;
4293 0 uint8_t tag = SHM_MAKE_TAG(hash);
4294  
4295 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
4296 0 uint32_t idx = (pos + i) & mask;
4297 0 uint8_t st = states[idx];
4298 0 __builtin_prefetch(&nodes[idx], 0, 1);
4299 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
4300 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
4301 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
4302  
4303 #ifdef SHM_KEY_IS_INT
4304 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
4305 #else
4306 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
4307 #endif
4308 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
4309 0 SHM_FN(expire_at)(h, idx);
4310 0 SHM_FN(maybe_shrink)(h);
4311 0 shm_seqlock_write_end(&hdr->seq);
4312 0 shm_rwlock_wrunlock(h);
4313 0 return 0;
4314 }
4315  
4316 #ifdef SHM_VAL_IS_STR
4317 {
4318 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
4319 0 shm_sig_hold_bytes(h, vl);
4320 0 ▶ if (!shm_ensure_copy_buf(h, vl)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
4321 0 shm_seqlock_write_end(&hdr->seq);
4322 0 shm_rwlock_wrunlock(h);
4323 0 return 0;
4324 }
4325 0 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4326 0 *out_str = h->copy_buf;
4327 0 *out_len = vl;
4328 0 *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4329 }
4330 #else
4331 0 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4332 #endif
4333 0 SHM_FN(remove_at)(h, idx);
4334  
4335 0 SHM_FN(maybe_shrink)(h);
4336 0 shm_seqlock_write_end(&hdr->seq);
4337 0 shm_rwlock_wrunlock(h);
4338 0 return 1;
4339 }
4340 }
4341  
4342 0 shm_seqlock_write_end(&hdr->seq);
4343 0 shm_rwlock_wrunlock(h);
4344 0 return 0;
4345 }
4346  
4347 /* ---- Compare-and-take: 1 if the value matched and the entry was removed ----
4348 * String values compare by bytes only. */
4349 0 static int SHM_FN(cas_take)(ShmHandle *h,
4350 #ifdef SHM_KEY_IS_INT
4351 SHM_KEY_INT_TYPE key,
4352 #else
4353 const char *key_str, uint32_t key_len, bool key_utf8,
4354 #endif
4355 #ifdef SHM_VAL_IS_STR
4356 const char *expected_str, uint32_t expected_len,
4357 const char **out_str, uint32_t *out_len, bool *out_utf8
4358 #else
4359 SHM_VAL_INT_TYPE expected, SHM_VAL_INT_TYPE *out_value
4360 #endif
4361 ) {
4362 #ifdef SHM_KEY_IS_INT
4363 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 4 4
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
4364 #else
4365 0 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 4 4
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
4366 #endif
4367 0 ShmHeader *hdr = h->hdr;
4368 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4369 0 uint8_t *states = h->states;
4370 0 shm_rwlock_wrlock(h);
4371 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
4372 0 shm_seqlock_write_begin(&hdr->seq);
4373  
4374 0 uint32_t mask = hdr->table_cap - 1;
4375 #ifdef SHM_KEY_IS_INT
4376 0 uint32_t hash = SHM_HASH_KEY(key);
4377 #else
4378 0 shm_sig_hold_bytes(h, key_len);
4379 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
4380 #endif
4381 0 uint32_t pos = hash & mask;
4382 0 uint8_t tag = SHM_MAKE_TAG(hash);
4383  
4384 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
4385 0 uint32_t idx = (pos + i) & mask;
4386 0 uint8_t st = states[idx];
4387 0 __builtin_prefetch(&nodes[idx], 0, 1);
4388 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
4389 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 4 4
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 1 3 4
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
4390 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 4 4
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 3 3
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
4391 #ifdef SHM_KEY_IS_INT
4392 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 3 0 3
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
4393 #else
4394 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 4 0 4
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
4395 #endif
4396 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
4397 0 SHM_FN(expire_at)(h, idx);
4398 0 SHM_FN(maybe_shrink)(h);
4399 0 shm_seqlock_write_end(&hdr->seq);
4400 0 shm_rwlock_wrunlock(h);
4401 0 return 0;
4402 }
4403 #ifdef SHM_VAL_IS_STR
4404 char ibuf[SHM_INLINE_MAX];
4405 uint32_t cur_len;
4406 0 const char *cur_str = shm_str_ptr(nodes[idx].val_off, nodes[idx].val_len,
4407 0 h->arena, h->hdr->arena_cap, ibuf, &cur_len);
4408 0 ▶ if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len);
Branch: 4 branches
BranchTrueFalseTotal
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 3 1 4
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 0 0 0
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 0 0 0
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 0 0 0
4409 0 ▶ if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) {
Branch: 8 branches
BranchTrueFalseTotal
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 3 1 4
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 3 3
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 0 0
4410 0 shm_seqlock_write_end(&hdr->seq);
4411 0 shm_rwlock_wrunlock(h);
4412 0 return 0;
4413 }
4414 0 ▶ if (!shm_ensure_copy_buf(h, cur_len)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, cur_len)) { 0 3 3
if (!shm_ensure_copy_buf(h, cur_len)) { 0 0 0
if (!shm_ensure_copy_buf(h, cur_len)) { 0 0 0
if (!shm_ensure_copy_buf(h, cur_len)) { 0 0 0
4415 0 shm_seqlock_write_end(&hdr->seq);
4416 0 shm_rwlock_wrunlock(h);
4417 0 return 0;
4418 }
4419 0 memcpy(h->copy_buf, cur_str, cur_len);
4420 0 *out_str = h->copy_buf;
4421 0 *out_len = cur_len;
4422 0 *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4423 #else
4424 0 ▶ if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) {
Branch: 6 branches
BranchTrueFalseTotal
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 1 2 3
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) { 0 0 0
4425 0 shm_seqlock_write_end(&hdr->seq);
4426 0 shm_rwlock_wrunlock(h);
4427 0 return 0;
4428 }
4429 0 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4430 #endif
4431 0 SHM_FN(remove_at)(h, idx);
4432 0 SHM_FN(maybe_shrink)(h);
4433 0 shm_seqlock_write_end(&hdr->seq);
4434 0 shm_rwlock_wrunlock(h);
4435 0 return 1;
4436 }
4437 }
4438  
4439 0 shm_seqlock_write_end(&hdr->seq);
4440 0 shm_rwlock_wrunlock(h);
4441 0 return 0;
4442 }
4443  
4444 /* ---- Pop: the LRU tail, else the next live slot from pop_cursor ---- */
4445  
4446 195 static int SHM_FN(pop)(ShmHandle *h,
4447 #ifdef SHM_KEY_IS_INT
4448 SHM_KEY_INT_TYPE *out_key,
4449 #else
4450 const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
4451 #endif
4452 #ifdef SHM_VAL_IS_STR
4453 const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
4454 #else
4455 SHM_VAL_INT_TYPE *out_value
4456 #endif
4457 ) {
4458 195 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 4 11 15
if (h->shard_handles) { 1 8 9
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 122 122
if (h->shard_handles) { 0 7 7
if (h->shard_handles) { 0 7 7
4459 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) { 4 0 4
for (uint32_t i = 0; i < h->num_shards; i++) { 1 0 1
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
4460 0 uint32_t si = (h->shard_rr + i) % h->num_shards;
4461 0 int rc = SHM_FN(pop)(h->shard_handles[si],
4462 #ifdef SHM_KEY_IS_INT
4463 out_key,
4464 #else
4465 out_key_str, out_key_len, out_key_utf8,
4466 #endif
4467 #ifdef SHM_VAL_IS_STR
4468 out_val_str, out_val_len, out_val_utf8
4469 #else
4470 out_value
4471 #endif
4472 );
4473 0 ▶ if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; }
Branch: 10 branches
BranchTrueFalseTotal
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 4 0 4
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 1 0 1
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
4474 }
4475 0 return 0;
4476 }
4477 190 ShmHeader *hdr = h->hdr;
4478 190 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4479 190 uint8_t *states = h->states;
4480 190 shm_rwlock_wrlock(h);
4481 190 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 11 11
uint32_t now = h->expires_at ? shm_now() : 0; 0 8 8
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 1 121 122
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
4482 190 shm_seqlock_write_begin(&hdr->seq);
4483  
4484 190 uint32_t idx = UINT32_MAX, size0 = hdr->size;
4485 191 ▶ if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) {
Branch: 20 branches
BranchTrueFalseTotal
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 11 11
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 8 8
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 1 121 122
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 1 0 1
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
4486 0 uint32_t pos = hdr->lru_tail;
4487 0 ▶ while (pos != SHM_LRU_NONE) {
Branch: 10 branches
BranchTrueFalseTotal
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 1 0 1
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
4488 0 ▶ if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
Branch: 30 branches
BranchTrueFalseTotal
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 1 1
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
4489 0 ▶ uint32_t prev = h->lru_prev[pos];
Branch: 10 branches
BranchTrueFalseTotal
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
4490 shm_sig_check(h);
4491 0 SHM_FN(expire_at)(h, pos);
4492 0 pos = prev;
4493 }
4494 } else {
4495 189 uint32_t cap = hdr->table_cap;
4496 189 ▶ uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 11 0 11
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 8 0 8
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 121 0 121
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 7 0 7
4497 9263 ▶ for (uint32_t n = 0; n < cap; n++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t n = 0; n < cap; n++) { 81 3 84
for (uint32_t n = 0; n < cap; n++) { 74 3 77
for (uint32_t n = 0; n < cap; n++) { 73 3 76
for (uint32_t n = 0; n < cap; n++) { 73 3 76
for (uint32_t n = 0; n < cap; n++) { 79 3 82
for (uint32_t n = 0; n < cap; n++) { 79 3 82
for (uint32_t n = 0; n < cap; n++) { 79 3 82
for (uint32_t n = 0; n < cap; n++) { 8535 5 8540
for (uint32_t n = 0; n < cap; n++) { 79 3 82
for (uint32_t n = 0; n < cap; n++) { 79 3 82
4498 9231 uint32_t i = (start + n) % cap;
4499 9231 shm_sig_check_every(h, n + 1, SHM_SIG_CHECK_SLOTS);
4500 9231 ▶ if (!SHM_IS_LIVE(states[i])) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (!SHM_IS_LIVE(states[i])) continue; 73 8 81
if (!SHM_IS_LIVE(states[i])) continue; 69 5 74
if (!SHM_IS_LIVE(states[i])) continue; 69 4 73
if (!SHM_IS_LIVE(states[i])) continue; 69 4 73
if (!SHM_IS_LIVE(states[i])) continue; 75 4 79
if (!SHM_IS_LIVE(states[i])) continue; 75 4 79
if (!SHM_IS_LIVE(states[i])) continue; 75 4 79
if (!SHM_IS_LIVE(states[i])) continue; 3419 5116 8535
if (!SHM_IS_LIVE(states[i])) continue; 75 4 79
if (!SHM_IS_LIVE(states[i])) continue; 75 4 79
4501 5157 ▶ if (SHM_IS_EXPIRED(h, i, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, i, now)) { 0 8 8
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 5 5
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 5000 116 5116
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
4502 shm_sig_check(h);
4503 0 SHM_FN(expire_at)(h, i);
4504 0 continue;
4505 }
4506 157 idx = i;
4507 157 hdr->pop_cursor = (i + 1) % cap;
4508 157 break;
4509 }
4510 }
4511  
4512 190 ▶ if (idx == UINT32_MAX) {
Branch: 10 branches
BranchTrueFalseTotal
if (idx == UINT32_MAX) { 3 8 11
if (idx == UINT32_MAX) { 3 5 8
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 5 117 122
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 4 7
4513 32 ▶ if (hdr->size != size0) SHM_FN(maybe_shrink)(h);
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 1 4 5
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
4514 32 shm_seqlock_write_end(&hdr->seq);
4515 32 shm_rwlock_wrunlock(h);
4516 32 return 0;
4517 }
4518  
4519 #ifdef SHM_KEY_IS_INT
4520 137 *out_key = nodes[idx].key;
4521 #else
4522 {
4523 21 ▶ uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 4 4 8
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 4 1 5
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 4 0 4
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 4 0 4
4524 21 shm_sig_hold_bytes(h, kl);
4525 21 ▶ if (!shm_ensure_copy_buf(h, kl + 1)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 8 8
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 5 5
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 4 4
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 4 4
4526 0 shm_seqlock_write_end(&hdr->seq);
4527 0 shm_rwlock_wrunlock(h);
4528 0 return 0;
4529 }
4530 21 shm_str_copy(h->copy_buf, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
4531 21 *out_key_str = h->copy_buf;
4532 21 *out_key_len = kl;
4533 21 *out_key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
4534 }
4535 #endif
4536 #ifdef SHM_VAL_IS_STR
4537 {
4538 20 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 4 4 8
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 4 0 4
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 4 0 4
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 4 0 4
4539 20 shm_sig_hold_bytes(h, vl);
4540 #ifndef SHM_KEY_IS_INT
4541 8 ▶ uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 4 4 8
4542 /* Value after the key; realloc may move the key too. */
4543 8 ▶ if (!shm_ensure_copy_buf(h, kl + vl + 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, kl + vl + 1)) { 0 8 8
4544 0 shm_seqlock_write_end(&hdr->seq);
4545 0 shm_rwlock_wrunlock(h);
4546 0 return 0;
4547 }
4548 8 *out_key_str = h->copy_buf;
4549 8 shm_str_copy(h->copy_buf + kl, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4550 8 *out_val_str = h->copy_buf + kl;
4551 #else
4552 12 ▶ if (!shm_ensure_copy_buf(h, vl + 1)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 4 4
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 4 4
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 4 4
4553 0 shm_seqlock_write_end(&hdr->seq);
4554 0 shm_rwlock_wrunlock(h);
4555 0 return 0;
4556 }
4557 12 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4558 12 *out_val_str = h->copy_buf;
4559 #endif
4560 20 *out_val_len = vl;
4561 20 *out_val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4562 }
4563 #else
4564 138 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4565 #endif
4566  
4567 158 SHM_FN(remove_at)(h, idx);
4568 158 SHM_FN(maybe_shrink)(h);
4569  
4570 158 shm_seqlock_write_end(&hdr->seq);
4571 158 shm_rwlock_wrunlock(h);
4572 158 return 1;
4573 }
4574  
4575 /* ---- Shift: the LRU head, else the next live slot below shift_cursor ---- */
4576  
4577 490 static int SHM_FN(shift)(ShmHandle *h,
4578 #ifdef SHM_KEY_IS_INT
4579 SHM_KEY_INT_TYPE *out_key,
4580 #else
4581 const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
4582 #endif
4583 #ifdef SHM_VAL_IS_STR
4584 const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
4585 #else
4586 SHM_VAL_INT_TYPE *out_value
4587 #endif
4588 ) {
4589 490 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 4 10 14
if (h->shard_handles) { 1 7 8
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 426 426
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
4590 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) { 4 0 4
for (uint32_t i = 0; i < h->num_shards; i++) { 1 0 1
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
4591 0 uint32_t si = (h->shard_rr + i) % h->num_shards;
4592 0 int rc = SHM_FN(shift)(h->shard_handles[si],
4593 #ifdef SHM_KEY_IS_INT
4594 out_key,
4595 #else
4596 out_key_str, out_key_len, out_key_utf8,
4597 #endif
4598 #ifdef SHM_VAL_IS_STR
4599 out_val_str, out_val_len, out_val_utf8
4600 #else
4601 out_value
4602 #endif
4603 );
4604 0 ▶ if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; }
Branch: 10 branches
BranchTrueFalseTotal
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 4 0 4
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 1 0 1
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; } 0 0 0
4605 }
4606 0 return 0;
4607 }
4608 485 ShmHeader *hdr = h->hdr;
4609 485 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4610 485 uint8_t *states = h->states;
4611 485 shm_rwlock_wrlock(h);
4612 485 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 10 10
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 1 425 426
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
4613 485 shm_seqlock_write_begin(&hdr->seq);
4614  
4615 485 uint32_t idx = UINT32_MAX, size0 = hdr->size;
4616 485 ▶ if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) {
Branch: 20 branches
BranchTrueFalseTotal
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 10 10
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 7 7
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 426 426
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 6 6
if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) { 0 0 0
4617 0 uint32_t pos = hdr->lru_head;
4618 0 ▶ while (pos != SHM_LRU_NONE) {
Branch: 10 branches
BranchTrueFalseTotal
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
4619 0 ▶ if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
Branch: 30 branches
BranchTrueFalseTotal
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
4620 0 ▶ uint32_t nxt = h->lru_next[pos];
Branch: 10 branches
BranchTrueFalseTotal
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
uint32_t nxt = h->lru_next[pos]; 0 0 0
4621 shm_sig_check(h);
4622 0 SHM_FN(expire_at)(h, pos);
4623 0 pos = nxt;
4624 }
4625 } else {
4626 485 uint32_t cap = hdr->table_cap;
4627 485 ▶ uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor;
Branch: 20 branches
BranchTrueFalseTotal
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 8 10
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 0 2
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 5 7
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 0 2
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 4 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 0 2
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 4 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 2 0 2
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 3 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 0 3
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 3 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 0 3
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 3 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 0 3
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 417 9 426
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 417 0 417
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 3 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 0 3
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 3 6
uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor; 3 0 3
4628 9723 ▶ for (uint32_t n = 0; n < cap; n++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t n = 0; n < cap; n++) { 73 3 76
for (uint32_t n = 0; n < cap; n++) { 65 3 68
for (uint32_t n = 0; n < cap; n++) { 64 3 67
for (uint32_t n = 0; n < cap; n++) { 64 3 67
for (uint32_t n = 0; n < cap; n++) { 61 3 64
for (uint32_t n = 0; n < cap; n++) { 61 3 64
for (uint32_t n = 0; n < cap; n++) { 61 3 64
for (uint32_t n = 0; n < cap; n++) { 9120 5 9125
for (uint32_t n = 0; n < cap; n++) { 61 3 64
for (uint32_t n = 0; n < cap; n++) { 61 3 64
4629 9691 uint32_t i = (start + cap - 1 - n) % cap;
4630 9691 shm_sig_check_every(h, n + 1, SHM_SIG_CHECK_SLOTS);
4631 9691 ▶ if (!SHM_IS_LIVE(states[i])) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (!SHM_IS_LIVE(states[i])) continue; 66 7 73
if (!SHM_IS_LIVE(states[i])) continue; 61 4 65
if (!SHM_IS_LIVE(states[i])) continue; 61 3 64
if (!SHM_IS_LIVE(states[i])) continue; 61 3 64
if (!SHM_IS_LIVE(states[i])) continue; 58 3 61
if (!SHM_IS_LIVE(states[i])) continue; 58 3 61
if (!SHM_IS_LIVE(states[i])) continue; 58 3 61
if (!SHM_IS_LIVE(states[i])) continue; 3699 5421 9120
if (!SHM_IS_LIVE(states[i])) continue; 58 3 61
if (!SHM_IS_LIVE(states[i])) continue; 58 3 61
4632 5453 ▶ if (SHM_IS_EXPIRED(h, i, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, i, now)) { 0 7 7
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 4 4
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 5000 421 5421
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
4633 shm_sig_check(h);
4634 0 SHM_FN(expire_at)(h, i);
4635 0 continue;
4636 }
4637 453 idx = i;
4638 453 hdr->shift_cursor = i;
4639 453 break;
4640 }
4641 }
4642  
4643 485 ▶ if (idx == UINT32_MAX) {
Branch: 10 branches
BranchTrueFalseTotal
if (idx == UINT32_MAX) { 3 7 10
if (idx == UINT32_MAX) { 3 4 7
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 5 421 426
if (idx == UINT32_MAX) { 3 3 6
if (idx == UINT32_MAX) { 3 3 6
4644 32 ▶ if (hdr->size != size0) SHM_FN(maybe_shrink)(h);
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 1 4 5
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 3 3
4645 32 shm_seqlock_write_end(&hdr->seq);
4646 32 shm_rwlock_wrunlock(h);
4647 32 return 0;
4648 }
4649  
4650 #ifdef SHM_KEY_IS_INT
4651 436 *out_key = nodes[idx].key;
4652 #else
4653 {
4654 17 ▶ uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 3 4 7
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 3 1 4
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 3 0 3
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 3 0 3
4655 17 shm_sig_hold_bytes(h, kl);
4656 17 ▶ if (!shm_ensure_copy_buf(h, kl + 1)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 7 7
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 4 4
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 3 3
if (!shm_ensure_copy_buf(h, kl + 1)) { 0 3 3
4657 0 shm_seqlock_write_end(&hdr->seq);
4658 0 shm_rwlock_wrunlock(h);
4659 0 return 0;
4660 }
4661 17 shm_str_copy(h->copy_buf, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
4662 17 *out_key_str = h->copy_buf;
4663 17 *out_key_len = kl;
4664 17 *out_key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
4665 }
4666 #endif
4667 #ifdef SHM_VAL_IS_STR
4668 {
4669 16 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 3 4 7
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 3 0 3
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 3 0 3
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 3 0 3
4670 16 shm_sig_hold_bytes(h, vl);
4671 #ifndef SHM_KEY_IS_INT
4672 7 ▶ uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 3 4 7
4673 /* Value after the key; realloc may move the key too. */
4674 7 ▶ if (!shm_ensure_copy_buf(h, kl + vl + 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, kl + vl + 1)) { 0 7 7
4675 0 shm_seqlock_write_end(&hdr->seq);
4676 0 shm_rwlock_wrunlock(h);
4677 0 return 0;
4678 }
4679 7 *out_key_str = h->copy_buf;
4680 7 shm_str_copy(h->copy_buf + kl, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4681 7 *out_val_str = h->copy_buf + kl;
4682 #else
4683 9 ▶ if (!shm_ensure_copy_buf(h, vl + 1)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 3 3
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 3 3
if (!shm_ensure_copy_buf(h, vl + 1)) { 0 3 3
4684 0 shm_seqlock_write_end(&hdr->seq);
4685 0 shm_rwlock_wrunlock(h);
4686 0 return 0;
4687 }
4688 9 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4689 9 *out_val_str = h->copy_buf;
4690 #endif
4691 16 *out_val_len = vl;
4692 16 *out_val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4693 }
4694 #else
4695 437 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4696 #endif
4697  
4698 453 SHM_FN(remove_at)(h, idx);
4699 453 SHM_FN(maybe_shrink)(h);
4700  
4701 453 shm_seqlock_write_end(&hdr->seq);
4702 453 shm_rwlock_wrunlock(h);
4703 453 return 1;
4704 }
4705  
4706 /* ---- Drain (pop up to N entries, returns count) ---- */
4707  
4708 typedef struct {
4709 #ifdef SHM_KEY_IS_INT
4710 SHM_KEY_INT_TYPE key;
4711 #else
4712 uint32_t key_off; /* offset into drain_buf */
4713 uint32_t key_len;
4714 bool key_utf8;
4715 #endif
4716 #ifdef SHM_VAL_IS_STR
4717 uint32_t val_off; /* offset into drain_buf */
4718 uint32_t val_len;
4719 bool val_utf8;
4720 #else
4721 SHM_VAL_INT_TYPE value;
4722 #endif
4723 } SHM_PASTE(SHM_PREFIX, drain_entry);
4724  
4725 0 static uint32_t SHM_FN(drain_inner)(ShmHandle *h, uint32_t limit,
4726 SHM_PASTE(SHM_PREFIX, drain_entry) *out, char **buf, uint32_t *buf_cap,
4727 uint32_t *buf_used_p) {
4728 0 ShmHeader *hdr = h->hdr;
4729 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4730 0 uint8_t *states = h->states;
4731 0 uint32_t count = 0;
4732 0 uint32_t buf_used = *buf_used_p;
4733 (void)buf; (void)buf_cap;
4734  
4735 0 ▶ if (limit == 0) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (limit == 0) return 0; 0 23 23
if (limit == 0) return 0; 0 2 2
if (limit == 0) return 0; 0 0 0
if (limit == 0) return 0; 0 0 0
if (limit == 0) return 0; 0 1 1
if (limit == 0) return 0; 0 0 0
if (limit == 0) return 0; 0 0 0
if (limit == 0) return 0; 0 18 18
if (limit == 0) return 0; 0 0 0
if (limit == 0) return 0; 0 0 0
4736  
4737 0 shm_rwlock_wrlock(h);
4738 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 23 23
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 1 17 18
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
4739 0 shm_seqlock_write_begin(&hdr->seq);
4740  
4741 0 uint32_t cap = hdr->table_cap;
4742 0 ▶ uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 23 0 23
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 2 0 2
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 1 0 1
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 18 0 18
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0; 0 0 0
4743 0 uint32_t scanned = 0, size0 = hdr->size;
4744 0 ▶ while (count < limit) {
Branch: 10 branches
BranchTrueFalseTotal
while (count < limit) { 33 18 51
while (count < limit) { 3 2 5
while (count < limit) { 0 0 0
while (count < limit) { 0 0 0
while (count < limit) { 1 1 2
while (count < limit) { 0 0 0
while (count < limit) { 0 0 0
while (count < limit) { 20334 13 20347
while (count < limit) { 0 0 0
while (count < limit) { 0 0 0
4745 0 uint32_t idx = UINT32_MAX;
4746  
4747 0 ▶ if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) {
Branch: 20 branches
BranchTrueFalseTotal
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 33 33
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 3 3
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 1 1
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 20334 20334
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) { 0 0 0
4748 0 uint32_t pos = hdr->lru_tail;
4749 0 ▶ while (pos != SHM_LRU_NONE) {
Branch: 10 branches
BranchTrueFalseTotal
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
while (pos != SHM_LRU_NONE) { 0 0 0
4750 0 ▶ if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
Branch: 30 branches
BranchTrueFalseTotal
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; } 0 0 0
4751 0 ▶ uint32_t prev = h->lru_prev[pos];
Branch: 10 branches
BranchTrueFalseTotal
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
uint32_t prev = h->lru_prev[pos]; 0 0 0
4752 shm_sig_check(h);
4753 0 SHM_FN(expire_at)(h, pos);
4754 0 pos = prev;
4755 }
4756 } else {
4757 0 ▶ while (scanned < cap) {
Branch: 10 branches
BranchTrueFalseTotal
while (scanned < cap) { 185 5 190
while (scanned < cap) { 20 0 20
while (scanned < cap) { 0 0 0
while (scanned < cap) { 0 0 0
while (scanned < cap) { 15 0 15
while (scanned < cap) { 0 0 0
while (scanned < cap) { 0 0 0
while (scanned < cap) { 41701 5 41706
while (scanned < cap) { 0 0 0
while (scanned < cap) { 0 0 0
4758 0 uint32_t i = cur;
4759 0 cur = (cur + 1) % cap;
4760 0 shm_sig_check_every(h, ++scanned, SHM_SIG_CHECK_SLOTS);
4761 0 ▶ if (!SHM_IS_LIVE(states[i])) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (!SHM_IS_LIVE(states[i])) continue; 157 28 185
if (!SHM_IS_LIVE(states[i])) continue; 17 3 20
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
if (!SHM_IS_LIVE(states[i])) continue; 14 1 15
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
if (!SHM_IS_LIVE(states[i])) continue; 16372 25329 41701
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
if (!SHM_IS_LIVE(states[i])) continue; 0 0 0
4762 0 ▶ if (SHM_IS_EXPIRED(h, i, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, i, now)) { 0 28 28
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 3 3
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 5000 20329 25329
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 5000 0 5000
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, i, now)) { 0 0 0
4763 shm_sig_check(h);
4764 0 SHM_FN(expire_at)(h, i);
4765 0 continue;
4766 }
4767 0 idx = i;
4768 0 break;
4769 }
4770 }
4771  
4772 0 ▶ if (idx == UINT32_MAX) break;
Branch: 10 branches
BranchTrueFalseTotal
if (idx == UINT32_MAX) break; 5 28 33
if (idx == UINT32_MAX) break; 0 3 3
if (idx == UINT32_MAX) break; 0 0 0
if (idx == UINT32_MAX) break; 0 0 0
if (idx == UINT32_MAX) break; 0 1 1
if (idx == UINT32_MAX) break; 0 0 0
if (idx == UINT32_MAX) break; 0 0 0
if (idx == UINT32_MAX) break; 5 20329 20334
if (idx == UINT32_MAX) break; 0 0 0
if (idx == UINT32_MAX) break; 0 0 0
4773  
4774 #ifdef SHM_KEY_IS_INT
4775 0 out[count].key = nodes[idx].key;
4776 #else
4777 {
4778 0 ▶ uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 16 12 28
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 1 2 3
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 0 0 0
uint32_t kl = SHM_STR_LEN(nodes[idx].key_len); 0 0 0
4779 0 shm_sig_hold_bytes(h, kl);
4780 0 ▶ if ((uint64_t)buf_used + kl > UINT32_MAX) break;
Branch: 4 branches
BranchTrueFalseTotal
if ((uint64_t)buf_used + kl > UINT32_MAX) break; 0 28 28
if ((uint64_t)buf_used + kl > UINT32_MAX) break; 0 3 3
if ((uint64_t)buf_used + kl > UINT32_MAX) break; 0 0 0
if ((uint64_t)buf_used + kl > UINT32_MAX) break; 0 0 0
4781 0 ▶ if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break;
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break; 0 28 28
if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break; 0 3 3
if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break; 0 0 0
if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break; 0 0 0
4782 0 shm_str_copy(*buf + buf_used, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
4783 0 out[count].key_off = buf_used;
4784 0 out[count].key_len = kl;
4785 0 out[count].key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
4786 0 buf_used += kl;
4787 }
4788 #endif
4789 #ifdef SHM_VAL_IS_STR
4790 {
4791 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 17 11 28
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 1 0 1
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
4792 0 shm_sig_hold_bytes(h, vl);
4793 0 ▶ if ((uint64_t)buf_used + vl > UINT32_MAX) break;
Branch: 4 branches
BranchTrueFalseTotal
if ((uint64_t)buf_used + vl > UINT32_MAX) break; 0 28 28
if ((uint64_t)buf_used + vl > UINT32_MAX) break; 0 1 1
if ((uint64_t)buf_used + vl > UINT32_MAX) break; 0 0 0
if ((uint64_t)buf_used + vl > UINT32_MAX) break; 0 0 0
4794 0 ▶ if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break;
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break; 0 28 28
if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break; 0 1 1
if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break; 0 0 0
if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break; 0 0 0
4795 0 shm_str_copy(*buf + buf_used, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
4796 0 out[count].val_off = buf_used;
4797 0 out[count].val_len = vl;
4798 0 out[count].val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
4799 0 buf_used += vl;
4800 }
4801 #else
4802 0 out[count].value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
4803 #endif
4804  
4805 0 SHM_FN(remove_at)(h, idx);
4806 0 shm_sig_check_every(h, ++count, SHM_SIG_CHECK_ENTRIES);
4807 }
4808  
4809 0 ▶ if (!h->lru_prev) hdr->pop_cursor = cur;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->lru_prev) hdr->pop_cursor = cur; 23 0 23
if (!h->lru_prev) hdr->pop_cursor = cur; 2 0 2
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
if (!h->lru_prev) hdr->pop_cursor = cur; 1 0 1
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
if (!h->lru_prev) hdr->pop_cursor = cur; 18 0 18
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
if (!h->lru_prev) hdr->pop_cursor = cur; 0 0 0
4810 0 ▶ if (hdr->size != size0) SHM_FN(maybe_shrink)(h);
Branch: 10 branches
BranchTrueFalseTotal
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 20 3 23
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 2 0 2
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 1 0 1
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 16 2 18
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
if (hdr->size != size0) SHM_FN(maybe_shrink)(h); 0 0 0
4811  
4812 0 shm_seqlock_write_end(&hdr->seq);
4813 0 shm_rwlock_wrunlock(h);
4814 0 *buf_used_p = buf_used;
4815 0 return count;
4816 }
4817  
4818 0 static uint32_t SHM_FN(drain)(ShmHandle *h, uint32_t limit,
4819 SHM_PASTE(SHM_PREFIX, drain_entry) *out, char **buf, uint32_t *buf_cap) {
4820 0 uint32_t buf_used = 0;
4821 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 14 4 18
if (h->shard_handles) { 1 1 2
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 2 12 14
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
4822 0 uint32_t total = 0;
4823 0 ▶ for (uint32_t i = 0; i < h->num_shards && total < limit; i++) {
Branch: 20 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 32 1 33
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 19 13 32
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 2 0 2
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 1 1 2
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 7 1 8
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 6 1 7
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards && total < limit; i++) { 0 0 0
4824 0 uint32_t si = (h->shard_rr + i) % h->num_shards;
4825 0 uint32_t got = SHM_FN(drain_inner)(h->shard_handles[si], limit - total,
4826 0 out + total, buf, buf_cap, &buf_used);
4827 0 total += got;
4828 }
4829 0 ▶ if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards;
Branch: 10 branches
BranchTrueFalseTotal
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 14 0 14
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 1 0 1
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 2 0 2
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards; 0 0 0
4830 0 return total;
4831 }
4832 0 return SHM_FN(drain_inner)(h, limit, out, buf, buf_cap, &buf_used);
4833 }
4834  
4835 /* ---- Counter operations, atomic max/min, and integer-value cas ---- */
4836  
4837 #ifdef SHM_HAS_COUNTERS
4838  
4839 9796 static inline int SHM_FN(find_slot)(ShmHandle *h,
4840 #ifdef SHM_KEY_IS_INT
4841 SHM_KEY_INT_TYPE key,
4842 #else
4843 const char *key_str, uint32_t key_len, bool key_utf8,
4844 #endif
4845 uint32_t *out_idx) {
4846 9796 ShmHeader *hdr = h->hdr;
4847 9796 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4848 9796 uint8_t *states = h->states;
4849 9796 uint32_t mask = hdr->table_cap - 1;
4850 #ifdef SHM_KEY_IS_INT
4851 9640 uint32_t hash = SHM_HASH_KEY(key);
4852 #else
4853 156 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
4854 #endif
4855 9796 uint32_t pos = hash & mask;
4856 9796 uint8_t tag = SHM_MAKE_TAG(hash);
4857 9796 uint32_t probe_start = 0;
4858 9796 __builtin_prefetch(&nodes[pos], 0, 1);
4859  
4860 #ifdef __SSE2__
4861 9796 ▶ if (pos + 16 <= hdr->table_cap) {
Branch: 6 branches
BranchTrueFalseTotal
if (pos + 16 <= hdr->table_cap) { 104 20 124
if (pos + 16 <= hdr->table_cap) { 1 14 15
if (pos + 16 <= hdr->table_cap) { 1 16 17
if (pos + 16 <= hdr->table_cap) { 9409 196 9605
if (pos + 16 <= hdr->table_cap) { 1 15 16
if (pos + 16 <= hdr->table_cap) { 1 18 19
4862 uint16_t mmask, emask;
4863 9517 shm_probe_group(states, pos, tag, &mmask, &emask);
4864 9517 ▶ uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF;
Branch: 6 branches
BranchTrueFalseTotal
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 104 0 104
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 1 0 1
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 1 0 1
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 9230 179 9409
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 0 1 1
uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF; 0 1 1
4865 9517 uint16_t relevant = mmask & cutoff;
4866 9545 ▶ while (relevant) {
Branch: 6 branches
BranchTrueFalseTotal
while (relevant) { 3 101 104
while (relevant) { 0 1 1
while (relevant) { 0 1 1
while (relevant) { 7019 2418 9437
while (relevant) { 0 1 1
while (relevant) { 0 1 1
4867 0 int bit = __builtin_ctz(relevant);
4868 0 uint32_t idx = pos + bit;
4869 #ifdef SHM_KEY_IS_INT
4870 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) { *out_idx = idx; return 1; }
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { *out_idx = idx; return 1; } 6991 28 7019
if (SHM_KEY_EQ(&nodes[idx], key)) { *out_idx = idx; return 1; } 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { *out_idx = idx; return 1; } 0 0 0
4871 #else
4872 106 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { *out_idx = idx; return 1; }
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { *out_idx = idx; return 1; } 3 0 3
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { *out_idx = idx; return 1; } 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { *out_idx = idx; return 1; } 0 0 0
4873 #endif
4874 0 relevant &= relevant - 1;
4875 }
4876 2523 ▶ if (emask) return 0;
Branch: 6 branches
BranchTrueFalseTotal
if (emask) return 0; 101 0 101
if (emask) return 0; 1 0 1
if (emask) return 0; 1 0 1
if (emask) return 0; 2378 40 2418
if (emask) return 0; 0 1 1
if (emask) return 0; 0 1 1
4877 0 probe_start = 16;
4878 }
4879 #endif
4880  
4881 1173 ▶ for (uint32_t i = probe_start; i <= mask; i++) {
Branch: 6 branches
BranchTrueFalseTotal
for (uint32_t i = probe_start; i <= mask; i++) { 101 5 106
for (uint32_t i = probe_start; i <= mask; i++) { 89 5 94
for (uint32_t i = probe_start; i <= mask; i++) { 91 5 96
for (uint32_t i = probe_start; i <= mask; i++) { 704 10 714
for (uint32_t i = probe_start; i <= mask; i++) { 75 5 80
for (uint32_t i = probe_start; i <= mask; i++) { 78 5 83
4882 1138 uint32_t idx = (pos + i) & mask;
4883 1138 uint8_t st = states[idx];
4884 1138 __builtin_prefetch(&nodes[idx], 0, 1);
4885 1138 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
4886 1138 ▶ if (st == SHM_EMPTY) return 0;
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) return 0; 4 97 101
if (st == SHM_EMPTY) return 0; 2 87 89
if (st == SHM_EMPTY) return 0; 2 89 91
if (st == SHM_EMPTY) return 0; 108 596 704
if (st == SHM_EMPTY) return 0; 3 72 75
if (st == SHM_EMPTY) return 0; 3 75 78
4887 1016 ▶ if (st != tag) continue;
Branch: 6 branches
BranchTrueFalseTotal
if (st != tag) continue; 86 11 97
if (st != tag) continue; 80 7 87
if (st != tag) continue; 80 9 89
if (st != tag) continue; 477 119 596
if (st != tag) continue; 63 9 72
if (st != tag) continue; 63 12 75
4888 #ifdef SHM_KEY_IS_INT
4889 140 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 118 1 119
if (SHM_KEY_EQ(&nodes[idx], key)) { 8 1 9
if (SHM_KEY_EQ(&nodes[idx], key)) { 11 1 12
4890 #else
4891 27 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 11 0 11
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 7 0 7
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 9 0 9
4892 #endif
4893 164 *out_idx = idx;
4894 164 return 1;
4895 }
4896 }
4897 35 return 0;
4898 }
4899  
4900 4886 static SHM_VAL_INT_TYPE SHM_FN(incr_by)(ShmHandle *h,
4901 #ifdef SHM_KEY_IS_INT
4902 SHM_KEY_INT_TYPE key,
4903 #else
4904 const char *key_str, uint32_t key_len, bool key_utf8,
4905 #endif
4906 SHM_VAL_INT_TYPE delta, int *ok) {
4907 #ifdef SHM_KEY_IS_INT
4908 4813 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 4795 4795
SHM_SHARD_DISPATCH(h, key); 0 7 7
SHM_SHARD_DISPATCH(h, key); 0 11 11
4909 #else
4910 73 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 60 60
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 5 5
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 8 8
4911 #endif
4912 4886 ShmHeader *hdr = h->hdr;
4913 4886 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
4914  
4915 /* No LRU or TTL: only the value word changes, which get() reads with one
4916 * atomic load, so the read lock suffices and seq is not bumped. */
4917 4886 ▶ if (!h->lru_prev && !h->expires_at) {
Branch: 12 branches
BranchTrueFalseTotal
if (!h->lru_prev && !h->expires_at) { 58 2 60
if (!h->lru_prev && !h->expires_at) { 57 1 58
if (!h->lru_prev && !h->expires_at) { 5 0 5
if (!h->lru_prev && !h->expires_at) { 5 0 5
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 7 1 8
if (!h->lru_prev && !h->expires_at) { 4779 16 4795
if (!h->lru_prev && !h->expires_at) { 4779 0 4779
if (!h->lru_prev && !h->expires_at) { 7 0 7
if (!h->lru_prev && !h->expires_at) { 6 1 7
if (!h->lru_prev && !h->expires_at) { 11 0 11
if (!h->lru_prev && !h->expires_at) { 9 2 11
4918 4863 shm_rwlock_rdlock(h);
4919 uint32_t idx;
4920 #ifdef SHM_KEY_IS_INT
4921 4794 ▶ if (SHM_FN(find_slot)(h, key, &idx)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(find_slot)(h, key, &idx)) { 3539 1240 4779
if (SHM_FN(find_slot)(h, key, &idx)) { 2 4 6
if (SHM_FN(find_slot)(h, key, &idx)) { 5 4 9
4922 #else
4923 69 ▶ if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 3 54 57
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 1 4 5
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 3 4 7
4924 #endif
4925 3553 SHM_VAL_INT_TYPE result =
4926 3553 __atomic_add_fetch(&nodes[idx].value, delta, __ATOMIC_ACQ_REL);
4927 3553 shm_rwlock_rdunlock(h);
4928 3553 *ok = 1;
4929 3553 return result;
4930 }
4931 1310 shm_rwlock_rdunlock(h);
4932 }
4933  
4934 1333 shm_rwlock_wrlock(h);
4935 1333 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 6 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 1 56 57
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 1 4 5
uint32_t now = h->expires_at ? shm_now() : 0; 0 1256 1256
uint32_t now = h->expires_at ? shm_now() : 0; 1 4 5
uint32_t now = h->expires_at ? shm_now() : 0; 2 4 6
4936 1333 shm_seqlock_write_begin(&hdr->seq);
4937  
4938 SHM_FN(maybe_grow)(h);
4939 1333 uint32_t mask = hdr->table_cap - 1;
4940 #ifdef SHM_KEY_IS_INT
4941 1267 uint32_t hash = SHM_HASH_KEY(key);
4942 #else
4943 66 shm_sig_hold_bytes(h, key_len);
4944 66 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
4945 #endif
4946 1333 uint32_t pos = hash & mask;
4947 1333 uint32_t insert_pos = UINT32_MAX;
4948  
4949 1333 uint8_t tag = SHM_MAKE_TAG(hash);
4950 5548 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 6 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 284 3 287
for (uint32_t i = 0; i <= mask; i++) { 49 3 52
for (uint32_t i = 0; i <= mask; i++) { 50 3 53
for (uint32_t i = 0; i <= mask; i++) { 5043 6 5049
for (uint32_t i = 0; i <= mask; i++) { 50 3 53
for (uint32_t i = 0; i <= mask; i++) { 51 3 54
4951 5527 uint32_t slot = (pos + i) & mask;
4952 5527 uint8_t st = h->states[slot];
4953 5527 __builtin_prefetch(&nodes[slot], 0, 1);
4954 5527 __builtin_prefetch(&nodes[(slot + 1) & mask], 0, 1);
4955 5527 ▶ if (st == SHM_EMPTY) {
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 54 230 284
if (st == SHM_EMPTY) { 1 48 49
if (st == SHM_EMPTY) { 1 49 50
if (st == SHM_EMPTY) { 1234 3809 5043
if (st == SHM_EMPTY) { 1 49 50
if (st == SHM_EMPTY) { 1 50 51
4956 1292 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 3 51 54
if (insert_pos == UINT32_MAX) insert_pos = slot; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = slot; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = slot; 194 1040 1234
if (insert_pos == UINT32_MAX) insert_pos = slot; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = slot; 1 0 1
4957 1292 break;
4958 }
4959 4235 ▶ if (st == SHM_TOMBSTONE) {
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 61 169 230
if (st == SHM_TOMBSTONE) { 0 48 48
if (st == SHM_TOMBSTONE) { 0 49 49
if (st == SHM_TOMBSTONE) { 1545 2264 3809
if (st == SHM_TOMBSTONE) { 0 49 49
if (st == SHM_TOMBSTONE) { 0 50 50
4960 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 51 10 61
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 1040 505 1545
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
4961 0 continue;
4962 }
4963 2629 ▶ if (st != tag) continue;
Branch: 6 branches
BranchTrueFalseTotal
if (st != tag) continue; 169 0 169
if (st != tag) continue; 48 0 48
if (st != tag) continue; 48 1 49
if (st != tag) continue; 2238 26 2264
if (st != tag) continue; 48 1 49
if (st != tag) continue; 48 2 50
4964 #ifdef SHM_KEY_IS_INT
4965 29 ▶ if (SHM_KEY_EQ(&nodes[slot], key)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[slot], key)) { 16 10 26
if (SHM_KEY_EQ(&nodes[slot], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[slot], key)) { 2 0 2
4966 #else
4967 0 ▶ if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
4968 #endif
4969 0 ▶ if (SHM_IS_EXPIRED(h, slot, now)) {
Branch: 18 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, slot, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, slot, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, slot, now)) { 0 16 16
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, slot, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, slot, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, slot, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, slot, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, slot, now)) { 0 2 2
4970 0 SHM_FN(expire_at)(h, slot);
4971 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
4972 0 break;
4973 }
4974  
4975 0 __atomic_add_fetch(&nodes[slot].value, delta, __ATOMIC_RELAXED);
4976 0 SHM_VAL_INT_TYPE result = __atomic_load_n(&nodes[slot].value, __ATOMIC_RELAXED);
4977 0 ▶ if (h->lru_prev) shm_lru_promote(h, slot);
Branch: 6 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 1 1
if (h->lru_prev) shm_lru_promote(h, slot); 16 0 16
if (h->lru_prev) shm_lru_promote(h, slot); 0 1 1
if (h->lru_prev) shm_lru_promote(h, slot); 0 2 2
4978 0 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0)
Branch: 18 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 16 16
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 2 0 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 2 0 2
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 2 0 2
4979 0 h->expires_at[slot] = shm_expiry_ts(hdr->default_ttl);
4980 0 shm_seqlock_write_end(&hdr->seq);
4981 0 shm_rwlock_wrunlock(h);
4982 0 *ok = 1;
4983 0 return result;
4984 }
4985 }
4986  
4987 1313 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) {
Branch: 6 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 3 54 57
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 3 1 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 3 1 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 6 1234 1240
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 3 1 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 3 1 4
4988 21 shm_seqlock_write_end(&hdr->seq);
4989 21 shm_rwlock_wrunlock(h);
4990 21 *ok = 0;
4991 21 return 0;
4992 }
4993  
4994 1292 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), 0);
4995  
4996 1292 int was_tombstone = (h->states[insert_pos] == SHM_TOMBSTONE);
4997 1292 SHM_FN(arena_reclaim)(h);
4998 #ifdef SHM_KEY_IS_INT
4999 1236 nodes[insert_pos].key = key;
5000 #else
5001 56 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 1 53 54
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 1 1
5002 0 shm_seqlock_write_end(&hdr->seq);
5003 0 shm_rwlock_wrunlock(h);
5004 0 *ok = 0;
5005 0 return 0;
5006 }
5007 #endif
5008 1291 __atomic_store_n(&nodes[insert_pos].value, delta, __ATOMIC_RELAXED);
5009 1291 ▶ if (h->expires_at)
Branch: 6 branches
BranchTrueFalseTotal
if (h->expires_at) 1 52 53
if (h->expires_at) 0 1 1
if (h->expires_at) 0 1 1
if (h->expires_at) 0 1234 1234
if (h->expires_at) 0 1 1
if (h->expires_at) 0 1 1
5010 0 ▶ h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0;
Branch: 6 branches
BranchTrueFalseTotal
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 1 0 1
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
5011 1291 shm_publish_tag(h->states, insert_pos, SHM_MAKE_TAG(hash));
5012 1291 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 6 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 51 2 53
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 1040 194 1234
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
5013 1291 hdr->size++;
5014  
5015 1291 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 6 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 1 52 53
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1234 1234
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
5016  
5017 1291 shm_seqlock_write_end(&hdr->seq);
5018 1291 shm_rwlock_wrunlock(h);
5019 1291 *ok = 1;
5020 1291 return delta;
5021 }
5022  
5023 /* ---- Atomic max / min: returns the stored value; an absent key gets `desired` ---- */
5024 4936 static SHM_VAL_INT_TYPE SHM_FN(set_minmax)(ShmHandle *h,
5025 #ifdef SHM_KEY_IS_INT
5026 SHM_KEY_INT_TYPE key,
5027 #else
5028 const char *key_str, uint32_t key_len, bool key_utf8,
5029 #endif
5030 SHM_VAL_INT_TYPE desired, int want_max, int *ok) {
5031 #ifdef SHM_KEY_IS_INT
5032 4842 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 4826 4826
SHM_SHARD_DISPATCH(h, key); 0 8 8
SHM_SHARD_DISPATCH(h, key); 0 8 8
5033 #else
5034 94 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 78 78
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 8 8
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 8 8
5035 #endif
5036 4936 ShmHeader *hdr = h->hdr;
5037 4936 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5038  
5039 /* Read-lock fast path, as in incr_by. */
5040 4936 ▶ if (!h->lru_prev && !h->expires_at) {
Branch: 12 branches
BranchTrueFalseTotal
if (!h->lru_prev && !h->expires_at) { 68 10 78
if (!h->lru_prev && !h->expires_at) { 64 4 68
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 4826 0 4826
if (!h->lru_prev && !h->expires_at) { 4826 0 4826
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
if (!h->lru_prev && !h->expires_at) { 8 0 8
5041 4922 shm_rwlock_rdlock(h);
5042 uint32_t idx;
5043 #ifdef SHM_KEY_IS_INT
5044 4842 ▶ if (SHM_FN(find_slot)(h, key, &idx)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(find_slot)(h, key, &idx)) { 3570 1256 4826
if (SHM_FN(find_slot)(h, key, &idx)) { 4 4 8
if (SHM_FN(find_slot)(h, key, &idx)) { 4 4 8
5045 #else
5046 80 ▶ if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 8 56 64
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 4 4 8
if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) { 4 4 8
5047 #endif
5048 3594 SHM_VAL_INT_TYPE cur = __atomic_load_n(&nodes[idx].value, __ATOMIC_ACQUIRE);
5049 3594 ▶ while (want_max ? (desired > cur) : (desired < cur)) {
Branch: 12 branches
BranchTrueFalseTotal
while (want_max ? (desired > cur) : (desired < cur)) { 4 4 8
while (want_max ? (desired > cur) : (desired < cur)) { 4 4 8
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 3568 2 3570
while (want_max ? (desired > cur) : (desired < cur)) { 1798 1772 3570
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
while (want_max ? (desired > cur) : (desired < cur)) { 2 2 4
5050 1810 ▶ if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired,
Branch: 6 branches
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 4 0 4
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 2 0 2
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 2 0 2
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 1798 0 1798
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 2 0 2
if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired, 2 0 2
5051 0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
5052 1810 cur = desired;
5053 1810 break;
5054 }
5055 }
5056 3594 shm_rwlock_rdunlock(h);
5057 3594 *ok = 1;
5058 3594 return cur;
5059 }
5060 1328 shm_rwlock_rdunlock(h);
5061 }
5062  
5063 1342 shm_rwlock_wrlock(h);
5064 1342 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 6 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 4 66 70
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 1256 1256
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
uint32_t now = h->expires_at ? shm_now() : 0; 0 4 4
5065 1342 shm_seqlock_write_begin(&hdr->seq);
5066  
5067 SHM_FN(maybe_grow)(h);
5068 1342 uint32_t mask = hdr->table_cap - 1;
5069 #ifdef SHM_KEY_IS_INT
5070 1264 uint32_t hash = SHM_HASH_KEY(key);
5071 #else
5072 78 shm_sig_hold_bytes(h, key_len);
5073 78 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5074 #endif
5075 1342 uint32_t pos = hash & mask;
5076 1342 uint32_t insert_pos = UINT32_MAX;
5077  
5078 1342 uint8_t tag = SHM_MAKE_TAG(hash);
5079 5594 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 6 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 272 2 274
for (uint32_t i = 0; i <= mask; i++) { 34 2 36
for (uint32_t i = 0; i <= mask; i++) { 34 2 36
for (uint32_t i = 0; i <= mask; i++) { 5172 4 5176
for (uint32_t i = 0; i <= mask; i++) { 34 2 36
for (uint32_t i = 0; i <= mask; i++) { 34 2 36
5080 5580 uint32_t slot = (pos + i) & mask;
5081 5580 uint8_t st = h->states[slot];
5082 5580 __builtin_prefetch(&nodes[slot], 0, 1);
5083 5580 __builtin_prefetch(&nodes[(slot + 1) & mask], 0, 1);
5084 5580 ▶ if (st == SHM_EMPTY) {
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 63 209 272
if (st == SHM_EMPTY) { 2 32 34
if (st == SHM_EMPTY) { 2 32 34
if (st == SHM_EMPTY) { 1252 3920 5172
if (st == SHM_EMPTY) { 2 32 34
if (st == SHM_EMPTY) { 2 32 34
5085 1323 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 13 50 63
if (insert_pos == UINT32_MAX) insert_pos = slot; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = slot; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = slot; 154 1098 1252
if (insert_pos == UINT32_MAX) insert_pos = slot; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = slot; 2 0 2
5086 1323 break;
5087 }
5088 4257 ▶ if (st == SHM_TOMBSTONE) {
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 60 149 209
if (st == SHM_TOMBSTONE) { 0 32 32
if (st == SHM_TOMBSTONE) { 0 32 32
if (st == SHM_TOMBSTONE) { 1588 2332 3920
if (st == SHM_TOMBSTONE) { 0 32 32
if (st == SHM_TOMBSTONE) { 0 32 32
5089 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 50 10 60
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 1098 490 1588
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
5090 0 continue;
5091 }
5092 2609 ▶ if (st != tag) continue;
Branch: 6 branches
BranchTrueFalseTotal
if (st != tag) continue; 144 5 149
if (st != tag) continue; 32 0 32
if (st != tag) continue; 32 0 32
if (st != tag) continue; 2322 10 2332
if (st != tag) continue; 31 1 32
if (st != tag) continue; 31 1 32
5093 #ifdef SHM_KEY_IS_INT
5094 12 ▶ if (SHM_KEY_EQ(&nodes[slot], key)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[slot], key)) { 0 10 10
if (SHM_KEY_EQ(&nodes[slot], key)) { 0 1 1
if (SHM_KEY_EQ(&nodes[slot], key)) { 0 1 1
5095 #else
5096 0 ▶ if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 5 0 5
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
5097 #endif
5098 0 ▶ if (SHM_IS_EXPIRED(h, slot, now)) {
Branch: 18 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, slot, now)) { 2 3 5
if (SHM_IS_EXPIRED(h, slot, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, slot, now)) { 1 1 2
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, slot, now)) { 0 0 0
5099 0 SHM_FN(expire_at)(h, slot);
5100 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = slot;
Branch: 6 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = slot; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = slot; 0 0 0
5101 0 break;
5102 }
5103  
5104 0 SHM_VAL_INT_TYPE cur = __atomic_load_n(&nodes[slot].value, __ATOMIC_RELAXED);
5105 0 ▶ SHM_VAL_INT_TYPE result =
Branch: 2 branches
BranchTrueFalseTotal
SHM_VAL_INT_TYPE result = 0 0 0
SHM_VAL_INT_TYPE result = 0 0 0
5106 want_max ? (desired > cur ? desired : cur)
5107 0 ▶ : (desired < cur ? desired : cur);
Branch: 4 branches
BranchTrueFalseTotal
: (desired < cur ? desired : cur); 2 2 4
: (desired < cur ? desired : cur); 0 0 0
: (desired < cur ? desired : cur); 0 0 0
: (desired < cur ? desired : cur); 0 0 0
5108 0 __atomic_store_n(&nodes[slot].value, result, __ATOMIC_RELAXED);
5109 0 ▶ if (h->lru_prev) shm_lru_promote(h, slot);
Branch: 6 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, slot); 3 1 4
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
if (h->lru_prev) shm_lru_promote(h, slot); 0 0 0
5110 0 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0)
Branch: 18 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 3 4
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0) 0 0 0
5111 0 h->expires_at[slot] = shm_expiry_ts(hdr->default_ttl);
5112 0 shm_seqlock_write_end(&hdr->seq);
5113 0 shm_rwlock_wrunlock(h);
5114 0 *ok = 1;
5115 0 return result;
5116 }
5117 }
5118  
5119 1338 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) {
Branch: 6 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 64 66
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 2 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 2 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 4 1252 1256
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 2 4
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 2 4
5120 14 shm_seqlock_write_end(&hdr->seq);
5121 14 shm_rwlock_wrunlock(h);
5122 14 *ok = 0;
5123 14 return 0;
5124 }
5125  
5126 1324 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), 0);
5127  
5128 1324 int was_tombstone = (h->states[insert_pos] == SHM_TOMBSTONE);
5129 1324 SHM_FN(arena_reclaim)(h);
5130 #ifdef SHM_KEY_IS_INT
5131 1256 nodes[insert_pos].key = key;
5132 #else
5133 68 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 1 63 64
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 2 2
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 2 2
5134 0 shm_seqlock_write_end(&hdr->seq);
5135 0 shm_rwlock_wrunlock(h);
5136 0 *ok = 0;
5137 0 return 0;
5138 }
5139 #endif
5140 1323 __atomic_store_n(&nodes[insert_pos].value, desired, __ATOMIC_RELAXED);
5141 1323 ▶ if (h->expires_at)
Branch: 6 branches
BranchTrueFalseTotal
if (h->expires_at) 3 60 63
if (h->expires_at) 0 2 2
if (h->expires_at) 0 2 2
if (h->expires_at) 0 1252 1252
if (h->expires_at) 0 2 2
if (h->expires_at) 0 2 2
5142 0 ▶ h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0;
Branch: 6 branches
BranchTrueFalseTotal
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 3 0 3
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
5143 1323 shm_publish_tag(h->states, insert_pos, SHM_MAKE_TAG(hash));
5144 1323 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 6 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 51 12 63
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 1098 154 1252
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 2 2
5145 1323 hdr->size++;
5146  
5147 1323 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 6 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 6 57 63
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1252 1252
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
5148  
5149 1323 shm_seqlock_write_end(&hdr->seq);
5150 1323 shm_rwlock_wrunlock(h);
5151 1323 *ok = 1;
5152 1323 return desired;
5153 }
5154  
5155 /* ---- Compare-and-swap (atomic, integer-value variants) ---- */
5156  
5157 12 static int SHM_FN(cas)(ShmHandle *h,
5158 #ifdef SHM_KEY_IS_INT
5159 SHM_KEY_INT_TYPE key,
5160 #else
5161 const char *key_str, uint32_t key_len, bool key_utf8,
5162 #endif
5163 SHM_VAL_INT_TYPE expected, SHM_VAL_INT_TYPE desired
5164 ) {
5165 #ifdef SHM_KEY_IS_INT
5166 5 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 2 2
5167 #else
5168 7 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 3 3
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
5169 #endif
5170 12 ShmHeader *hdr = h->hdr;
5171 12 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5172 12 uint8_t *states = h->states;
5173  
5174 /* No LRU or TTL: only the value word changes, as in incr_by's fast path. */
5175 12 ▶ if (!h->lru_prev && !h->expires_at) {
Branch: 12 branches
BranchTrueFalseTotal
if (!h->lru_prev && !h->expires_at) { 3 0 3
if (!h->lru_prev && !h->expires_at) { 3 0 3
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 0 1 1
if (!h->lru_prev && !h->expires_at) { 0 0 0
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
5176 0 shm_rwlock_rdlock(h);
5177 uint32_t idx;
5178 #ifdef SHM_KEY_IS_INT
5179 0 int ok = SHM_FN(find_slot)(h, key, &idx);
5180 #else
5181 7 int ok = SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx);
5182 #endif
5183 0 ▶ ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired,
Branch: 12 branches
BranchTrueFalseTotal
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 3 0 3
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 2 1 3
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 2 0 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 1 1 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 2 0 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 1 1 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 0 0 0
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 0 0 0
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 2 0 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 1 1 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 2 0 2
ok = ok && __atomic_compare_exchange_n(&nodes[idx].value, &expected, desired, 1 1 2
5184 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
5185 0 shm_rwlock_rdunlock(h);
5186 0 return ok;
5187 }
5188  
5189 0 shm_rwlock_wrlock(h);
5190 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 6 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
5191 0 shm_seqlock_write_begin(&hdr->seq);
5192  
5193 0 uint32_t mask = hdr->table_cap - 1;
5194 #ifdef SHM_KEY_IS_INT
5195 0 uint32_t hash = SHM_HASH_KEY(key);
5196 #else
5197 0 shm_sig_hold_bytes(h, key_len);
5198 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5199 #endif
5200 0 uint32_t pos = hash & mask;
5201 0 uint8_t tag = SHM_MAKE_TAG(hash);
5202  
5203 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 6 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
5204 0 uint32_t idx = (pos + i) & mask;
5205 0 uint8_t st = states[idx];
5206 0 __builtin_prefetch(&nodes[idx], 0, 1);
5207 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5208 0 ▶ if (st == SHM_EMPTY) break;
Branch: 6 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
5209 0 ▶ if (st != tag) continue;
Branch: 6 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
5210 #ifdef SHM_KEY_IS_INT
5211 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
5212 #else
5213 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
5214 #endif
5215 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 18 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
5216 0 SHM_FN(expire_at)(h, idx);
5217 0 SHM_FN(maybe_shrink)(h);
5218 0 shm_seqlock_write_end(&hdr->seq);
5219 0 shm_rwlock_wrunlock(h);
5220 0 return 0;
5221 }
5222 0 ▶ if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) {
Branch: 6 branches
BranchTrueFalseTotal
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 1 0 1
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 0 0 0
if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) { 0 0 0
5223 0 __atomic_store_n(&nodes[idx].value, desired, __ATOMIC_RELAXED);
5224 0 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 6 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 1 0 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
5225 0 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
Branch: 18 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 1 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
5226 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
5227 0 shm_seqlock_write_end(&hdr->seq);
5228 0 shm_rwlock_wrunlock(h);
5229 0 return 1;
5230 }
5231 0 shm_seqlock_write_end(&hdr->seq);
5232 0 shm_rwlock_wrunlock(h);
5233 0 return 0;
5234 }
5235 }
5236  
5237 0 shm_seqlock_write_end(&hdr->seq);
5238 0 shm_rwlock_wrunlock(h);
5239 0 return 0;
5240 }
5241  
5242 #endif /* SHM_HAS_COUNTERS */
5243  
5244 /* ---- Compare-and-swap (atomic, string-value variants) ---- */
5245  
5246 #ifdef SHM_VAL_IS_STR
5247 33 static int SHM_FN(cas)(ShmHandle *h,
5248 #ifdef SHM_KEY_IS_INT
5249 SHM_KEY_INT_TYPE key,
5250 #else
5251 const char *key_str, uint32_t key_len, bool key_utf8,
5252 #endif
5253 const char *expected_str, uint32_t expected_len,
5254 const char *desired_str, uint32_t desired_len, bool desired_utf8
5255 ) {
5256 #ifdef SHM_KEY_IS_INT
5257 21 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 3 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 7 7
SHM_SHARD_DISPATCH(h, key); 0 7 7
SHM_SHARD_DISPATCH(h, key); 0 7 7
5258 #else
5259 12 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 1 branch
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 12 12
5260 #endif
5261 33 ShmHeader *hdr = h->hdr;
5262 33 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5263 33 uint8_t *states = h->states;
5264 33 shm_rwlock_wrlock(h);
5265 33 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 4 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 1 11 12
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
uint32_t now = h->expires_at ? shm_now() : 0; 0 7 7
5266 33 shm_seqlock_write_begin(&hdr->seq);
5267  
5268 33 uint32_t mask = hdr->table_cap - 1;
5269 #ifdef SHM_KEY_IS_INT
5270 21 uint32_t hash = SHM_HASH_KEY(key);
5271 #else
5272 12 shm_sig_hold_bytes(h, key_len);
5273 12 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5274 #endif
5275 33 uint32_t pos = hash & mask;
5276 33 uint8_t tag = SHM_MAKE_TAG(hash);
5277  
5278 42 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 4 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 12 0 12
for (uint32_t i = 0; i <= mask; i++) { 10 0 10
for (uint32_t i = 0; i <= mask; i++) { 10 0 10
for (uint32_t i = 0; i <= mask; i++) { 10 0 10
5279 42 uint32_t idx = (pos + i) & mask;
5280 42 uint8_t st = states[idx];
5281 42 __builtin_prefetch(&nodes[idx], 0, 1);
5282 42 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5283 42 ▶ if (st == SHM_EMPTY) break;
Branch: 4 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 1 11 12
if (st == SHM_EMPTY) break; 1 9 10
if (st == SHM_EMPTY) break; 1 9 10
if (st == SHM_EMPTY) break; 1 9 10
5284 38 ▶ if (st != tag) continue;
Branch: 4 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 11 11
if (st != tag) continue; 3 6 9
if (st != tag) continue; 3 6 9
if (st != tag) continue; 3 6 9
5285 #ifdef SHM_KEY_IS_INT
5286 18 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 6 0 6
if (SHM_KEY_EQ(&nodes[idx], key)) { 6 0 6
if (SHM_KEY_EQ(&nodes[idx], key)) { 6 0 6
5287 #else
5288 11 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 1 branch
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 11 0 11
5289 #endif
5290 29 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 12 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 10 11
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 6 6
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 6 6
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 6 6
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
5291 0 SHM_FN(expire_at)(h, idx);
5292 0 SHM_FN(maybe_shrink)(h);
5293 0 shm_seqlock_write_end(&hdr->seq);
5294 0 shm_rwlock_wrunlock(h);
5295 0 return 0;
5296 }
5297 char ibuf[SHM_INLINE_MAX];
5298 uint32_t cur_len;
5299 29 const char *cur_str = shm_str_ptr(nodes[idx].val_off, nodes[idx].val_len,
5300 29 h->arena, h->hdr->arena_cap, ibuf, &cur_len);
5301 29 ▶ if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len);
Branch: 4 branches
BranchTrueFalseTotal
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 10 1 11
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 5 1 6
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 5 1 6
if (cur_len == expected_len) shm_sig_hold_bytes(h, cur_len); 5 1 6
5302 29 ▶ if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) {
Branch: 8 branches
BranchTrueFalseTotal
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 10 1 11
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 1 9 10
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 5 1 6
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 5 5
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 5 1 6
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 5 5
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 5 1 6
if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) { 0 5 5
5303 5 shm_seqlock_write_end(&hdr->seq);
5304 5 shm_rwlock_wrunlock(h);
5305 5 return 0;
5306 }
5307 24 SHM_FN(arena_reclaim)(h); /* cur_str is done with: the compare is above */
5308 24 uint32_t old_off = nodes[idx].val_off;
5309 24 uint32_t old_lf = nodes[idx].val_len;
5310 24 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len,
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, 0 9 9
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, 0 5 5
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, 0 5 5
if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, 0 5 5
5311 desired_str, desired_len, desired_utf8, idx)) {
5312 0 shm_seqlock_write_end(&hdr->seq);
5313 0 shm_rwlock_wrunlock(h);
5314 0 return 0;
5315 }
5316 24 shm_str_free(hdr, h->arena, old_off, old_lf);
5317 24 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 4 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 2 7 9
if (h->lru_prev) shm_lru_promote(h, idx); 0 5 5
if (h->lru_prev) shm_lru_promote(h, idx); 0 5 5
if (h->lru_prev) shm_lru_promote(h, idx); 0 5 5
5318 24 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
Branch: 12 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 8 9
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 5 5
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 5 5
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 5 5
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
5319 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
5320 24 shm_seqlock_write_end(&hdr->seq);
5321 24 shm_rwlock_wrunlock(h);
5322 24 return 1;
5323 }
5324 }
5325  
5326 4 shm_seqlock_write_end(&hdr->seq);
5327 4 shm_rwlock_wrunlock(h);
5328 4 return 0;
5329 }
5330 #endif /* SHM_VAL_IS_STR */
5331  
5332 523 static inline uint64_t SHM_FN(size)(ShmHandle *h) {
5333 523 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 15 201 216
if (h->shard_handles) { 4 56 60
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 19 19
if (h->shard_handles) { 0 16 16
if (h->shard_handles) { 0 17 17
if (h->shard_handles) { 9 165 174
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 5 5
5334 0 uint64_t total = 0;
5335 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 60 15 75
for (uint32_t i = 0; i < h->num_shards; i++) 16 4 20
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 40 9 49
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5336 0 total += SHM_FN(size)(h->shard_handles[i]);
5337 0 return total;
5338 }
5339 495 return __atomic_load_n(&h->hdr->size, __ATOMIC_ACQUIRE);
5340 }
5341  
5342 0 static inline uint64_t SHM_FN(max_entries)(ShmHandle *h) {
5343 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 50 50
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 2 24 26
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5344 0 uint64_t total = 0;
5345 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 8 2 10
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5346 0 total += SHM_FN(max_entries)(h->shard_handles[i]);
5347 0 return total;
5348 }
5349 /* 2^31 * 3 overflows uint32 */
5350 0 return (uint64_t)h->hdr->max_table_cap * 3 / 4;
5351 }
5352  
5353 0 static inline uint64_t SHM_FN(max_size)(ShmHandle *h) {
5354 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 51 51
if (h->shard_handles) { 0 13 13
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 3 19 22
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5355 0 uint64_t total = 0;
5356 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 10 3 13
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5357 0 total += SHM_FN(max_size)(h->shard_handles[i]);
5358 0 return total;
5359 }
5360 0 return h->hdr->max_size;
5361 }
5362  
5363 0 static inline uint32_t SHM_FN(ttl)(ShmHandle *h) {
5364 0 ▶ if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]);
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 50 50
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 12 12
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 1 8 9
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]); 0 0 0
5365 0 return h->hdr->default_ttl;
5366 }
5367  
5368 /* *out_ttl_remaining (may be NULL): -1 without TTL, 0 permanent, else seconds left. */
5369 0 static int SHM_FN(get_with_ttl)(ShmHandle *h,
5370 #ifdef SHM_KEY_IS_INT
5371 SHM_KEY_INT_TYPE key,
5372 #else
5373 const char *key_str, uint32_t key_len, bool key_utf8,
5374 #endif
5375 #ifdef SHM_VAL_IS_STR
5376 const char **out_str, uint32_t *out_len, bool *out_utf8,
5377 #else
5378 SHM_VAL_INT_TYPE *out_value,
5379 #endif
5380 int64_t *out_ttl_remaining
5381 ) {
5382 #ifdef SHM_KEY_IS_INT
5383 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 3 3
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 54 54
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
5384 #else
5385 0 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
5386 #endif
5387 0 ShmHeader *hdr = h->hdr;
5388 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5389 0 uint8_t *states = h->states;
5390 0 shm_rwlock_rdlock(h);
5391 0 ▶ uint32_t now_ts = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now_ts = h->expires_at ? shm_now() : 0; 2 0 2
uint32_t now_ts = h->expires_at ? shm_now() : 0; 1 0 1
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now_ts = h->expires_at ? shm_now() : 0; 3 0 3
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now_ts = h->expires_at ? shm_now() : 0; 49 5 54
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now_ts = h->expires_at ? shm_now() : 0; 0 0 0
5392  
5393 0 uint32_t mask = hdr->table_cap - 1;
5394 #ifdef SHM_KEY_IS_INT
5395 0 uint32_t hash = SHM_HASH_KEY(key);
5396 #else
5397 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5398 #endif
5399 0 uint32_t pos = hash & mask;
5400 0 uint8_t tag = SHM_MAKE_TAG(hash);
5401  
5402 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 3 0 3
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 54 0 54
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
5403 0 uint32_t idx = (pos + i) & mask;
5404 0 uint8_t st = states[idx];
5405 0 __builtin_prefetch(&nodes[idx], 0, 1);
5406 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5407 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 3 3
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 1 53 54
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
5408 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 3 3
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 53 53
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
5409 #ifdef SHM_KEY_IS_INT
5410 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 3 0 3
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 53 0 53
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
5411 #else
5412 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
5413 #endif
5414 0 ▶ if (SHM_IS_EXPIRED(h, idx, now_ts)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 2 2
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 3 0 3
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 2 1 3
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 2 2
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 49 4 53
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 48 1 49
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 33 15 48
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now_ts)) { 0 0 0
5415 0 shm_rwlock_rdunlock(h);
5416 0 return 0;
5417 }
5418 #ifdef SHM_VAL_IS_STR
5419 {
5420 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 1 1 2
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 3 0 3
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
5421 0 ▶ if (!shm_ensure_copy_buf(h, vl)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl)) { 0 2 2
if (!shm_ensure_copy_buf(h, vl)) { 0 3 3
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
5422 0 shm_rwlock_rdunlock(h);
5423 0 return 0;
5424 }
5425 0 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
5426 0 *out_str = h->copy_buf;
5427 0 *out_len = vl;
5428 0 *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
5429 }
5430 #else
5431 0 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
5432 #endif
5433 0 shm_lru_mark(h, idx);
5434 0 ▶ if (out_ttl_remaining) {
Branch: 10 branches
BranchTrueFalseTotal
if (out_ttl_remaining) { 2 0 2
if (out_ttl_remaining) { 1 0 1
if (out_ttl_remaining) { 0 0 0
if (out_ttl_remaining) { 0 0 0
if (out_ttl_remaining) { 3 0 3
if (out_ttl_remaining) { 0 0 0
if (out_ttl_remaining) { 0 0 0
if (out_ttl_remaining) { 20 0 20
if (out_ttl_remaining) { 0 0 0
if (out_ttl_remaining) { 0 0 0
5435 0 ▶ if (!h->expires_at) *out_ttl_remaining = -1;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) *out_ttl_remaining = -1; 0 2 2
if (!h->expires_at) *out_ttl_remaining = -1; 0 1 1
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
if (!h->expires_at) *out_ttl_remaining = -1; 0 3 3
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
if (!h->expires_at) *out_ttl_remaining = -1; 4 16 20
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
if (!h->expires_at) *out_ttl_remaining = -1; 0 0 0
5436 0 ▶ else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0;
Branch: 10 branches
BranchTrueFalseTotal
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 2 2
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 1 1
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 1 2 3
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 1 15 16
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0; 0 0 0
5437 0 else *out_ttl_remaining = (int64_t)(h->expires_at[idx] - now_ts);
5438 }
5439 0 shm_rwlock_rdunlock(h);
5440 0 return 1;
5441 }
5442 }
5443  
5444 0 shm_rwlock_rdunlock(h);
5445 0 return 0;
5446 }
5447  
5448 /* ---- TTL remaining for a key (-1 = not found/expired, 0 = permanent) ---- */
5449  
5450 0 static int64_t SHM_FN(ttl_remaining)(ShmHandle *h,
5451 #ifdef SHM_KEY_IS_INT
5452 SHM_KEY_INT_TYPE key
5453 #else
5454 const char *key_str, uint32_t key_len, bool key_utf8
5455 #endif
5456 ) {
5457 #ifdef SHM_KEY_IS_INT
5458 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 67 67
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 4 4
5459 #else
5460 13 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 8 8
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
5461 #endif
5462 0 ▶ if (!h->expires_at) return -1;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) return -1; 0 8 8
if (!h->expires_at) return -1; 0 2 2
if (!h->expires_at) return -1; 0 2 2
if (!h->expires_at) return -1; 0 1 1
if (!h->expires_at) return -1; 0 1 1
if (!h->expires_at) return -1; 0 1 1
if (!h->expires_at) return -1; 0 0 0
if (!h->expires_at) return -1; 1 66 67
if (!h->expires_at) return -1; 0 0 0
if (!h->expires_at) return -1; 0 4 4
5463  
5464 0 ShmHeader *hdr = h->hdr;
5465 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5466 0 uint8_t *states = h->states;
5467  
5468 0 shm_rwlock_rdlock(h);
5469  
5470 0 uint32_t mask = hdr->table_cap - 1;
5471 #ifdef SHM_KEY_IS_INT
5472 0 uint32_t hash = SHM_HASH_KEY(key);
5473 #else
5474 13 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5475 #endif
5476 0 uint32_t pos = hash & mask;
5477 0 uint8_t tag = SHM_MAKE_TAG(hash);
5478  
5479 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 9 0 9
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 66 0 66
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 4 0 4
5480 0 uint32_t idx = (pos + i) & mask;
5481 0 uint8_t st = states[idx];
5482 0 __builtin_prefetch(&nodes[idx], 0, 1);
5483 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5484 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 1 8 9
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 1 65 66
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 4 4
5485 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 1 7 8
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 65 65
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 4 4
5486 #ifdef SHM_KEY_IS_INT
5487 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 65 0 65
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 4 0 4
5488 #else
5489 12 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 7 0 7
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 2 0 2
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
5490 #endif
5491 0 uint32_t exp = h->expires_at[idx];
5492 0 ▶ if (exp == 0) {
Branch: 10 branches
BranchTrueFalseTotal
if (exp == 0) { 0 7 7
if (exp == 0) { 0 2 2
if (exp == 0) { 1 1 2
if (exp == 0) { 1 0 1
if (exp == 0) { 1 0 1
if (exp == 0) { 1 0 1
if (exp == 0) { 0 0 0
if (exp == 0) { 5 60 65
if (exp == 0) { 0 0 0
if (exp == 0) { 1 3 4
5493 0 shm_rwlock_rdunlock(h);
5494 0 return 0;
5495 }
5496 0 uint32_t now = shm_now();
5497 0 ▶ if (now >= exp) {
Branch: 10 branches
BranchTrueFalseTotal
if (now >= exp) { 0 7 7
if (now >= exp) { 0 2 2
if (now >= exp) { 0 1 1
if (now >= exp) { 0 0 0
if (now >= exp) { 0 0 0
if (now >= exp) { 0 0 0
if (now >= exp) { 0 0 0
if (now >= exp) { 31 29 60
if (now >= exp) { 0 0 0
if (now >= exp) { 0 3 3
5498 0 shm_rwlock_rdunlock(h);
5499 0 return -1;
5500 }
5501 0 int64_t remaining = (int64_t)(exp - now);
5502 0 shm_rwlock_rdunlock(h);
5503 0 return remaining;
5504 }
5505 }
5506  
5507 0 shm_rwlock_rdunlock(h);
5508 0 return -1;
5509 }
5510  
5511 /* expires_at changes under the seqlock: get() reads it lock-free. */
5512  
5513 0 static int SHM_FN(persist)(ShmHandle *h,
5514 #ifdef SHM_KEY_IS_INT
5515 SHM_KEY_INT_TYPE key
5516 #else
5517 const char *key_str, uint32_t key_len, bool key_utf8
5518 #endif
5519 ) {
5520 #ifdef SHM_KEY_IS_INT
5521 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 2 2
SHM_SHARD_DISPATCH(h, key); 0 1 1
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 1 1
5522 #else
5523 0 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
5524 #endif
5525 0 ▶ if (!h->expires_at) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) return 0; 0 0 0
if (!h->expires_at) return 0; 0 0 0
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 2 2
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 0 0
if (!h->expires_at) return 0; 0 0 0
if (!h->expires_at) return 0; 0 0 0
if (!h->expires_at) return 0; 0 1 1
5526  
5527 0 ShmHeader *hdr = h->hdr;
5528 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5529 0 uint8_t *states = h->states;
5530 0 shm_rwlock_wrlock(h);
5531 0 uint32_t now = shm_now();
5532  
5533 0 uint32_t mask = hdr->table_cap - 1;
5534 #ifdef SHM_KEY_IS_INT
5535 0 uint32_t hash = SHM_HASH_KEY(key);
5536 #else
5537 0 shm_sig_hold_bytes(h, key_len);
5538 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5539 #endif
5540 0 uint32_t pos = hash & mask;
5541 0 uint8_t tag = SHM_MAKE_TAG(hash);
5542  
5543 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
5544 0 uint32_t idx = (pos + i) & mask;
5545 0 uint8_t st = states[idx];
5546 0 __builtin_prefetch(&nodes[idx], 0, 1);
5547 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5548 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 2 2
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 1 1
5549 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 2 2
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 1 1
5550 #ifdef SHM_KEY_IS_INT
5551 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 2 0 2
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 1 0 1
5552 #else
5553 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
5554 #endif
5555 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now)) { 2 0 2
if (SHM_IS_EXPIRED(h, idx, now)) { 0 2 2
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
5556 0 shm_seqlock_write_begin(&hdr->seq);
5557 0 SHM_FN(expire_at)(h, idx);
5558 0 SHM_FN(maybe_shrink)(h);
5559 0 shm_seqlock_write_end(&hdr->seq);
5560 0 shm_rwlock_wrunlock(h);
5561 0 return 0;
5562 }
5563 0 shm_seqlock_write_begin(&hdr->seq);
5564 0 h->expires_at[idx] = 0;
5565 0 shm_seqlock_write_end(&hdr->seq);
5566 0 shm_rwlock_wrunlock(h);
5567 0 return 1;
5568 }
5569 }
5570  
5571 0 shm_rwlock_wrunlock(h);
5572 0 return 0;
5573 }
5574  
5575 307 static int SHM_FN(set_ttl)(ShmHandle *h,
5576 #ifdef SHM_KEY_IS_INT
5577 SHM_KEY_INT_TYPE key,
5578 #else
5579 const char *key_str, uint32_t key_len, bool key_utf8,
5580 #endif
5581 uint32_t ttl_sec
5582 ) {
5583 #ifdef SHM_KEY_IS_INT
5584 184 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 31 31
SHM_SHARD_DISPATCH(h, key); 0 30 30
SHM_SHARD_DISPATCH(h, key); 0 30 30
SHM_SHARD_DISPATCH(h, key); 0 32 32
SHM_SHARD_DISPATCH(h, key); 0 30 30
SHM_SHARD_DISPATCH(h, key); 0 31 31
5585 #else
5586 123 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 30 30
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 31 31
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 31 31
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 31 31
5587 #endif
5588 307 ▶ if (!h->expires_at) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) return 0; 0 30 30
if (!h->expires_at) return 0; 0 31 31
if (!h->expires_at) return 0; 0 31 31
if (!h->expires_at) return 0; 0 31 31
if (!h->expires_at) return 0; 0 31 31
if (!h->expires_at) return 0; 0 30 30
if (!h->expires_at) return 0; 0 30 30
if (!h->expires_at) return 0; 0 32 32
if (!h->expires_at) return 0; 0 30 30
if (!h->expires_at) return 0; 0 31 31
5589  
5590 307 ShmHeader *hdr = h->hdr;
5591 307 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5592 307 uint8_t *states = h->states;
5593 307 shm_rwlock_wrlock(h);
5594 307 uint32_t now = shm_now();
5595  
5596 307 uint32_t mask = hdr->table_cap - 1;
5597 #ifdef SHM_KEY_IS_INT
5598 184 uint32_t hash = SHM_HASH_KEY(key);
5599 #else
5600 123 shm_sig_hold_bytes(h, key_len);
5601 123 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5602 #endif
5603 307 uint32_t pos = hash & mask;
5604 307 uint8_t tag = SHM_MAKE_TAG(hash);
5605  
5606 477 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 38 0 38
for (uint32_t i = 0; i <= mask; i++) { 39 0 39
for (uint32_t i = 0; i <= mask; i++) { 39 0 39
for (uint32_t i = 0; i <= mask; i++) { 39 0 39
for (uint32_t i = 0; i <= mask; i++) { 48 0 48
for (uint32_t i = 0; i <= mask; i++) { 47 0 47
for (uint32_t i = 0; i <= mask; i++) { 47 0 47
for (uint32_t i = 0; i <= mask; i++) { 85 0 85
for (uint32_t i = 0; i <= mask; i++) { 47 0 47
for (uint32_t i = 0; i <= mask; i++) { 48 0 48
5607 477 uint32_t idx = (pos + i) & mask;
5608 477 uint8_t st = states[idx];
5609 477 __builtin_prefetch(&nodes[idx], 0, 1);
5610 477 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5611 477 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 0 38 38
if (st == SHM_EMPTY) break; 0 39 39
if (st == SHM_EMPTY) break; 0 39 39
if (st == SHM_EMPTY) break; 0 39 39
if (st == SHM_EMPTY) break; 0 48 48
if (st == SHM_EMPTY) break; 0 47 47
if (st == SHM_EMPTY) break; 0 47 47
if (st == SHM_EMPTY) break; 0 85 85
if (st == SHM_EMPTY) break; 0 47 47
if (st == SHM_EMPTY) break; 0 48 48
5612 477 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 8 30 38
if (st != tag) continue; 8 31 39
if (st != tag) continue; 8 31 39
if (st != tag) continue; 8 31 39
if (st != tag) continue; 17 31 48
if (st != tag) continue; 17 30 47
if (st != tag) continue; 17 30 47
if (st != tag) continue; 53 32 85
if (st != tag) continue; 17 30 47
if (st != tag) continue; 17 31 48
5613 #ifdef SHM_KEY_IS_INT
5614 184 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 31 0 31
if (SHM_KEY_EQ(&nodes[idx], key)) { 30 0 30
if (SHM_KEY_EQ(&nodes[idx], key)) { 30 0 30
if (SHM_KEY_EQ(&nodes[idx], key)) { 32 0 32
if (SHM_KEY_EQ(&nodes[idx], key)) { 30 0 30
if (SHM_KEY_EQ(&nodes[idx], key)) { 31 0 31
5615 #else
5616 123 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 30 0 30
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 31 0 31
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 31 0 31
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 31 0 31
5617 #endif
5618 307 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 0 31 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 30 1 31
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 0 31 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 0 31 31
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 32 0 32
if (SHM_IS_EXPIRED(h, idx, now)) { 30 2 32
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 30 0 30
if (SHM_IS_EXPIRED(h, idx, now)) { 0 30 30
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 31 0 31
if (SHM_IS_EXPIRED(h, idx, now)) { 0 31 31
5619 0 shm_seqlock_write_begin(&hdr->seq);
5620 0 SHM_FN(expire_at)(h, idx);
5621 0 SHM_FN(maybe_shrink)(h);
5622 0 shm_seqlock_write_end(&hdr->seq);
5623 0 shm_rwlock_wrunlock(h);
5624 0 return 0;
5625 }
5626 307 shm_seqlock_write_begin(&hdr->seq);
5627 307 ▶ if (ttl_sec == 0)
Branch: 10 branches
BranchTrueFalseTotal
if (ttl_sec == 0) 0 30 30
if (ttl_sec == 0) 0 31 31
if (ttl_sec == 0) 0 31 31
if (ttl_sec == 0) 0 31 31
if (ttl_sec == 0) 0 31 31
if (ttl_sec == 0) 0 30 30
if (ttl_sec == 0) 0 30 30
if (ttl_sec == 0) 0 32 32
if (ttl_sec == 0) 0 30 30
if (ttl_sec == 0) 0 31 31
5628 0 h->expires_at[idx] = 0;
5629 else
5630 307 h->expires_at[idx] = shm_expiry_ts(ttl_sec);
5631 307 shm_seqlock_write_end(&hdr->seq);
5632 307 shm_rwlock_wrunlock(h);
5633 307 return 1;
5634 }
5635 }
5636  
5637 0 shm_rwlock_wrunlock(h);
5638 0 return 0;
5639 }
5640  
5641 274 static inline uint64_t SHM_FN(capacity)(ShmHandle *h) {
5642 274 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 79 79
if (h->shard_handles) { 0 48 48
if (h->shard_handles) { 0 3 3
if (h->shard_handles) { 0 3 3
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 5 115 120
if (h->shard_handles) { 0 3 3
if (h->shard_handles) { 0 3 3
5643 0 uint64_t total = 0;
5644 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 9 5 14
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5645 0 total += SHM_FN(capacity)(h->shard_handles[i]);
5646 0 return total;
5647 }
5648 269 return __atomic_load_n(&h->hdr->table_cap, __ATOMIC_ACQUIRE);
5649 }
5650  
5651 0 static inline uint64_t SHM_FN(tombstones)(ShmHandle *h) {
5652 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 53 53
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 2 24 26
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5653 0 uint64_t total = 0;
5654 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 6 2 8
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5655 0 total += SHM_FN(tombstones)(h->shard_handles[i]);
5656 0 return total;
5657 }
5658 0 return __atomic_load_n(&h->hdr->tombstones, __ATOMIC_ACQUIRE);
5659 }
5660  
5661 0 static inline size_t SHM_FN(mmap_size)(ShmHandle *h) {
5662 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 50 50
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 2 17 19
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5663 0 size_t total = 0;
5664 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 8 2 10
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5665 0 total += SHM_FN(mmap_size)(h->shard_handles[i]);
5666 0 return total;
5667 }
5668 0 return h->mmap_size;
5669 }
5670  
5671 /* *done_out: the shared flush_cursor completed a cycle. */
5672 2562 static uint32_t SHM_FN(flush_expired_partial)(ShmHandle *h, uint32_t limit, int *done_out) {
5673 2562 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 16 16
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 1172 1262 2434
if (h->shard_handles) { 0 14 14
if (h->shard_handles) { 0 14 14
5674 /* A shard that finished its cycle waits for the others. */
5675 0 uint32_t total = 0;
5676 0 int all_done = 1;
5677 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 2344 1172 3516
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
5678 0 ShmHandle *s = h->shard_handles[i];
5679 0 ▶ if (!s->flush_done) {
Branch: 10 branches
BranchTrueFalseTotal
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 1182 1162 2344
if (!s->flush_done) { 0 0 0
if (!s->flush_done) { 0 0 0
5680 0 int done = 0;
5681 0 total += SHM_FN(flush_expired_partial)(s, limit, &done);
5682 0 s->flush_done = (uint8_t)done;
5683 }
5684 0 all_done &= s->flush_done;
5685 }
5686 0 ▶ if (all_done)
Branch: 10 branches
BranchTrueFalseTotal
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 0 0 0
if (all_done) 2 1170 1172
if (all_done) 0 0 0
if (all_done) 0 0 0
5687 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0;
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 4 2 6
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) h->shard_handles[i]->flush_done = 0; 0 0 0
5688 0 ▶ if (done_out) *done_out = all_done;
Branch: 10 branches
BranchTrueFalseTotal
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 1172 0 1172
if (done_out) *done_out = all_done; 0 0 0
if (done_out) *done_out = all_done; 0 0 0
5689 0 return total;
5690 }
5691 1390 ▶ if (!h->expires_at) {
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 16 16
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 1262 1262
if (!h->expires_at) { 0 14 14
if (!h->expires_at) { 0 14 14
5692 0 ▶ if (done_out) *done_out = 1;
Branch: 10 branches
BranchTrueFalseTotal
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
if (done_out) *done_out = 1; 0 0 0
5693 0 return 0;
5694 }
5695 1390 ▶ if (done_out) *done_out = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 16 0 16
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 1262 0 1262
if (done_out) *done_out = 0; 14 0 14
if (done_out) *done_out = 0; 14 0 14
5696  
5697 1390 ShmHeader *hdr = h->hdr;
5698 1390 uint8_t *states = h->states;
5699 1390 uint32_t flushed = 0;
5700  
5701 1390 shm_rwlock_wrlock(h);
5702 1390 uint32_t now = shm_now();
5703 1390 shm_seqlock_write_begin(&hdr->seq);
5704  
5705 1390 uint32_t cap = hdr->table_cap;
5706 1390 uint32_t start = hdr->flush_cursor;
5707 1390 ▶ if (start >= cap) start = 0;
Branch: 10 branches
BranchTrueFalseTotal
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 16 16
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 1262 1262
if (start >= cap) start = 0; 0 14 14
if (start >= cap) start = 0; 0 14 14
5708 /* Behind where this handle left the cursor, another flusher ended the cycle. */
5709 1390 ▶ int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen;
Branch: 20 branches
BranchTrueFalseTotal
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 12 4 16
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 12 12
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 1248 14 1262
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 2 1246 1248
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 11 3 14
int lapped = hdr->table_gen == h->flush_gen && start < h->flush_seen; 0 11 11
5710 1390 ▶ if (limit == 0) limit = 1;
Branch: 10 branches
BranchTrueFalseTotal
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 16 16
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 1262 1262
if (limit == 0) limit = 1; 0 14 14
if (limit == 0) limit = 1; 0 14 14
5711 1390 ▶ if (limit >= cap) { start = 0; limit = cap; }
Branch: 10 branches
BranchTrueFalseTotal
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 2 14 16
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 49 1213 1262
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
if (limit >= cap) { start = 0; limit = cap; } 1 13 14
5712 1331 ▶ else if (limit > cap - start) limit = cap - start;
Branch: 10 branches
BranchTrueFalseTotal
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 14 14
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 7 1206 1213
else if (limit > cap - start) limit = cap - start; 0 13 13
else if (limit > cap - start) limit = cap - start; 0 13 13
5713 1390 shm_sig_hold(h, limit);
5714  
5715 63218 ▶ for (uint32_t i = start; i < start + limit; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 1696 16 1712
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 58852 1262 60114
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
for (uint32_t i = start; i < start + limit; i++) { 160 14 174
5716 61828 ▶ if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 99 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 816 880 1696
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 816 0 816
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 561 255 816
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 99 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 99 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 62 98 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 1 62
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 62 98 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 1 62
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 62 98 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 1 62
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 24486 34366 58852
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 20379 4107 24486
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 20276 103 20379
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 62 98 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 1 62
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 62 98 160
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 1 62
if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) { 61 0 61
5717 shm_sig_check(h);
5718 21325 SHM_FN(expire_at)(h, i);
5719 21325 flushed++;
5720 }
5721 }
5722  
5723 1390 int done = start + limit == cap;
5724 1390 ▶ hdr->flush_cursor = done ? 0 : start + limit;
Branch: 10 branches
BranchTrueFalseTotal
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 12 4 16
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 1201 61 1262
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
hdr->flush_cursor = done ? 0 : start + limit; 11 3 14
5725  
5726 1390 ▶ if (done_out) *done_out = done || lapped;
Branch: 30 branches
BranchTrueFalseTotal
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 16 0 16
if (done_out) *done_out = done || lapped; 12 4 16
if (done_out) *done_out = done || lapped; 0 12 12
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 1262 0 1262
if (done_out) *done_out = done || lapped; 1201 61 1262
if (done_out) *done_out = done || lapped; 2 1199 1201
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
if (done_out) *done_out = done || lapped; 14 0 14
if (done_out) *done_out = done || lapped; 11 3 14
if (done_out) *done_out = done || lapped; 0 11 11
5727  
5728 /* Shrink only at a cycle's end: a rehash could move entries behind the cursor. */
5729 1390 ▶ if (flushed) h->flush_freed = 1;
Branch: 10 branches
BranchTrueFalseTotal
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 14 2 16
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 104 1158 1262
if (flushed) h->flush_freed = 1; 13 1 14
if (flushed) h->flush_freed = 1; 13 1 14
5730 1390 ▶ if (done && h->flush_freed) {
Branch: 20 branches
BranchTrueFalseTotal
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 4 12 16
if (done && h->flush_freed) { 3 1 4
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 61 1201 1262
if (done && h->flush_freed) { 42 19 61
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
if (done && h->flush_freed) { 3 11 14
if (done && h->flush_freed) { 2 1 3
5731 61 h->flush_freed = 0;
5732 61 SHM_FN(maybe_shrink)(h);
5733 }
5734 1390 h->flush_seen = hdr->flush_cursor;
5735 1390 h->flush_gen = hdr->table_gen;
5736  
5737 1390 shm_seqlock_write_end(&hdr->seq);
5738 1390 shm_rwlock_wrunlock(h);
5739 1390 return flushed;
5740 }
5741  
5742 15 static uint32_t SHM_FN(flush_expired)(ShmHandle *h) {
5743 15 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
5744 0 uint32_t total = 0;
5745 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5746 0 total += SHM_FN(flush_expired)(h->shard_handles[i]);
5747 0 return total;
5748 }
5749 15 ▶ if (!h->expires_at) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 2 2
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 5 5
if (!h->expires_at) return 0; 0 1 1
if (!h->expires_at) return 0; 0 1 1
5750 int done;
5751 15 return SHM_FN(flush_expired_partial)(h, UINT32_MAX, &done);
5752 }
5753  
5754 0 static int SHM_FN(touch)(ShmHandle *h,
5755 #ifdef SHM_KEY_IS_INT
5756 SHM_KEY_INT_TYPE key
5757 #else
5758 const char *key_str, uint32_t key_len, bool key_utf8
5759 #endif
5760 ) {
5761 #ifdef SHM_KEY_IS_INT
5762 0 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 25 25
SHM_SHARD_DISPATCH(h, key); 0 0 0
SHM_SHARD_DISPATCH(h, key); 0 0 0
5763 #else
5764 0 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 1 1
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 0 0
5765 #endif
5766 0 ▶ if (!h->lru_prev && !h->expires_at) return 0;
Branch: 20 branches
BranchTrueFalseTotal
if (!h->lru_prev && !h->expires_at) return 0; 2 0 2
if (!h->lru_prev && !h->expires_at) return 0; 0 2 2
if (!h->lru_prev && !h->expires_at) return 0; 1 0 1
if (!h->lru_prev && !h->expires_at) return 0; 0 1 1
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 7 18 25
if (!h->lru_prev && !h->expires_at) return 0; 1 6 7
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
if (!h->lru_prev && !h->expires_at) return 0; 0 0 0
5767  
5768 0 ShmHeader *hdr = h->hdr;
5769 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5770 0 uint8_t *states = h->states;
5771 0 shm_rwlock_wrlock(h);
5772 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 2 0 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 0 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 6 18 24
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
5773  
5774 0 uint32_t mask = hdr->table_cap - 1;
5775 #ifdef SHM_KEY_IS_INT
5776 0 uint32_t hash = SHM_HASH_KEY(key);
5777 #else
5778 0 shm_sig_hold_bytes(h, key_len);
5779 0 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
5780 #endif
5781 0 uint32_t pos = hash & mask;
5782 0 uint8_t tag = SHM_MAKE_TAG(hash);
5783  
5784 0 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 2 0 2
for (uint32_t i = 0; i <= mask; i++) { 1 0 1
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 24 0 24
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
for (uint32_t i = 0; i <= mask; i++) { 0 0 0
5785 0 uint32_t idx = (pos + i) & mask;
5786 0 uint8_t st = states[idx];
5787 0 __builtin_prefetch(&nodes[idx], 0, 1);
5788 0 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
5789 0 ▶ if (st == SHM_EMPTY) break;
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) break; 1 1 2
if (st == SHM_EMPTY) break; 0 1 1
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 1 23 24
if (st == SHM_EMPTY) break; 0 0 0
if (st == SHM_EMPTY) break; 0 0 0
5790 0 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 1 1
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 23 23
if (st != tag) continue; 0 0 0
if (st != tag) continue; 0 0 0
5791 #ifdef SHM_KEY_IS_INT
5792 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 23 0 23
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
5793 #else
5794 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
5795 #endif
5796 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 5 18 23
if (SHM_IS_EXPIRED(h, idx, now)) { 4 1 5
if (SHM_IS_EXPIRED(h, idx, now)) { 1 3 4
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
5797 0 shm_seqlock_write_begin(&hdr->seq);
5798 0 SHM_FN(expire_at)(h, idx);
5799 0 SHM_FN(maybe_shrink)(h);
5800 0 shm_seqlock_write_end(&hdr->seq);
5801 0 shm_rwlock_wrunlock(h);
5802 0 return 0;
5803 }
5804 0 shm_seqlock_write_begin(&hdr->seq);
5805 0 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 18 4 22
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
5806 0 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) {
Branch: 30 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 4 18 22
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 4 0 4
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 3 1 4
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) { 0 0 0
5807 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
5808 }
5809 0 shm_seqlock_write_end(&hdr->seq);
5810 0 shm_rwlock_wrunlock(h);
5811 0 return 1;
5812 }
5813 }
5814  
5815 0 shm_rwlock_wrunlock(h);
5816 0 return 0;
5817 }
5818  
5819 35 static int SHM_FN(reserve)(ShmHandle *h, uint32_t target) {
5820 35 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 5 5
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 0 19 19
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
5821 0 int ok = 1;
5822 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5823 0 ok &= SHM_FN(reserve)(h->shard_handles[i], target);
5824 0 return ok;
5825 }
5826 35 ShmHeader *hdr = h->hdr;
5827 /* The ceiling matches max_entries(). */
5828 35 ▶ if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 5 5
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 1 1
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 1 1
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 1 1
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 2 2
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 2 2
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 2 2
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 5 14 19
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 1 1
if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0; 0 1 1
5829 30 uint32_t needed = shm_next_pow2(target + target / 3 + 1);
5830 /* The top of the range rounds above max_table_cap but still fits in it. */
5831 30 ▶ if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap;
Branch: 20 branches
BranchTrueFalseTotal
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 5 0 5
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 5 5
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 1 0 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 1 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 1 0 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 1 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 1 0 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 1 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 2 0 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 2 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 2 0 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 2 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 2 0 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 2 2
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 14 0 14
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 3 11 14
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 1 0 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 1 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 1 0 1
if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap; 0 1 1
5832 30 ▶ if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP;
Branch: 10 branches
BranchTrueFalseTotal
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 5 5
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 1 1
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 1 1
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 1 1
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 2 2
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 2 2
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 2 2
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 14 14
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 1 1
if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP; 0 1 1
5833 30 ▶ if (needed <= hdr->table_cap) return 1;
Branch: 10 branches
BranchTrueFalseTotal
if (needed <= hdr->table_cap) return 1; 0 5 5
if (needed <= hdr->table_cap) return 1; 0 1 1
if (needed <= hdr->table_cap) return 1; 0 1 1
if (needed <= hdr->table_cap) return 1; 0 1 1
if (needed <= hdr->table_cap) return 1; 0 2 2
if (needed <= hdr->table_cap) return 1; 0 2 2
if (needed <= hdr->table_cap) return 1; 0 2 2
if (needed <= hdr->table_cap) return 1; 1 13 14
if (needed <= hdr->table_cap) return 1; 0 1 1
if (needed <= hdr->table_cap) return 1; 0 1 1
5834  
5835 29 shm_rwlock_wrlock(h);
5836 29 shm_seqlock_write_begin(&hdr->seq);
5837 29 int ok = 1;
5838 29 ▶ if (needed > hdr->table_cap)
Branch: 10 branches
BranchTrueFalseTotal
if (needed > hdr->table_cap) 5 0 5
if (needed > hdr->table_cap) 1 0 1
if (needed > hdr->table_cap) 1 0 1
if (needed > hdr->table_cap) 1 0 1
if (needed > hdr->table_cap) 2 0 2
if (needed > hdr->table_cap) 2 0 2
if (needed > hdr->table_cap) 2 0 2
if (needed > hdr->table_cap) 13 0 13
if (needed > hdr->table_cap) 1 0 1
if (needed > hdr->table_cap) 1 0 1
5839 29 ok = SHM_FN(resize)(h, needed);
5840 29 shm_seqlock_write_end(&hdr->seq);
5841 29 shm_rwlock_wrunlock(h);
5842 29 return ok;
5843 }
5844  
5845 0 static inline uint64_t SHM_FN(stat_evictions)(ShmHandle *h) {
5846 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 51 51
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 1 19 20
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5847 0 uint64_t total = 0;
5848 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5849 0 total += SHM_FN(stat_evictions)(h->shard_handles[i]);
5850 0 return (uint64_t)total;
5851 }
5852 0 return __atomic_load_n(&h->hdr->stat_evictions, __ATOMIC_RELAXED);
5853 }
5854  
5855 0 static inline uint64_t SHM_FN(stat_expired)(ShmHandle *h) {
5856 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 56 56
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 0 6 6
if (h->shard_handles) { 1 15 16
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5857 0 uint64_t total = 0;
5858 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5859 0 total += SHM_FN(stat_expired)(h->shard_handles[i]);
5860 0 return (uint64_t)total;
5861 }
5862 0 return __atomic_load_n(&h->hdr->stat_expired, __ATOMIC_RELAXED);
5863 }
5864  
5865 0 static inline uint64_t SHM_FN(stat_recoveries)(ShmHandle *h) {
5866 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 50 50
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 2 13 15
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5867 0 uint64_t total = 0;
5868 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 6 2 8
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5869 0 total += SHM_FN(stat_recoveries)(h->shard_handles[i]);
5870 0 return total;
5871 }
5872 0 return __atomic_load_n(&h->hdr->stat_recoveries, __ATOMIC_RELAXED);
5873 }
5874  
5875 0 static uint64_t SHM_FN(compact)(ShmHandle *h) {
5876 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 20 20
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5877 0 uint64_t total = 0;
5878 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5879 0 total += SHM_FN(compact)(h->shard_handles[i]);
5880 0 return total;
5881 }
5882 0 ▶ if (!h->arena) return 0;
Branch: 10 branches
BranchTrueFalseTotal
if (!h->arena) return 0; 0 20 20
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 1 0 1
if (!h->arena) return 0; 0 0 0
if (!h->arena) return 0; 0 0 0
5883 0 uint64_t freed = 0;
5884 0 shm_rwlock_wrlock(h);
5885 0 shm_seqlock_write_begin(&h->hdr->seq);
5886 #if SHM_HAS_ARENA
5887 0 freed = SHM_FN(arena_compact)(h, 0);
5888 #endif
5889 0 h->arena_failed = 0;
5890 0 h->compact_backoff = 0;
5891 0 h->compact_period = 0;
5892 0 shm_seqlock_write_end(&h->hdr->seq);
5893 0 shm_rwlock_wrunlock(h);
5894 0 return freed;
5895 }
5896  
5897 0 static inline uint64_t SHM_FN(arena_used)(ShmHandle *h) {
5898 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 1424 1424
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 1 11 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5899 0 uint64_t total = 0;
5900 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5901 0 total += SHM_FN(arena_used)(h->shard_handles[i]);
5902 0 return (uint64_t)total;
5903 }
5904 0 ▶ return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0;
Branch: 10 branches
BranchTrueFalseTotal
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 1424 0 1424
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 12 0 12
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 2 0 2
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 11 11
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0; 0 0 0
5905 }
5906  
5907 0 static inline uint64_t SHM_FN(arena_cap)(ShmHandle *h) {
5908 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 1 1421 1422
if (h->shard_handles) { 0 12 12
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 1 12 13
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5909 0 uint64_t total = 0;
5910 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5911 0 total += SHM_FN(arena_cap)(h->shard_handles[i]);
5912 0 return (uint64_t)total;
5913 }
5914 0 ▶ return h->arena ? h->hdr->arena_cap : 0;
Branch: 10 branches
BranchTrueFalseTotal
return h->arena ? h->hdr->arena_cap : 0; 1421 0 1421
return h->arena ? h->hdr->arena_cap : 0; 12 0 12
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 12 12
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
return h->arena ? h->hdr->arena_cap : 0; 0 0 0
5915 }
5916  
5917 0 static void SHM_FN(clear)(ShmHandle *h) {
5918 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 2 2
if (h->shard_handles) { 1 4 5
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 3 3
if (h->shard_handles) { 0 1 1
if (h->shard_handles) { 0 1 1
5919 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++)
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 4 1 5
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) 0 0 0
5920 0 SHM_FN(clear)(h->shard_handles[i]);
5921 /* Or a later each() resumes mid-set and skips the shards below. */
5922 0 h->shard_iter = 0;
5923 0 return;
5924 }
5925 0 ShmHeader *hdr = h->hdr;
5926  
5927 0 shm_rwlock_wrlock(h);
5928 0 shm_sig_hold(h, hdr->table_cap);
5929 0 shm_seqlock_write_begin(&hdr->seq);
5930  
5931 0 hdr->rz_seq = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
5932 0 __atomic_signal_fence(__ATOMIC_SEQ_CST);
5933 0 __atomic_store_n(&hdr->rz_phase, SHM_RZ_CLEAR, __ATOMIC_RELEASE);
5934 0 __atomic_signal_fence(__ATOMIC_SEQ_CST);
5935 0 shm_clear_run(h);
5936  
5937 0 shm_seqlock_write_end(&hdr->seq);
5938 0 shm_rwlock_wrunlock(h);
5939  
5940 0 h->iter_pos = 0;
5941 0 ▶ if (h->iter_active) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->iter_active) { 0 2 2
if (h->iter_active) { 1 3 4
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 3 3
if (h->iter_active) { 0 1 1
if (h->iter_active) { 0 1 1
5942 0 h->iter_active = 0;
5943 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 1 0 1
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
5944 }
5945 0 h->deferred = 0;
5946 }
5947  
5948 0 static inline void SHM_FN(iter_reset)(ShmHandle *h) {
5949 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 1 3 4
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
5950 0 ▶ for (uint32_t i = 0; i < h->num_shards; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 1 1 2
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
for (uint32_t i = 0; i < h->num_shards; i++) { 0 0 0
5951 0 SHM_FN(iter_reset)(h->shard_handles[i]);
5952 /* Deferred work is on the shards, never the dispatcher. */
5953 0 SHM_FN(flush_deferred)(h->shard_handles[i]);
5954 }
5955 0 h->shard_iter = 0;
5956 0 return;
5957 }
5958 0 ▶ if (h->iter_active) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
if (h->iter_active) { 3 0 3
if (h->iter_active) { 0 0 0
if (h->iter_active) { 0 0 0
5959 0 h->iter_active = 0;
5960 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 3 0 3
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
5961 }
5962 0 h->iter_pos = 0;
5963 }
5964  
5965 128 static int SHM_FN(get_or_set)(ShmHandle *h,
5966 #ifdef SHM_KEY_IS_INT
5967 SHM_KEY_INT_TYPE key,
5968 #else
5969 const char *key_str, uint32_t key_len, bool key_utf8,
5970 #endif
5971 #ifdef SHM_VAL_IS_STR
5972 const char *def_str, uint32_t def_len, bool def_utf8,
5973 const char **out_str, uint32_t *out_len, bool *out_utf8
5974 #else
5975 SHM_VAL_INT_TYPE def_value,
5976 SHM_VAL_INT_TYPE *out_value
5977 #endif
5978 ) {
5979 #ifdef SHM_KEY_IS_INT
5980 58 ▶ SHM_SHARD_DISPATCH(h, key);
Branch: 6 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key); 0 3 3
SHM_SHARD_DISPATCH(h, key); 0 3 3
SHM_SHARD_DISPATCH(h, key); 0 5 5
SHM_SHARD_DISPATCH(h, key); 0 39 39
SHM_SHARD_DISPATCH(h, key); 0 4 4
SHM_SHARD_DISPATCH(h, key); 0 4 4
5981 #else
5982 70 ▶ SHM_SHARD_DISPATCH(h, key_str, key_len);
Branch: 4 branches
BranchTrueFalseTotal
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 61 61
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 3 3
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 2 2
SHM_SHARD_DISPATCH(h, key_str, key_len); 0 4 4
5983 #endif
5984 128 ShmHeader *hdr = h->hdr;
5985 128 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
5986 128 uint8_t *states = h->states;
5987  
5988 /* No LRU or TTL: a hit changes nothing, so get() answers it lock-free. */
5989 128 ▶ if (!h->lru_prev && !h->expires_at) {
Branch: 20 branches
BranchTrueFalseTotal
if (!h->lru_prev && !h->expires_at) { 59 2 61
if (!h->lru_prev && !h->expires_at) { 56 3 59
if (!h->lru_prev && !h->expires_at) { 3 0 3
if (!h->lru_prev && !h->expires_at) { 2 1 3
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 2 0 2
if (!h->lru_prev && !h->expires_at) { 4 0 4
if (!h->lru_prev && !h->expires_at) { 4 0 4
if (!h->lru_prev && !h->expires_at) { 3 0 3
if (!h->lru_prev && !h->expires_at) { 2 1 3
if (!h->lru_prev && !h->expires_at) { 3 0 3
if (!h->lru_prev && !h->expires_at) { 2 1 3
if (!h->lru_prev && !h->expires_at) { 5 0 5
if (!h->lru_prev && !h->expires_at) { 4 1 5
if (!h->lru_prev && !h->expires_at) { 23 16 39
if (!h->lru_prev && !h->expires_at) { 22 1 23
if (!h->lru_prev && !h->expires_at) { 4 0 4
if (!h->lru_prev && !h->expires_at) { 4 0 4
if (!h->lru_prev && !h->expires_at) { 4 0 4
if (!h->lru_prev && !h->expires_at) { 4 0 4
5990 #if defined(SHM_KEY_IS_INT) && defined(SHM_VAL_IS_STR)
5991 8 ▶ if (SHM_FN(get)(h, key, out_str, out_len, out_utf8)) return 1;
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(get)(h, key, out_str, out_len, out_utf8)) return 1; 1 1 2
if (SHM_FN(get)(h, key, out_str, out_len, out_utf8)) return 1; 1 1 2
if (SHM_FN(get)(h, key, out_str, out_len, out_utf8)) return 1; 2 2 4
5992 #elif defined(SHM_KEY_IS_INT)
5993 30 ▶ if (SHM_FN(get)(h, key, out_value)) return 1;
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(get)(h, key, out_value)) return 1; 3 19 22
if (SHM_FN(get)(h, key, out_value)) return 1; 2 2 4
if (SHM_FN(get)(h, key, out_value)) return 1; 2 2 4
5994 #elif defined(SHM_VAL_IS_STR)
5995 56 ▶ if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_str, out_len, out_utf8)) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_str, out_len, out_utf8)) return 1; 2 54 56
5996 #else
5997 8 ▶ if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_value)) return 1;
Branch: 3 branches
BranchTrueFalseTotal
if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_value)) return 1; 1 1 2
if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_value)) return 1; 1 1 2
if (SHM_FN(get)(h, key_str, key_len, key_utf8, out_value)) return 1; 2 2 4
5998 #endif
5999 }
6000  
6001 111 shm_rwlock_wrlock(h);
6002 111 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 3 56 59
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 0 1 1
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 1 2
uint32_t now = h->expires_at ? shm_now() : 0; 1 2 3
uint32_t now = h->expires_at ? shm_now() : 0; 1 35 36
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
uint32_t now = h->expires_at ? shm_now() : 0; 0 2 2
6003 111 shm_seqlock_write_begin(&hdr->seq);
6004  
6005 SHM_FN(maybe_grow)(h);
6006 111 uint32_t mask = hdr->table_cap - 1;
6007 #ifdef SHM_KEY_IS_INT
6008 47 uint32_t hash = SHM_HASH_KEY(key);
6009 #else
6010 64 shm_sig_hold_bytes(h, key_len);
6011 64 uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
6012 #endif
6013 111 uint32_t pos = hash & mask;
6014 111 uint32_t insert_pos = UINT32_MAX;
6015  
6016 111 uint8_t tag = SHM_MAKE_TAG(hash);
6017 483 ▶ for (uint32_t i = 0; i <= mask; i++) {
Branch: 10 branches
BranchTrueFalseTotal
for (uint32_t i = 0; i <= mask; i++) { 247 1 248
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
for (uint32_t i = 0; i <= mask; i++) { 16 1 17
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
for (uint32_t i = 0; i <= mask; i++) { 18 1 19
for (uint32_t i = 0; i <= mask; i++) { 89 2 91
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
for (uint32_t i = 0; i <= mask; i++) { 17 1 18
6018 472 uint32_t idx = (pos + i) & mask;
6019 472 uint8_t st = states[idx];
6020 472 __builtin_prefetch(&nodes[idx], 0, 1);
6021 472 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
6022  
6023 472 ▶ if (st == SHM_EMPTY) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_EMPTY) { 57 190 247
if (st == SHM_EMPTY) { 1 16 17
if (st == SHM_EMPTY) { 0 16 16
if (st == SHM_EMPTY) { 1 16 17
if (st == SHM_EMPTY) { 1 16 17
if (st == SHM_EMPTY) { 1 16 17
if (st == SHM_EMPTY) { 2 16 18
if (st == SHM_EMPTY) { 17 72 89
if (st == SHM_EMPTY) { 1 16 17
if (st == SHM_EMPTY) { 1 16 17
6024 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 7 50 57
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 2 0 2
if (insert_pos == UINT32_MAX) insert_pos = idx; 17 0 17
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
if (insert_pos == UINT32_MAX) insert_pos = idx; 1 0 1
6025 0 break;
6026 }
6027 390 ▶ if (st == SHM_TOMBSTONE) {
Branch: 10 branches
BranchTrueFalseTotal
if (st == SHM_TOMBSTONE) { 60 130 190
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 72 72
if (st == SHM_TOMBSTONE) { 0 16 16
if (st == SHM_TOMBSTONE) { 0 16 16
6028 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 50 10 60
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
6029 0 continue;
6030 }
6031 330 ▶ if (st != tag) continue;
Branch: 10 branches
BranchTrueFalseTotal
if (st != tag) continue; 129 1 130
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
if (st != tag) continue; 55 17 72
if (st != tag) continue; 16 0 16
if (st != tag) continue; 16 0 16
6032 #ifdef SHM_KEY_IS_INT
6033 0 ▶ if (SHM_KEY_EQ(&nodes[idx], key)) {
Branch: 6 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 17 0 17
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
if (SHM_KEY_EQ(&nodes[idx], key)) { 0 0 0
6034 #else
6035 0 ▶ if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
Branch: 4 branches
BranchTrueFalseTotal
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 1 0 1
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { 0 0 0
6036 #endif
6037 0 ▶ if (SHM_IS_EXPIRED(h, idx, now)) {
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 1 16 17
if (SHM_IS_EXPIRED(h, idx, now)) { 1 0 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 1 1
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
if (SHM_IS_EXPIRED(h, idx, now)) { 0 0 0
6038 0 SHM_FN(expire_at)(h, idx);
6039 0 ▶ if (insert_pos == UINT32_MAX) insert_pos = idx;
Branch: 10 branches
BranchTrueFalseTotal
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
if (insert_pos == UINT32_MAX) insert_pos = idx; 0 0 0
6040 0 break;
6041 }
6042  
6043 #ifdef SHM_VAL_IS_STR
6044 {
6045 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 1 0 1
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[idx].val_len); 0 0 0
6046 0 shm_sig_hold_bytes(h, vl);
6047 0 ▶ if (!shm_ensure_copy_buf(h, vl)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl)) { 0 1 1
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
6048 0 shm_seqlock_write_end(&hdr->seq);
6049 0 shm_rwlock_wrunlock(h);
6050 0 return 0;
6051 }
6052 0 shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
6053 0 *out_str = h->copy_buf;
6054 0 *out_len = vl;
6055 0 *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
6056 }
6057 #else
6058 0 *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
6059 #endif
6060 0 ▶ if (h->lru_prev) shm_lru_promote(h, idx);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_promote(h, idx); 0 1 1
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 16 1 17
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
if (h->lru_prev) shm_lru_promote(h, idx); 0 0 0
6061 0 ▶ if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
Branch: 30 branches
BranchTrueFalseTotal
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 16 17
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 1 0 1
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) 0 0 0
6062 0 h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
6063 0 shm_seqlock_write_end(&hdr->seq);
6064 0 shm_rwlock_wrunlock(h);
6065 0 return 1;
6066 }
6067 }
6068  
6069 93 ▶ if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) {
Branch: 10 branches
BranchTrueFalseTotal
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 57 58
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 0 1
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 2 3
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 2 17 19
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
if ((insert_pos = SHM_FN(insert_slot)(h, pos, mask, insert_pos)) == UINT32_MAX) { 1 1 2
6070 11 shm_seqlock_write_end(&hdr->seq);
6071 11 shm_rwlock_wrunlock(h);
6072 11 return 0;
6073 }
6074  
6075 0 SHM_FN(evict_for_insert)(h, SHM_KEY_SLEN(key_len), SHM_VAL_SLEN(def_len));
6076  
6077 0 int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
6078 0 SHM_FN(arena_reclaim)(h);
6079  
6080 #ifdef SHM_KEY_IS_INT
6081 23 nodes[insert_pos].key = key;
6082 #else
6083 0 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 57 57
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 0 0
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) { 0 1 1
6084 0 shm_seqlock_write_end(&hdr->seq);
6085 0 shm_rwlock_wrunlock(h);
6086 0 return 0;
6087 }
6088 #endif
6089  
6090 #ifdef SHM_VAL_IS_STR
6091 61 ▶ if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) {
Branch: 8 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 57 0 57
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 0 57 57
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 1 0 1
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 0 1 1
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 1 0 1
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 0 1 1
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 2 0 2
if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) { 0 2 2
6092 #ifndef SHM_KEY_IS_INT
6093 0 shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
6094 #endif
6095 0 shm_seqlock_write_end(&hdr->seq);
6096 0 shm_rwlock_wrunlock(h);
6097 0 return 0;
6098 }
6099 61 ▶ if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) { 2 55 57
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) { 0 1 1
if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) { 0 2 2
6100 #ifndef SHM_KEY_IS_INT
6101 2 shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
6102 #endif
6103 0 shm_seqlock_write_end(&hdr->seq);
6104 0 shm_rwlock_wrunlock(h);
6105 0 return 0;
6106 }
6107 #else
6108 0 __atomic_store_n(&nodes[insert_pos].value, def_value, __ATOMIC_RELAXED);
6109 #endif
6110  
6111 0 ▶ if (h->expires_at)
Branch: 10 branches
BranchTrueFalseTotal
if (h->expires_at) 2 53 55
if (h->expires_at) 1 0 1
if (h->expires_at) 0 0 0
if (h->expires_at) 0 1 1
if (h->expires_at) 1 0 1
if (h->expires_at) 1 0 1
if (h->expires_at) 1 1 2
if (h->expires_at) 0 17 17
if (h->expires_at) 0 1 1
if (h->expires_at) 0 1 1
6112 0 ▶ h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0;
Branch: 10 branches
BranchTrueFalseTotal
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 2 0 2
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 1 0 1
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 1 0 1
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 1 0 1
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 1 0 1
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
h->expires_at[insert_pos] = hdr->default_ttl > 0 ? shm_expiry_ts(hdr->default_ttl) : 0; 0 0 0
6113 0 shm_publish_tag(states, insert_pos, SHM_MAKE_TAG(hash));
6114 /* tombstones-- before size++: see put_inner */
6115 0 ▶ if (was_tombstone) hdr->tombstones--;
Branch: 10 branches
BranchTrueFalseTotal
if (was_tombstone) hdr->tombstones--; 50 5 55
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 0 0
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 2 2
if (was_tombstone) hdr->tombstones--; 0 17 17
if (was_tombstone) hdr->tombstones--; 0 1 1
if (was_tombstone) hdr->tombstones--; 0 1 1
6116 0 hdr->size++;
6117  
6118 0 ▶ if (h->lru_prev) shm_lru_push_front(h, insert_pos);
Branch: 10 branches
BranchTrueFalseTotal
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 1 54 55
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 0 0
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 2 2
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 17 17
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
if (h->lru_prev) shm_lru_push_front(h, insert_pos); 0 1 1
6119  
6120 #ifdef SHM_VAL_IS_STR
6121 59 memcpy(h->copy_buf, def_str, def_len);
6122 59 *out_str = h->copy_buf;
6123 59 *out_len = def_len;
6124 59 *out_utf8 = def_utf8;
6125 #else
6126 0 *out_value = def_value;
6127 #endif
6128 0 shm_seqlock_write_end(&hdr->seq);
6129 0 shm_rwlock_wrunlock(h);
6130 0 return 2; /* inserted */
6131 }
6132  
6133 0 static int SHM_FN(each)(ShmHandle *h,
6134 #ifdef SHM_KEY_IS_INT
6135 SHM_KEY_INT_TYPE *out_key,
6136 #else
6137 const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
6138 #endif
6139 #ifdef SHM_VAL_IS_STR
6140 const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
6141 #else
6142 SHM_VAL_INT_TYPE *out_value
6143 #endif
6144 ) {
6145 0 ▶ if (h->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->shard_handles) { 0 1581 1581
if (h->shard_handles) { 178 191 369
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 7901 17480 25381
if (h->shard_handles) { 0 0 0
if (h->shard_handles) { 0 0 0
6146 0 ▶ while (h->shard_iter < h->num_shards) {
Branch: 10 branches
BranchTrueFalseTotal
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 191 4 195
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 7901 0 7901
while (h->shard_iter < h->num_shards) { 0 0 0
while (h->shard_iter < h->num_shards) { 0 0 0
6147 0 int rc = SHM_FN(each)(h->shard_handles[h->shard_iter],
6148 #ifdef SHM_KEY_IS_INT
6149 out_key,
6150 #else
6151 out_key_str, out_key_len, out_key_utf8,
6152 #endif
6153 #ifdef SHM_VAL_IS_STR
6154 out_val_str, out_val_len, out_val_utf8
6155 #else
6156 out_value
6157 #endif
6158 );
6159 0 ▶ if (rc) return 1;
Branch: 10 branches
BranchTrueFalseTotal
if (rc) return 1; 0 0 0
if (rc) return 1; 174 17 191
if (rc) return 1; 0 0 0
if (rc) return 1; 0 0 0
if (rc) return 1; 0 0 0
if (rc) return 1; 0 0 0
if (rc) return 1; 0 0 0
if (rc) return 1; 7901 0 7901
if (rc) return 1; 0 0 0
if (rc) return 1; 0 0 0
6160 0 SHM_FN(flush_deferred)(h->shard_handles[h->shard_iter]);
6161 0 h->shard_iter++;
6162 }
6163 0 h->shard_iter = 0;
6164 0 return 0;
6165 }
6166 0 ShmHeader *hdr = h->hdr;
6167 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
6168 0 uint8_t *states = h->states;
6169 0 shm_rwlock_rdlock(h);
6170 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 574 1007 1581
uint32_t now = h->expires_at ? shm_now() : 0; 0 191 191
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 17480 17480
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
6171  
6172 0 ▶ if (!h->iter_active) {
Branch: 10 branches
BranchTrueFalseTotal
if (!h->iter_active) { 5 1576 1581
if (!h->iter_active) { 18 173 191
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 11 17469 17480
if (!h->iter_active) { 0 0 0
if (!h->iter_active) { 0 0 0
6173 0 h->iter_active = 1;
6174 0 h->iter_gen = hdr->table_gen;
6175 0 h->iterating++;
6176 }
6177  
6178 0 ▶ if (h->iter_gen != hdr->table_gen) {
Branch: 10 branches
BranchTrueFalseTotal
if (h->iter_gen != hdr->table_gen) { 0 1581 1581
if (h->iter_gen != hdr->table_gen) { 0 191 191
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 17480 17480
if (h->iter_gen != hdr->table_gen) { 0 0 0
if (h->iter_gen != hdr->table_gen) { 0 0 0
6179 0 h->iter_pos = 0;
6180 0 h->iter_gen = hdr->table_gen;
6181 }
6182  
6183 0 ▶ while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) {
Branch: 10 branches
BranchTrueFalseTotal
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 1576 5 1581
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 174 17 191
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 17472 8 17480
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) { 0 0 0
6184 0 uint32_t pos = h->iter_pos++;
6185 {
6186 0 ▶ if (SHM_IS_EXPIRED(h, pos, now))
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, pos, now)) 572 1004 1576
if (SHM_IS_EXPIRED(h, pos, now)) 572 0 572
if (SHM_IS_EXPIRED(h, pos, now)) 0 572 572
if (SHM_IS_EXPIRED(h, pos, now)) 0 174 174
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 17472 17472
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
6187 0 continue;
6188  
6189 #ifdef SHM_KEY_IS_INT
6190 0 *out_key = nodes[pos].key;
6191 #else
6192 {
6193 0 ▶ uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 1064 512 1576
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 0 174 174
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 0 0 0
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 0 0 0
6194 #ifdef SHM_VAL_IS_STR
6195 1576 ▶ uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 1514 62 1576
6196 1576 uint32_t total = kl + vl;
6197 #else
6198 0 uint32_t total = kl;
6199 #endif
6200 0 ▶ if (!shm_ensure_copy_buf(h, total)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, total)) { 0 1576 1576
if (!shm_ensure_copy_buf(h, total)) { 0 174 174
if (!shm_ensure_copy_buf(h, total)) { 0 0 0
if (!shm_ensure_copy_buf(h, total)) { 0 0 0
6201 0 h->iter_pos = 0;
6202 0 h->iter_active = 0;
6203 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 4 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
6204 0 shm_rwlock_rdunlock(h);
6205 0 return 0;
6206 }
6207 0 shm_str_copy(h->copy_buf, nodes[pos].key_off, nodes[pos].key_len, h->arena, h->hdr->arena_cap, kl);
6208 0 *out_key_str = h->copy_buf;
6209 0 *out_key_len = kl;
6210 0 *out_key_utf8 = SHM_UNPACK_UTF8(nodes[pos].key_len);
6211 }
6212 #endif
6213 #ifdef SHM_VAL_IS_STR
6214 {
6215 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 1514 62 1576
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 0 0 0
6216 #ifndef SHM_KEY_IS_INT
6217 1576 ▶ uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 1064 512 1576
6218 1576 shm_str_copy(h->copy_buf + kl, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
6219 1576 *out_val_str = h->copy_buf + kl;
6220 #else
6221 0 ▶ if (!shm_ensure_copy_buf(h, vl)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
if (!shm_ensure_copy_buf(h, vl)) { 0 0 0
6222 0 h->iter_pos = 0;
6223 0 h->iter_active = 0;
6224 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 3 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
6225 0 shm_rwlock_rdunlock(h);
6226 0 return 0;
6227 }
6228 0 shm_str_copy(h->copy_buf, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
6229 0 *out_val_str = h->copy_buf;
6230 #endif
6231 0 *out_val_len = vl;
6232 0 *out_val_utf8 = SHM_UNPACK_UTF8(nodes[pos].val_len);
6233 }
6234 #else
6235 0 *out_value = __atomic_load_n(&nodes[pos].value, __ATOMIC_RELAXED);
6236 #endif
6237 0 shm_rwlock_rdunlock(h);
6238 0 return 1;
6239 }
6240 }
6241  
6242 0 h->iter_pos = 0;
6243 0 h->iter_active = 0;
6244 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 5 0 5
if (h->iterating > 0) h->iterating--; 17 0 17
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 8 0 8
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
6245 0 shm_rwlock_rdunlock(h);
6246 0 return 0; /* caller should call flush_deferred */
6247 }
6248  
6249 0 static int SHM_FN(cursor_next)(ShmCursor *c,
6250 #ifdef SHM_KEY_IS_INT
6251 SHM_KEY_INT_TYPE *out_key,
6252 #else
6253 const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
6254 #endif
6255 #ifdef SHM_VAL_IS_STR
6256 const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
6257 #else
6258 SHM_VAL_INT_TYPE *out_value
6259 #endif
6260 ) {
6261 0 ▶ while (c->shard_idx < c->shard_count) {
Branch: 10 branches
BranchTrueFalseTotal
while (c->shard_idx < c->shard_count) { 1016 3 1019
while (c->shard_idx < c->shard_count) { 6 1 7
while (c->shard_idx < c->shard_count) { 0 0 0
while (c->shard_idx < c->shard_count) { 0 0 0
while (c->shard_idx < c->shard_count) { 0 0 0
while (c->shard_idx < c->shard_count) { 0 0 0
while (c->shard_idx < c->shard_count) { 3 1 4
while (c->shard_idx < c->shard_count) { 2502 15 2517
while (c->shard_idx < c->shard_count) { 0 0 0
while (c->shard_idx < c->shard_count) { 0 0 0
6262 0 ShmHandle *h = c->current;
6263 0 ShmHeader *hdr = h->hdr;
6264 0 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
6265 0 uint8_t *states = h->states;
6266 0 shm_rwlock_rdlock(h);
6267 0 ▶ uint32_t now = h->expires_at ? shm_now() : 0;
Branch: 10 branches
BranchTrueFalseTotal
uint32_t now = h->expires_at ? shm_now() : 0; 0 1016 1016
uint32_t now = h->expires_at ? shm_now() : 0; 0 6 6
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 3 3
uint32_t now = h->expires_at ? shm_now() : 0; 0 2502 2502
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
uint32_t now = h->expires_at ? shm_now() : 0; 0 0 0
6268  
6269 0 ▶ if (c->gen != hdr->table_gen) {
Branch: 10 branches
BranchTrueFalseTotal
if (c->gen != hdr->table_gen) { 0 1016 1016
if (c->gen != hdr->table_gen) { 0 6 6
if (c->gen != hdr->table_gen) { 0 0 0
if (c->gen != hdr->table_gen) { 0 0 0
if (c->gen != hdr->table_gen) { 0 0 0
if (c->gen != hdr->table_gen) { 0 0 0
if (c->gen != hdr->table_gen) { 0 3 3
if (c->gen != hdr->table_gen) { 0 2502 2502
if (c->gen != hdr->table_gen) { 0 0 0
if (c->gen != hdr->table_gen) { 0 0 0
6270 0 c->iter_pos = 0;
6271 0 c->gen = hdr->table_gen;
6272 }
6273  
6274 0 ▶ while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) {
Branch: 10 branches
BranchTrueFalseTotal
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 1013 3 1016
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 5 1 6
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 2 1 3
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 2478 24 2502
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) { 0 0 0
6275 0 uint32_t pos = c->iter_pos++;
6276 {
6277 0 ▶ if (SHM_IS_EXPIRED(h, pos, now))
Branch: 30 branches
BranchTrueFalseTotal
if (SHM_IS_EXPIRED(h, pos, now)) 0 1013 1013
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 5 5
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 2 2
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 2478 2478
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
if (SHM_IS_EXPIRED(h, pos, now)) 0 0 0
6278 0 continue;
6279  
6280 #ifdef SHM_KEY_IS_INT
6281 0 *out_key = nodes[pos].key;
6282 #else
6283 {
6284 0 ▶ uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 1012 1 1013
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 5 0 5
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 0 0 0
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 0 0 0
6285 #ifdef SHM_VAL_IS_STR
6286 1013 ▶ uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 1012 1 1013
6287 1013 uint32_t total = kl + vl;
6288 #else
6289 0 uint32_t total = kl;
6290 #endif
6291 0 ▶ if (!shm_cursor_ensure_copy_buf(c, total)) {
Branch: 4 branches
BranchTrueFalseTotal
if (!shm_cursor_ensure_copy_buf(c, total)) { 0 1013 1013
if (!shm_cursor_ensure_copy_buf(c, total)) { 0 5 5
if (!shm_cursor_ensure_copy_buf(c, total)) { 0 0 0
if (!shm_cursor_ensure_copy_buf(c, total)) { 0 0 0
6292 0 shm_rwlock_rdunlock(h);
6293 0 return 0;
6294 }
6295 0 shm_str_copy(c->copy_buf, nodes[pos].key_off, nodes[pos].key_len, h->arena, h->hdr->arena_cap, kl);
6296 0 *out_key_str = c->copy_buf;
6297 0 *out_key_len = kl;
6298 0 *out_key_utf8 = SHM_UNPACK_UTF8(nodes[pos].key_len);
6299 }
6300 #endif
6301 #ifdef SHM_VAL_IS_STR
6302 {
6303 0 ▶ uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
Branch: 4 branches
BranchTrueFalseTotal
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 1012 1 1013
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 0 0 0
uint32_t vl = SHM_STR_LEN(nodes[pos].val_len); 2 0 2
6304 #ifndef SHM_KEY_IS_INT
6305 1013 ▶ uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
Branch: 1 branch
BranchTrueFalseTotal
uint32_t kl = SHM_STR_LEN(nodes[pos].key_len); 1012 1 1013
6306 1013 shm_str_copy(c->copy_buf + kl, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
6307 1013 *out_val_str = c->copy_buf + kl;
6308 #else
6309 0 ▶ if (!shm_cursor_ensure_copy_buf(c, vl)) {
Branch: 3 branches
BranchTrueFalseTotal
if (!shm_cursor_ensure_copy_buf(c, vl)) { 0 0 0
if (!shm_cursor_ensure_copy_buf(c, vl)) { 0 0 0
if (!shm_cursor_ensure_copy_buf(c, vl)) { 0 2 2
6310 0 shm_rwlock_rdunlock(h);
6311 0 return 0;
6312 }
6313 0 shm_str_copy(c->copy_buf, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
6314 0 *out_val_str = c->copy_buf;
6315 #endif
6316 0 *out_val_len = vl;
6317 0 *out_val_utf8 = SHM_UNPACK_UTF8(nodes[pos].val_len);
6318 }
6319 #else
6320 0 *out_value = __atomic_load_n(&nodes[pos].value, __ATOMIC_RELAXED);
6321 #endif
6322 0 shm_rwlock_rdunlock(h);
6323 0 return 1;
6324 }
6325 }
6326  
6327 0 shm_rwlock_rdunlock(h);
6328  
6329 0 ▶ if (h->iterating > 0) h->iterating--;
Branch: 10 branches
BranchTrueFalseTotal
if (h->iterating > 0) h->iterating--; 3 0 3
if (h->iterating > 0) h->iterating--; 1 0 1
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 1 0 1
if (h->iterating > 0) h->iterating--; 24 0 24
if (h->iterating > 0) h->iterating--; 0 0 0
if (h->iterating > 0) h->iterating--; 0 0 0
6330 0 SHM_FN(flush_deferred)(h);
6331 0 c->shard_idx++;
6332 0 ▶ if (c->shard_idx < c->shard_count) {
Branch: 10 branches
BranchTrueFalseTotal
if (c->shard_idx < c->shard_count) { 0 3 3
if (c->shard_idx < c->shard_count) { 0 1 1
if (c->shard_idx < c->shard_count) { 0 0 0
if (c->shard_idx < c->shard_count) { 0 0 0
if (c->shard_idx < c->shard_count) { 0 0 0
if (c->shard_idx < c->shard_count) { 0 0 0
if (c->shard_idx < c->shard_count) { 0 1 1
if (c->shard_idx < c->shard_count) { 10 14 24
if (c->shard_idx < c->shard_count) { 0 0 0
if (c->shard_idx < c->shard_count) { 0 0 0
6333 0 ShmHandle *parent = c->handle;
6334 0 c->current = parent->shard_handles[c->shard_idx];
6335 0 c->current->iterating++;
6336 0 c->iter_pos = 0;
6337 0 c->gen = c->current->hdr->table_gen;
6338 } else {
6339 /* Or destroy, reset and seek decrement the last shard's `iterating` again. */
6340 0 c->current = NULL;
6341 }
6342 }
6343  
6344 0 return 0;
6345 }
6346  
6347 0 static inline void SHM_FN(cursor_reset)(ShmCursor *c) {
6348 0 ▶ if (c->current && c->current->iterating > 0)
Branch: 20 branches
BranchTrueFalseTotal
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 2 2
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
if (c->current && c->current->iterating > 0) 0 0 0
6349 0 c->current->iterating--;
6350 0 SHM_FN(flush_deferred)(c->current);
6351 0 c->shard_idx = 0;
6352 0 ▶ if (c->handle->shard_handles) {
Branch: 10 branches
BranchTrueFalseTotal
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 2 2
if (c->handle->shard_handles) { 0 0 0
if (c->handle->shard_handles) { 0 0 0
6353 0 c->current = c->handle->shard_handles[0];
6354 } else {
6355 0 c->current = c->handle;
6356 }
6357 0 c->current->iterating++;
6358 0 c->iter_pos = 0;
6359 0 c->gen = c->current->hdr->table_gen;
6360 0 }
6361  
6362 /* The next cursor_next returns this key's entry, then continues forward. */
6363 55 static int SHM_FN(cursor_seek)(ShmCursor *c,
6364 #ifdef SHM_KEY_IS_INT
6365 SHM_KEY_INT_TYPE key
6366 #else
6367 const char *key_str, uint32_t key_len, bool key_utf8
6368 #endif
6369 ) {
6370 #ifdef SHM_KEY_IS_INT
6371 49 uint64_t hash64 = SHM_HASH_KEY64(key);
6372 #else
6373 6 uint64_t hash64 = SHM_HASH_KEY_STR64(key_str, key_len);
6374 #endif
6375 55 uint32_t hash = (uint32_t)hash64;
6376 55 ShmHandle *parent = c->handle;
6377 ShmHandle *target;
6378 55 uint32_t target_shard = 0;
6379 55 if (parent->shard_handles) {
6380 48 target_shard = shm_shard_index(parent, hash64);
6381 48 target = parent->shard_handles[target_shard];
6382 } else {
6383 7 target = parent;
6384 }
6385  
6386 /* Probe first: a seek that finds nothing leaves the iteration where it was. */
6387 55 ShmHandle *h = target;
6388 55 ShmHeader *hdr = h->hdr;
6389 55 SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
6390 55 uint8_t *states = h->states;
6391 55 shm_rwlock_rdlock(h);
6392 55 uint32_t now = h->expires_at ? shm_now() : 0;
6393  
6394 55 uint32_t mask = hdr->table_cap - 1;
6395 55 uint32_t pos = hash & mask;
6396 55 uint8_t tag = SHM_MAKE_TAG(hash);
6397  
6398 61 for (uint32_t i = 0; i <= mask; i++) {
6399 61 uint32_t idx = (pos + i) & mask;
6400 61 uint8_t st = states[idx];
6401 61 __builtin_prefetch(&nodes[idx], 0, 1);
6402 61 __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
6403 61 if (st == SHM_EMPTY) break;
6404 17 if (st != tag) continue;
6405 #ifdef SHM_KEY_IS_INT
6406 5 if (SHM_KEY_EQ(&nodes[idx], key)) {
6407 #else
6408 6 if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
6409 #endif
6410 11 if (SHM_IS_EXPIRED(h, idx, now)) {
6411 1 shm_rwlock_rdunlock(h);
6412 1 return 0;
6413 }
6414 10 uint32_t found_gen = hdr->table_gen;
6415 10 shm_rwlock_rdunlock(h);
6416 /* gen as read under the lock: a resize since makes cursor_next reset. */
6417 10 if (target != c->current) {
6418 5 if (c->current && c->current->iterating > 0)
6419 5 c->current->iterating--;
6420 5 SHM_FN(flush_deferred)(c->current);
6421 5 c->current = target;
6422 5 c->current->iterating++;
6423 5 c->shard_idx = target_shard;
6424 }
6425 10 c->iter_pos = idx;
6426 10 c->gen = found_gen;
6427 10 return 1;
6428 }
6429 }
6430  
6431 44 shm_rwlock_rdunlock(h);
6432 44 return 0;
6433 }
6434  
6435 #undef SHM_PASTE2
6436 #undef SHM_PASTE
6437 #undef SHM_FN
6438 #undef SHM_NODE_TYPE
6439 #undef SHM_PREFIX
6440 #undef SHM_VARIANT_ID
6441 #undef SHM_HAS_ARENA
6442 #undef SHM_COMPACT_BACKOFF
6443 #undef SHM_COMPACT_BACKOFF_MAX
6444 #undef SHM_KEY_SLEN
6445 #undef SHM_VAL_SLEN
6446  
6447 #ifdef SHM_KEY_IS_INT
6448 #undef SHM_KEY_IS_INT
6449 #undef SHM_KEY_INT_TYPE
6450 #undef SHM_HASH_KEY
6451 #undef SHM_HASH_KEY64
6452 #undef SHM_KEY_EQ
6453 #else
6454 #undef SHM_HASH_KEY_STR
6455 #undef SHM_HASH_KEY_STR64
6456 #undef SHM_KEY_EQ_STR
6457 #endif
6458  
6459 #ifdef SHM_VAL_IS_STR
6460 #undef SHM_VAL_IS_STR
6461 #else
6462 #undef SHM_VAL_INT_TYPE
6463 #endif
6464  
6465 #ifdef SHM_HAS_COUNTERS
6466 #undef SHM_HAS_COUNTERS
6467 #endif
« xs/si32.xs ↑ Summary xs/is.xs »