hist.h

filter: Statementstmt 72.9% 405 / 555 Branchbran 47.5% 216 / 454 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 61.5% 621 / 1009 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« blib/lib/Data/Histogram/Shared.pm ↑ Summary Shared.xs »
1 /*
2 * hist.h -- Shared-memory HdrHistogram for Linux
3 *
4 * A High Dynamic Range histogram: records integer values across a very wide
5 * range and answers percentile / min / max / mean queries within a fixed,
6 * configurable relative error. Values are bucketed logarithmically (one bucket
7 * per power of two of magnitude) and linearly within each bucket (a fixed
8 * number of sub-buckets per power of two), so a constant number of significant
9 * figures is preserved across the whole range. The counts array lives in a
10 * shared mapping so several processes share one histogram; a write-preferring
11 * futex rwlock with reader-slot dead-process recovery guards mutation. Two
12 * histograms of equal geometry can be merged (cellwise add -> combined stream).
13 *
14 * Layout: Header -> reader_slots[1024] -> occ bitmap -> counts[counts_len] (each int64_t)
15 */
16  
17 #ifndef HIST_H
18 #define HIST_H
19  
20 #include <stddef.h>
21 #include <stdint.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <unistd.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <time.h>
28 #include <limits.h>
29 #include <float.h>
30 #include <signal.h>
31 #include <stdio.h>
32 #include <math.h>
33 #include <sys/mman.h>
34 #include <sys/stat.h>
35 #include <sys/file.h>
36 #include <sys/syscall.h>
37 #include <linux/futex.h>
38 #include <pthread.h>
39  
40 #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
41 #error "hist.h: requires little-endian architecture"
42 #endif
43  
44  
45 /* ================================================================
46 * Constants
47 * ================================================================ */
48  
49 #define HIST_MAGIC 0x54534948U /* "HIST" (little-endian) */
50 #define HIST_VERSION 2 /* 2: added the occupancy bitmap region (layout change) */
51 #define HIST_ERR_BUFLEN (PATH_MAX + 256)
52 #ifndef HIST_READER_SLOTS
53 #define HIST_READER_SLOTS 1024 /* max concurrent reader processes for dead-process recovery */
54 #endif
55 /* Occupancy bitmap: one bit per reader slot, set when a process claims a slot and
56 * cleared on clean release. A writer scans these HIST_OCC_WORDS words to visit
57 * only OCCUPIED slots (O(words + live readers)) instead of all HIST_READER_SLOTS. */
58 #define HIST_OCC_WORDS (((HIST_READER_SLOTS) + 63) / 64) /* 16 for 1024 slots */
59 #define HIST_OCC_BYTES ((uint64_t)HIST_OCC_WORDS * 8) /* 128 bytes */
60 #define HIST_MIN_SIG 1 /* significant figures range */
61 #define HIST_MAX_SIG 5
62  
63 #define HIST_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, HIST_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while (0)
64  
65 /* ================================================================
66 * Structs
67 * ================================================================ */
68  
69 /* Per-process slot for dead-process recovery. In the reader-slots-only rwlock a
70 * reader's ENTIRE contribution to the shared lock is `rdepth` in its OWN slot --
71 * there is no separate shared reader counter to fall out of sync with it -- so a
72 * dead reader's contribution is exactly this one word, which a draining writer
73 * neutralises by clearing the slot's pid (the scan then ignores the slot). No
74 * orphaned counter can exist, so there is no quiescent force-reset and sustained
75 * readers cannot starve a writer. _rsv1/_rsv2 are kept only to preserve the
76 * 16-byte slot size across the already-released builds. */
77 typedef struct {
78 uint32_t pid; /* 0 = unclaimed */
79 uint32_t rdepth; /* read-locks THIS process currently holds (recursion-safe) */
80 uint32_t _rsv1; /* reserved (was waiters_parked); unused, kept for layout size */
81 uint32_t _rsv2; /* reserved (was writers_parked); unused, kept for layout size */
82 } HistReaderSlot;
83  
84 struct HistHeader {
85 uint32_t magic, version; /* 0,4 */
86 uint32_t _pad0; /* 8 */
87 uint32_t _pad1; /* 12 */
88  
89 /* ---- configuration ---- */
90 int64_t lowest; /* 16 lowest trackable value (>= 1) */
91 int64_t highest; /* 24 highest trackable value */
92 int32_t sig_figs; /* 32 significant figures, [1,5] */
93 int32_t unit_magnitude; /* 36 floor(log2(lowest)) */
94  
95 /* ---- derived geometry ---- */
96 int32_t sub_bucket_count_magnitude; /* 40 */
97 int32_t sub_bucket_half_count_magnitude; /* 44 */
98 int32_t sub_bucket_count; /* 48 */
99 int32_t sub_bucket_half_count; /* 52 */
100 int64_t sub_bucket_mask; /* 56 */
101 int32_t bucket_count; /* 64 */
102 int32_t _pad2; /* 68 */
103 int64_t counts_len; /* 72 number of int64 counts */
104  
105 /* ---- recorded data ---- */
106 int64_t total_count; /* 80 sum of all recorded counts */
107 int64_t min_value; /* 88 min recorded value (INT64_MAX init)*/
108 int64_t max_value; /* 96 max recorded value (0 init) */
109  
110 /* ---- offsets / size ---- */
111 uint64_t total_size; /* 104 */
112 uint64_t reader_slots_off; /* 112 */
113 uint64_t counts_off; /* 120 */
114  
115 /* ---- lock + stats ---- */
116 uint32_t wlock; /* 128 WRITER word ONLY: 0 (free) or WRITER_BIT|pid. NOT a reader count. */
117 uint32_t rwait; /* 132 parked-waiter hint (readers+writers blocked on wlock); over-count-safe */
118 uint32_t drain_seq; /* 136 futex bumped by a reader releasing under a draining writer (wakes it) */
119 uint32_t slotless_rdepth; /* live readers holding with no reader-slot (documented residual) */
120 uint64_t stat_ops; /* 144 */
121 uint8_t sealed; /* 152 0 = mutable, 1 = frozen (read-only; lock-free reads) */
122 uint8_t _pad[103]; /* 153..255 */
123 };
124 typedef struct HistHeader HistHeader;
125  
126 _Static_assert(sizeof(HistHeader) == 256, "HistHeader must be 256 bytes");
127  
128 /* ---- Process-local handle ---- */
129  
130 typedef struct HistHandle {
131 HistHeader *hdr;
132 HistReaderSlot *reader_slots; /* HIST_READER_SLOTS entries */
133 uint64_t *occ; /* HIST_OCC_WORDS-word slot-occupancy bitmap (trusted layout offset) */
134 void *base; /* mmap base */
135 size_t mmap_size;
136 uint64_t counts_off; /* validated counts offset, cached: never re-read from the peer-writable header */
137 char *path; /* backing file path (strdup'd) */
138 int backing_fd; /* memfd or reopened-fd to close on destroy, -1 for file/anon */
139 uint32_t my_slot_idx; /* UINT32_MAX if all slots taken (no recovery for this handle) */
140 uint32_t cached_pid; /* getpid() cached at last slot claim */
141 uint32_t cached_fork_gen; /* hist_fork_gen value at last slot claim */
142 uint32_t slotless_held; /* read-locks this process holds with no reader-slot */
143 int readonly; /* 1 = frozen O_RDONLY/PROT_READ view: lock-free reads, mutation croaks */
144 } HistHandle;
145  
146 /* ================================================================
147 * Futex-based write-preferring read-write lock (reader-slots-only)
148 * with dead-process recovery
149 *
150 * The reader count is NOT stored in a shared counter. It is DISTRIBUTED across
151 * per-process reader slots: each slot's `rdepth` is that process's entire
152 * contribution to the lock. A reader publishes its presence in its own slot and
153 * then re-checks the writer word; a writer publishes the writer word and then
154 * scans every slot until all live readers' rdepth reach 0. Sequentially-
155 * consistent store+load on each side (a Dekker handshake) gives mutual exclusion.
156 *
157 * Because a reader's whole contribution is ONE atomic word owned by ONE process,
158 * a crashed reader is recovered by clearing that one slot (CAS its pid to 0) --
159 * there is no second counter to strand, no orphaned +1, and therefore no
160 * quiescent force-reset. A reader killed anywhere in rdlock/rdunlock leaves at
161 * most `rdepth>0` in its dead slot, which the draining writer clears directly, so
162 * sustained read traffic can never starve a writer. Write-preference is inherent
163 * in the gate (new readers see wlock!=0 and yield), so there is no reader-count
164 * yield hack.
165 * ================================================================ */
166  
167 #define HIST_RWLOCK_SPIN_LIMIT 32
168 #define HIST_LOCK_TIMEOUT_SEC 2 /* FUTEX_WAIT timeout for stale-lock detection / drain re-scan */
169  
170 0 static inline void hist_rwlock_spin_pause(void) {
171 #if defined(__x86_64__) || defined(__i386__)
172 0 __asm__ volatile("pause" ::: "memory");
173 #elif defined(__aarch64__)
174 __asm__ volatile("yield" ::: "memory");
175 #else
176 __asm__ volatile("" ::: "memory");
177 #endif
178 0 }
179  
180 /* Writer word encoding: WRITER_BIT|pid when write-locked, 0 when free. */
181 #define HIST_RWLOCK_WRITER_BIT 0x80000000U
182 #define HIST_RWLOCK_PID_MASK 0x7FFFFFFFU
183 #define HIST_RWLOCK_WR(pid) (HIST_RWLOCK_WRITER_BIT | ((uint32_t)(pid) & HIST_RWLOCK_PID_MASK))
184  
185 /* A zombie (dead but not yet reaped) still answers kill(pid,0) as alive, so a
186 * process that crashed while holding the lock and lingers unreaped would never
187 * be recovered. Treat /proc/<pid>/stat state 'Z' as dead. Linux-only (as is
188 * this module); if /proc is unreadable we fall back to "alive" (safe: we never
189 * force-recover a possibly-live holder). */
190 0 static inline int hist_pid_is_zombie(uint32_t pid) {
191 char path[32], buf[256];
192 0 snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
193 0 int fd = open(path, O_RDONLY | O_CLOEXEC);
194 0 ▶ if (fd < 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return 0; 0 0 0
195 0 ssize_t n = read(fd, buf, sizeof(buf) - 1);
196 0 close(fd);
197 0 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 0 0
198 0 buf[n] = '\0';
199 /* "pid (comm) state ..."; comm may contain ')', so scan to the last one. */
200 0 char *rp = strrchr(buf, ')');
201 0 ▶ if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */
Branch: 2 branches
BranchTrueFalseTotal
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 0 0 0
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 0 0 0
202 0 ▶ return rp[1] == ' ' && rp[2] == 'Z';
Branch: 2 branches
BranchTrueFalseTotal
return rp[1] == ' ' && rp[2] == 'Z'; 0 0 0
return rp[1] == ' ' && rp[2] == 'Z'; 0 0 0
203 }
204 /* 1 if alive or unknown, 0 if definitely dead. Cannot detect PID reuse: a
205 * recycled PID reports "alive" and the slot is not reclaimed until that
206 * process exits. See "Crash Safety" in the POD. */
207 0 static inline int hist_pid_alive(uint32_t pid) {
208 0 ▶ if (pid == 0) return 0; /* no owner recorded, treat as dead */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) return 0; /* no owner recorded, treat as dead */ 0 0 0
209 0 ▶ if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */
Branch: 2 branches
BranchTrueFalseTotal
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 0 0 0
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 0 0 0
210 0 return !hist_pid_is_zombie(pid); /* kill() also succeeds for a zombie -> treat as dead */
211 }
212  
213 /* Force-recover a stale WRITE lock left by a dead writer (held or mid-drain).
214 * CAS to OUR pid to hold the lock while fixing shared state, then release.
215 * Using our pid (not a bare WRITER_BIT sentinel) means a subsequent recovering
216 * process can detect and re-recover if we crash mid-recovery. */
217 0 static inline void hist_recover_stale_lock(HistHandle *h, uint32_t observed_wlock) {
218 0 HistHeader *hdr = h->hdr;
219 0 uint32_t mypid = HIST_RWLOCK_WR((uint32_t)getpid());
220 0 ▶ if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock, 0 0 0
221 mypid, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
222 0 return;
223 /* We now hold the write lock as mypid. No additional shared state needs
224 * repair here (this module has no seqlock); just release the lock. */
225 0 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
226 0 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 0 0 0
227 0 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
228 }
229  
230 static const struct timespec hist_lock_timeout = { HIST_LOCK_TIMEOUT_SEC, 0 };
231  
232 /* Occupancy bitmap: set a slot's bit when it is claimed, clear it on clean
233 * release. SEQ_CST so a set bit is ordered before the slot's rdepth can go
234 * non-zero (bit set in claim, which precedes any rdlock), letting a writer's
235 * SEQ_CST bitmap scan never miss a slot a committed reader holds. */
236 51 static inline void hist_occ_set(HistHandle *h, uint32_t s) {
237 51 __atomic_fetch_or(&h->occ[s >> 6], (uint64_t)1 << (s & 63), __ATOMIC_SEQ_CST);
238 51 }
239 51 static inline void hist_occ_clear(HistHandle *h, uint32_t s) {
240 51 __atomic_fetch_and(&h->occ[s >> 6], ~((uint64_t)1 << (s & 63)), __ATOMIC_SEQ_CST);
241 51 }
242  
243 /* Process-global fork-generation counter. Incremented in the pthread_atfork
244 * child callback so every open handle detects a fork transition on the next
245 * lock call without paying a getpid() syscall on the hot path. */
246 static uint32_t hist_fork_gen = 1;
247 static pthread_once_t hist_atfork_once = PTHREAD_ONCE_INIT;
248 0 static void hist_on_fork_child(void) {
249 0 __atomic_add_fetch(&hist_fork_gen, 1, __ATOMIC_RELAXED);
250 0 }
251 4 static void hist_atfork_init(void) {
252 4 pthread_atfork(NULL, NULL, hist_on_fork_child);
253 4 }
254  
255 /* Ensure this process owns a reader slot. Called from the lock helpers so
256 * that fork()'d children pick up their own slot lazily instead of sharing
257 * the parent's. Hot-path is a single relaxed load + compare; only on a
258 * fork-generation mismatch do we touch getpid() and scan slots. */
259 327286 static inline void hist_claim_reader_slot(HistHandle *h) {
260 327286 uint32_t cur_gen = __atomic_load_n(&hist_fork_gen, __ATOMIC_RELAXED);
261 327286 ▶ if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1))
Branch: 2 branches
BranchTrueFalseTotal
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 327235 51 327286
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 327235 0 327235
262 327235 return;
263 /* Cold path -- register the atfork hook once per process, then claim. */
264 51 pthread_once(&hist_atfork_once, hist_atfork_init);
265 /* Re-read after pthread_once: hist_on_fork_child may have bumped it. */
266 51 cur_gen = __atomic_load_n(&hist_fork_gen, __ATOMIC_RELAXED);
267 51 uint32_t now_pid = (uint32_t)getpid();
268 51 h->cached_pid = now_pid;
269 51 ▶ if (cur_gen != h->cached_fork_gen) h->slotless_held = 0; /* fork: child holds none of the parent's slotless read locks */
Branch: 1 branch
BranchTrueFalseTotal
if (cur_gen != h->cached_fork_gen) h->slotless_held = 0; /* fork: child holds none of the parent's slotless read locks */ 51 0 51
270 51 h->cached_fork_gen = cur_gen;
271 51 h->my_slot_idx = UINT32_MAX;
272 51 uint32_t start = now_pid % HIST_READER_SLOTS;
273 /* Pass 1: take a free slot. */
274 53 ▶ for (uint32_t i = 0; i < HIST_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < HIST_READER_SLOTS; i++) { 53 0 53
275 53 uint32_t s = (start + i) % HIST_READER_SLOTS;
276 53 uint32_t expected = 0;
277 53 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[s].pid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[s].pid, 51 2 53
278 &expected, now_pid, 0,
279 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
280 /* Fresh owner holds no read locks yet; clear any stale rdepth left by
281 * a dead predecessor (its contribution is dropped as we take over). */
282 51 __atomic_store_n(&h->reader_slots[s].rdepth, 0, __ATOMIC_RELAXED);
283 51 hist_occ_set(h, s); /* mark occupied BEFORE any rdlock can bump rdepth */
284 51 h->my_slot_idx = s;
285 51 return;
286 }
287 }
288 /* Pass 2: no free slot -- reclaim one whose owner is dead. Safe to take even
289 * if its rdepth>0: clearing pid drops the dead reader's entire contribution
290 * (a writer scan ignores rdepth when pid==0) and we reset rdepth to 0 as we
291 * claim it. */
292 0 ▶ for (uint32_t i = 0; i < HIST_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < HIST_READER_SLOTS; i++) { 0 0 0
293 0 uint32_t dpid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
294 0 ▶ if (dpid == 0 || dpid == now_pid || hist_pid_alive(dpid)) continue;
Branch: 3 branches
BranchTrueFalseTotal
if (dpid == 0 || dpid == now_pid || hist_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || hist_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || hist_pid_alive(dpid)) continue; 0 0 0
295 0 uint32_t expected = dpid;
296 0 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0, 0 0 0
297 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
298 0 __atomic_store_n(&h->reader_slots[i].rdepth, 0, __ATOMIC_RELAXED);
299 0 hist_occ_set(h, i);
300 0 h->my_slot_idx = i;
301 0 return;
302 }
303 }
304 /* Table full -- leave my_slot_idx = UINT32_MAX so this handle takes the
305 * slotless path (lock still works; recovery of THIS reader's death is the
306 * documented slotless limitation). */
307 }
308  
309 /* Inspect the writer word after a futex-wait timeout. If a dead writer holds
310 * it, force-recover. Dead READERS need no action here: only a writer that owns
311 * wlock drains readers, and it clears dead readers inline in its own scan. */
312 0 static inline void hist_recover_after_timeout(HistHandle *h) {
313 0 uint32_t val = __atomic_load_n(&h->hdr->wlock, __ATOMIC_RELAXED);
314 0 ▶ if (val >= HIST_RWLOCK_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= HIST_RWLOCK_WRITER_BIT) { 0 0 0
315 0 uint32_t pid = val & HIST_RWLOCK_PID_MASK;
316 0 ▶ if (!hist_pid_alive(pid))
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_pid_alive(pid)) 0 0 0
317 0 hist_recover_stale_lock(h, val);
318 }
319 0 }
320  
321 /* Bump/drop the parked-waiter hint. Both readers (blocked at the gate) and
322 * writers (blocked acquiring wlock) wait on the wlock futex and use this, so
323 * wrunlock/recover know whether a FUTEX_WAKE is worth a syscall. A waiter
324 * SIGKILLed while parked leaves rwait over-counted -> at most a spurious wake
325 * (harmless); it can never under-count, so no wakeup is lost. */
326 0 static inline void hist_park(HistHandle *h) {
327 0 __atomic_add_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
328 0 }
329 0 static inline void hist_unpark(HistHandle *h) {
330 0 __atomic_sub_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
331 0 }
332  
333 /* Publish (inc) / retract (dec) this reader's presence -- its ENTIRE
334 * contribution to the lock. A slotted reader uses its slot's rdepth; a reader
335 * that could not claim a slot uses the global slotless_rdepth. inc() is SEQ_CST
336 * so the wlock re-check that follows it in rdlock forms a Dekker handshake with
337 * the writer's SEQ_CST wlock-store + rdepth-scan. leave() peels slotless first
338 * so a slot claimed mid-hold cannot misattribute the decrement. */
339 6141 static inline void hist_rdepth_inc(HistHandle *h) {
340 6141 ▶ if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->my_slot_idx != UINT32_MAX) { 6141 0 6141
341 6141 __atomic_add_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_SEQ_CST);
342 } else {
343 0 __atomic_add_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
344 0 h->slotless_held++;
345 }
346 6141 }
347 6141 static inline void hist_rdepth_dec(HistHandle *h) {
348 6141 ▶ if (h->slotless_held > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->slotless_held > 0) { 0 6141 6141
349 0 h->slotless_held--;
350 0 __atomic_sub_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_RELEASE);
351 6141 ▶ } else if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (h->my_slot_idx != UINT32_MAX) { 6141 0 6141
352 6141 __atomic_sub_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_RELEASE);
353 }
354 6141 }
355  
356 /* Wake a writer that may be draining readers (it waits on drain_seq). Called
357 * after every rdepth decrement so a released read lock lets the writer re-scan
358 * promptly instead of waiting out its timeout. */
359 6141 static inline void hist_reader_wake_drain(HistHandle *h) {
360 6141 ▶ if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_ACQUIRE) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_ACQUIRE) != 0) { 0 6141 6141
361 0 __atomic_add_fetch(&h->hdr->drain_seq, 1, __ATOMIC_RELEASE);
362 0 syscall(SYS_futex, &h->hdr->drain_seq, FUTEX_WAKE, 1, NULL, NULL, 0);
363 }
364 6141 }
365  
366 6141 static inline void hist_rwlock_rdlock(HistHandle *h) {
367 6141 hist_claim_reader_slot(h);
368 6141 HistHeader *hdr = h->hdr;
369 6141 for (int spin = 0; ; spin++) {
370 6141 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
371 6141 ▶ if (cur == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) { 6141 0 6141
372 /* Optimistically take the read: publish rdepth, then re-check wlock.
373 * SEQ_CST inc + SEQ_CST load vs the writer's SEQ_CST wlock CAS +
374 * SEQ_CST rdepth scan: by the single total order of SEQ_CST ops the
375 * two sides cannot both miss each other, so we never hold
376 * concurrently with a writer. */
377 6141 hist_rdepth_inc(h);
378 6141 ▶ if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0) 6141 0 6141
379 6141 return; /* no writer after our publish -> we hold the read lock */
380 /* A writer appeared during our publish -- yield to it (write-preferring). */
381 0 hist_rdepth_dec(h);
382 0 hist_reader_wake_drain(h); /* let the draining writer see rdepth drop */
383 0 spin = 0;
384 0 continue;
385 }
386 /* wlock != 0: a writer holds or is acquiring. Recover if it is dead. */
387 0 if (cur >= HIST_RWLOCK_WRITER_BIT &&
388 0 !hist_pid_alive(cur & HIST_RWLOCK_PID_MASK)) {
389 0 hist_recover_stale_lock(h, cur);
390 0 spin = 0;
391 0 continue;
392 }
393 0 ▶ if (__builtin_expect(spin < HIST_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < HIST_RWLOCK_SPIN_LIMIT, 1)) { 0 0 0
394 0 hist_rwlock_spin_pause();
395 0 continue;
396 }
397 0 hist_park(h);
398 0 cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
399 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
400 0 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
401 &hist_lock_timeout, NULL, 0);
402 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
403 0 hist_unpark(h);
404 0 hist_recover_after_timeout(h);
405 0 spin = 0;
406 0 continue;
407 }
408 }
409 0 hist_unpark(h);
410 0 spin = 0;
411 }
412 }
413  
414 6141 static inline void hist_rwlock_rdunlock(HistHandle *h) {
415 6141 hist_rdepth_dec(h); /* RELEASE: drop our entire contribution */
416 6141 hist_reader_wake_drain(h); /* if a writer is draining, wake it to re-scan */
417 6141 }
418  
419 321145 static inline void hist_rwlock_wrlock(HistHandle *h) {
420 321145 hist_claim_reader_slot(h); /* refresh cached_pid across fork */
421 321145 HistHeader *hdr = h->hdr;
422 /* Encode PID in the wlock word itself (0x80000000 | pid) to eliminate any
423 * crash window between acquiring the lock and storing the owner. */
424 321145 uint32_t mypid = HIST_RWLOCK_WR(h->cached_pid);
425 /* Phase 1: acquire the writer word (mutual exclusion among writers). */
426 321145 for (int spin = 0; ; spin++) {
427 321145 uint32_t expected = 0;
428 321145 ▶ if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid, 321145 0 321145
429 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
430 321145 break;
431 /* Contended: expected now holds the current wlock value. */
432 0 if (expected >= HIST_RWLOCK_WRITER_BIT &&
433 0 !hist_pid_alive(expected & HIST_RWLOCK_PID_MASK)) {
434 0 hist_recover_stale_lock(h, expected);
435 0 spin = 0;
436 0 continue;
437 }
438 0 ▶ if (__builtin_expect(spin < HIST_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < HIST_RWLOCK_SPIN_LIMIT, 1)) { 0 0 0
439 0 hist_rwlock_spin_pause();
440 0 continue;
441 }
442 0 hist_park(h);
443 0 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
444 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
445 0 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
446 &hist_lock_timeout, NULL, 0);
447 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
448 0 hist_unpark(h);
449 0 hist_recover_after_timeout(h);
450 0 spin = 0;
451 0 continue;
452 }
453 }
454 0 hist_unpark(h);
455 0 spin = 0;
456 }
457 /* Phase 2: we own wlock, so no NEW reader can join (they see wlock!=0 and
458 * yield). Drain the readers that were already holding when we won the CAS.
459 * The SEQ_CST CAS above + the SEQ_CST rdepth loads below are the writer side
460 * of the Dekker handshake. */
461 0 for (;;) {
462 321145 uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_ACQUIRE); /* snapshot BEFORE scan */
463 321145 int busy = 0;
464 /* Visit only OCCUPIED slots via the occupancy bitmap (SEQ_CST: a committed
465 * reader's bit -- set in claim, before its rdepth++ -- is ordered before
466 * this scan, so no held slot is skipped). O(HIST_OCC_WORDS + live readers)
467 * instead of O(HIST_READER_SLOTS). */
468 5459465 ▶ for (uint32_t w = 0; w < HIST_OCC_WORDS; w++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t w = 0; w < HIST_OCC_WORDS; w++) { 5138320 321145 5459465
469 5138320 uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
470 5459468 ▶ while (word) {
Branch: 1 branch
BranchTrueFalseTotal
while (word) { 321148 5138320 5459468
471 321148 uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
472 321148 word &= word - 1; /* consume this bit (local copy) */
473 321148 uint32_t rd = __atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST);
474 321148 ▶ if (rd == 0) continue; /* occupied but not read-locking now */
Branch: 1 branch
BranchTrueFalseTotal
if (rd == 0) continue; /* occupied but not read-locking now */ 321148 0 321148
475 0 uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
476 0 ▶ if (pid == 0) continue; /* stale rdepth on a freed slot */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) continue; /* stale rdepth on a freed slot */ 0 0 0
477 0 ▶ if (!hist_pid_alive(pid)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_pid_alive(pid)) { 0 0 0
478 /* Dead reader: drop its pid so the slot no longer counts. Leave
479 * the occ bit set (harmless -- a later scan hits pid==0 and skips,
480 * a re-claim re-sets it) to avoid racing a concurrent claimant. */
481 0 uint32_t ep = pid;
482 0 __atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
483 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
484 0 continue;
485 }
486 0 busy = 1; /* live reader still holding */
487 }
488 }
489 /* A live slotless reader keeps us waiting; a crashed slotless reader that
490 * cannot be attributed to a pid is the documented slotless limitation. */
491 321145 ▶ if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0) 0 321145 321145
492 0 busy = 1;
493 321145 ▶ if (!busy)
Branch: 1 branch
BranchTrueFalseTotal
if (!busy) 321145 0 321145
494 321145 return; /* exclusive: wlock held + every rdepth 0 */
495 /* Wait for a reader to release (drain_seq bump) or time out to re-scan
496 * (which reclaims any newly-dead slotted reader). */
497 0 syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, &hist_lock_timeout, NULL, 0);
498 }
499 }
500  
501 321145 static inline void hist_rwlock_wrunlock(HistHandle *h) {
502 321145 HistHeader *hdr = h->hdr;
503 321145 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
504 321145 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 0 321145 321145
505 0 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
506 321145 }
507  
508 /* ================================================================
509 * Layout math + create / open / destroy
510 *
511 * Layout: Header -> reader_slots[1024] -> occ bitmap -> counts[counts_len] (int64_t)
512 * ================================================================ */
513  
514 /* Single source of truth for the mmap region layout offsets:
515 * Header -> reader_slots[] -> occ bitmap -> counts[]. */
516 typedef struct { uint64_t reader_slots, occ, counts; } HistLayout;
517  
518 223 static inline HistLayout hist_layout(void) {
519 HistLayout L;
520 223 L.reader_slots = sizeof(HistHeader);
521 223 L.occ = L.reader_slots + (uint64_t)HIST_READER_SLOTS * sizeof(HistReaderSlot);
522 223 L.counts = L.occ + HIST_OCC_BYTES;
523 223 L.counts = (L.counts + 7) & ~(uint64_t)7; /* 8-byte align the counts array (int64_t words) */
524 223 return L;
525 }
526  
527 81 static inline uint64_t hist_total_size(int64_t counts_len) {
528 81 HistLayout L = hist_layout();
529 81 return L.counts + (uint64_t)counts_len * sizeof(int64_t); /* counts_len int64_t cells */
530 }
531  
532 330603 static inline int64_t *hist_counts(HistHandle *h) {
533 330603 return (int64_t *)((char *)h->base + h->counts_off);
534 }
535  
536 /* ---- Layer B: at-use bounds for attacker-controlled file-stored values ----
537 * The backing file is mmap'd MAP_SHARED, so a local peer with write access can
538 * mutate counts_off / counts_len live, AFTER the open-time header validation.
539 * Anchor every counts[] index/length on the process-local mmap_size (kept in
540 * our PRIVATE handle, not the shared segment, hence trustworthy). For a valid
541 * histogram counts_off == the layout constant and counts_off + counts_len*8 ==
542 * mmap_size, so all clamps below are never-taken branches in normal use. */
543  
544 /* int64 count cells that actually fit in our mapping given (untrusted) counts_off. */
545 330603 static inline int64_t hist_counts_capacity(HistHandle *h) {
546 330603 uint64_t off = h->counts_off;
547 330603 ▶ if (off > (uint64_t)h->mmap_size) return 0; /* wild offset: nothing fits */
Branch: 1 branch
BranchTrueFalseTotal
if (off > (uint64_t)h->mmap_size) return 0; /* wild offset: nothing fits */ 0 330603 330603
548 330603 return (int64_t)(((uint64_t)h->mmap_size - off) / sizeof(int64_t));
549 }
550  
551 /* counts_len clamped to what fits (also rejects a negative/huge stored len). */
552 6058 static inline int64_t hist_counts_len_safe(HistHandle *h) {
553 6058 int64_t cap = hist_counts_capacity(h);
554 6058 int64_t len = h->hdr->counts_len;
555 6058 ▶ return (len < 0 || len > cap) ? cap : len;
Branch: 2 branches
BranchTrueFalseTotal
return (len < 0 || len > cap) ? cap : len; 6058 0 6058
return (len < 0 || len > cap) ? cap : len; 0 6058 6058
556 }
557  
558 /* ================================================================
559 * HdrHistogram geometry -- canonical formulas (see HdrHistogram_c).
560 * All derived fields are computed once here and stored in the header.
561 * ================================================================ */
562  
563 typedef struct {
564 int64_t lowest;
565 int64_t highest;
566 int32_t sig_figs;
567 int32_t unit_magnitude;
568 int32_t sub_bucket_count_magnitude;
569 int32_t sub_bucket_half_count_magnitude;
570 int32_t sub_bucket_count;
571 int32_t sub_bucket_half_count;
572 int64_t sub_bucket_mask;
573 int32_t bucket_count;
574 int64_t counts_len;
575 } HistGeometry;
576  
577 /* Validate args + compute the full geometry. Single source of truth: the XS
578 * layer does NOT duplicate these range checks. */
579 88 static int hist_validate_create_args(int64_t lowest, int64_t highest, int32_t sig_figs,
580 HistGeometry *g, char *errbuf) {
581 88 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 77 11 88
582 88 ▶ if (lowest < 1) { HIST_ERR("lowest must be >= 1"); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (lowest < 1) { HIST_ERR("lowest must be >= 1"); return 0; } 2 86 88
if (lowest < 1) { HIST_ERR("lowest must be >= 1"); return 0; } 2 0 2
583 86 ▶ if (highest < 2 * lowest) { HIST_ERR("highest must be >= 2 * lowest"); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (highest < 2 * lowest) { HIST_ERR("highest must be >= 2 * lowest"); return 0; } 1 85 86
if (highest < 2 * lowest) { HIST_ERR("highest must be >= 2 * lowest"); return 0; } 1 0 1
584 85 ▶ if (sig_figs < HIST_MIN_SIG || sig_figs > HIST_MAX_SIG) {
Branch: 2 branches
BranchTrueFalseTotal
if (sig_figs < HIST_MIN_SIG || sig_figs > HIST_MAX_SIG) { 84 1 85
if (sig_figs < HIST_MIN_SIG || sig_figs > HIST_MAX_SIG) { 2 82 84
585 3 ▶ HIST_ERR("sig_figs must be between %d and %d", HIST_MIN_SIG, HIST_MAX_SIG); return 0;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("sig_figs must be between %d and %d", HIST_MIN_SIG, HIST_MAX_SIG); return 0; 3 0 3
586 }
587  
588 82 int32_t unit_magnitude = (int32_t)floor(log2((double)lowest));
589 82 int32_t sbc_magnitude = (int32_t)ceil(log2(2.0 * pow(10.0, (double)sig_figs)));
590 82 ▶ if (sbc_magnitude < 1) sbc_magnitude = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (sbc_magnitude < 1) sbc_magnitude = 1; 0 82 82
591 82 int32_t shc_magnitude = sbc_magnitude - 1;
592 82 ▶ if (unit_magnitude + shc_magnitude > 61) {
Branch: 1 branch
BranchTrueFalseTotal
if (unit_magnitude + shc_magnitude > 61) { 1 81 82
593 1 ▶ HIST_ERR("lowest too large for sig_figs (unit_magnitude %d + sub_bucket_half_count_magnitude %d exceeds 61)", unit_magnitude, shc_magnitude);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("lowest too large for sig_figs (unit_magnitude %d + sub_bucket_half_count_magnitude %d exceeds 61)", unit_magnitude, shc_magnitude); 1 0 1
594 1 return 0;
595 }
596 81 int32_t sub_bucket_count = (int32_t)(1 << sbc_magnitude);
597 81 int32_t sub_bucket_half_count = sub_bucket_count / 2;
598 81 int64_t sub_bucket_mask = ((int64_t)sub_bucket_count - 1) << unit_magnitude;
599  
600 /* bucket_count: smallest count of buckets covering 'highest' */
601 81 int64_t smallest_untrackable = (int64_t)sub_bucket_count << unit_magnitude;
602 81 int32_t bucket_count = 1;
603 750 ▶ while (smallest_untrackable <= highest) {
Branch: 1 branch
BranchTrueFalseTotal
while (smallest_untrackable <= highest) { 669 81 750
604 669 ▶ if (smallest_untrackable > (INT64_MAX / 2)) { bucket_count++; break; }
Branch: 1 branch
BranchTrueFalseTotal
if (smallest_untrackable > (INT64_MAX / 2)) { bucket_count++; break; } 0 669 669
605 669 smallest_untrackable <<= 1;
606 669 bucket_count++;
607 }
608 81 int64_t counts_len = (int64_t)(bucket_count + 1) * sub_bucket_half_count;
609  
610 81 g->lowest = lowest;
611 81 g->highest = highest;
612 81 g->sig_figs = sig_figs;
613 81 g->unit_magnitude = unit_magnitude;
614 81 g->sub_bucket_count_magnitude = sbc_magnitude;
615 81 g->sub_bucket_half_count_magnitude = shc_magnitude;
616 81 g->sub_bucket_count = sub_bucket_count;
617 81 g->sub_bucket_half_count = sub_bucket_half_count;
618 81 g->sub_bucket_mask = sub_bucket_mask;
619 81 g->bucket_count = bucket_count;
620 81 g->counts_len = counts_len;
621 81 return 1;
622 }
623  
624 61 static inline void hist_init_header(void *base, const HistGeometry *g, uint64_t total_size) {
625 61 HistLayout L = hist_layout();
626 61 HistHeader *hdr = (HistHeader *)base;
627 /* Explicitly zero the header + reader-slot region (lock-recovery state);
628 the counts array relies on the fresh mapping being OS zero-filled. */
629 61 memset(base, 0, (size_t)L.counts);
630 61 hdr->version = HIST_VERSION;
631 61 hdr->lowest = g->lowest;
632 61 hdr->highest = g->highest;
633 61 hdr->sig_figs = g->sig_figs;
634 61 hdr->unit_magnitude = g->unit_magnitude;
635 61 hdr->sub_bucket_count_magnitude = g->sub_bucket_count_magnitude;
636 61 hdr->sub_bucket_half_count_magnitude = g->sub_bucket_half_count_magnitude;
637 61 hdr->sub_bucket_count = g->sub_bucket_count;
638 61 hdr->sub_bucket_half_count = g->sub_bucket_half_count;
639 61 hdr->sub_bucket_mask = g->sub_bucket_mask;
640 61 hdr->bucket_count = g->bucket_count;
641 61 hdr->counts_len = g->counts_len;
642 61 hdr->total_count = 0;
643 61 hdr->min_value = INT64_MAX;
644 61 hdr->max_value = 0;
645 61 hdr->total_size = total_size;
646 61 hdr->reader_slots_off = L.reader_slots;
647 61 hdr->counts_off = L.counts;
648 /* Publish magic LAST, as a release store: it is the commit point, so a
649 creator killed before it leaves magic==0 and never a file mistaken for
650 a valid one. A kill during the field stores leaves one to remove by
651 hand. */
652 61 __atomic_store_n(&hdr->magic, HIST_MAGIC, __ATOMIC_RELEASE);
653 61 __atomic_thread_fence(__ATOMIC_SEQ_CST);
654 61 }
655  
656 70 static inline HistHandle *hist_setup(void *base, size_t map_size,
657 const char *path, int backing_fd) {
658 70 HistHeader *hdr = (HistHeader *)base;
659 70 HistHandle *h = (HistHandle *)calloc(1, sizeof(HistHandle));
660 70 ▶ if (!h) {
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { 0 70 70
661 0 munmap(base, map_size);
662 0 ▶ if (backing_fd >= 0) close(backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (backing_fd >= 0) close(backing_fd); 0 0 0
663 0 return NULL;
664 }
665 70 h->hdr = hdr;
666 70 h->base = base;
667 70 h->reader_slots = (HistReaderSlot *)((uint8_t *)base + sizeof(HistHeader)); /* trusted layout, not the peer-writable header offset */
668 70 h->occ = (uint64_t *)((uint8_t *)base + hist_layout().occ); /* trusted layout offset */
669 70 h->mmap_size = map_size;
670 70 h->counts_off = hdr->counts_off; /* validated at open (== L.counts); cache so the bound and the pointer use one value */
671 70 ▶ h->path = path ? strdup(path) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->path = path ? strdup(path) : NULL; 17 53 70
672 70 h->backing_fd = backing_fd;
673 70 h->my_slot_idx = UINT32_MAX;
674 70 return h;
675 }
676  
677 /* Validate a mapped header (shared by hist_create reopen and hist_open_fd).
678 * Stored geometry wins on reopen; we re-derive it from lowest/highest/sig_figs
679 * and require every cached field to match, then require total_size == the size
680 * the geometry implies and == the actual file size. */
681 15 static inline int hist_validate_header(const HistHeader *hdr, uint64_t file_size) {
682 15 ▶ if (hdr->magic != HIST_MAGIC) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->magic != HIST_MAGIC) return 0; 4 11 15
683 11 ▶ if (hdr->version != HIST_VERSION) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->version != HIST_VERSION) return 0; 0 11 11
684 11 ▶ if (hdr->sig_figs < HIST_MIN_SIG || hdr->sig_figs > HIST_MAX_SIG) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->sig_figs < HIST_MIN_SIG || hdr->sig_figs > HIST_MAX_SIG) return 0; 11 0 11
if (hdr->sig_figs < HIST_MIN_SIG || hdr->sig_figs > HIST_MAX_SIG) return 0; 0 11 11
685 11 ▶ if (hdr->lowest < 1) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->lowest < 1) return 0; 0 11 11
686 11 ▶ if (hdr->highest < 2 * hdr->lowest) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->highest < 2 * hdr->lowest) return 0; 0 11 11
687  
688 HistGeometry g;
689 11 ▶ if (!hist_validate_create_args(hdr->lowest, hdr->highest, hdr->sig_figs, &g, NULL))
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_create_args(hdr->lowest, hdr->highest, hdr->sig_figs, &g, NULL)) 0 11 11
690 0 return 0;
691 11 ▶ if (hdr->unit_magnitude != g.unit_magnitude) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->unit_magnitude != g.unit_magnitude) return 0; 0 11 11
692 11 ▶ if (hdr->sub_bucket_count_magnitude != g.sub_bucket_count_magnitude) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sub_bucket_count_magnitude != g.sub_bucket_count_magnitude) return 0; 0 11 11
693 11 ▶ if (hdr->sub_bucket_half_count_magnitude != g.sub_bucket_half_count_magnitude) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sub_bucket_half_count_magnitude != g.sub_bucket_half_count_magnitude) return 0; 0 11 11
694 11 ▶ if (hdr->sub_bucket_count != g.sub_bucket_count) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sub_bucket_count != g.sub_bucket_count) return 0; 0 11 11
695 11 ▶ if (hdr->sub_bucket_half_count != g.sub_bucket_half_count) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sub_bucket_half_count != g.sub_bucket_half_count) return 0; 0 11 11
696 11 ▶ if (hdr->sub_bucket_mask != g.sub_bucket_mask) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->sub_bucket_mask != g.sub_bucket_mask) return 0; 0 11 11
697 11 ▶ if (hdr->bucket_count != g.bucket_count) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->bucket_count != g.bucket_count) return 0; 0 11 11
698 11 ▶ if (hdr->counts_len != g.counts_len) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->counts_len != g.counts_len) return 0; 0 11 11
699  
700 11 ▶ if (hdr->total_size != file_size) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->total_size != file_size) return 0; 0 11 11
701 11 ▶ if (hdr->total_size != hist_total_size(hdr->counts_len)) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->total_size != hist_total_size(hdr->counts_len)) return 0; 0 11 11
702 11 HistLayout L = hist_layout();
703 11 ▶ if (hdr->reader_slots_off != L.reader_slots) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->reader_slots_off != L.reader_slots) return 0; 0 11 11
704 11 ▶ if (hdr->counts_off != L.counts) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->counts_off != L.counts) return 0; 0 11 11
705 11 return 1;
706 }
707  
708 /* Securely obtain a fd: create exclusively (O_CREAT|O_EXCL|O_NOFOLLOW at mode,
709 * default 0600), or attach an existing file (O_RDWR|O_NOFOLLOW, no O_CREAT). */
710 18 static int hist_secure_open(const char *path, mode_t mode, char *errbuf) {
711 18 ▶ for (int attempt = 0; attempt < 100; attempt++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int attempt = 0; attempt < 100; attempt++) { 18 0 18
712 18 int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, mode);
713 18 ▶ if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */ 8 10 18
714 10 ▶ if (errno != EEXIST) { HIST_ERR("create %s: %s", path, strerror(errno)); return -1; }
Branch: 2 branches
BranchTrueFalseTotal
if (errno != EEXIST) { HIST_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 10 10
if (errno != EEXIST) { HIST_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 0 0
715 10 fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
716 10 ▶ if (fd >= 0) return fd;
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) return fd; 10 0 10
717 0 ▶ if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */
Branch: 1 branch
BranchTrueFalseTotal
if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */ 0 0 0
718 0 ▶ HIST_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */ 0 0 0
719 0 return -1;
720 }
721 0 ▶ HIST_ERR("open %s: create/attach kept racing", path);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("open %s: create/attach kept racing", path); 0 0 0
722 0 return -1;
723 }
724  
725 /* True iff the whole file is zero -- what an abandoned mid-init
726 creator leaves. Lets recovery re-init only a provably-empty file, never
727 one that merely starts with a zero word. Cold path, so a byte scan is
728 fine. Read, not mapped: a read fault on a tmpfs hole allocates the page. */
729 2 static int hist_file_is_zero(int fd, uint64_t size) {
730 char buf[16384];
731 15 ▶ for (uint64_t off = 0; off < size; ) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint64_t off = 0; off < size; ) { 14 1 15
732 14 ssize_t n = pread(fd, buf, size - off < sizeof buf ? (size_t)(size - off) : sizeof buf, (off_t)off);
733 14 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 14 14
734 213773 ▶ for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0;
Branch: 2 branches
BranchTrueFalseTotal
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 1 213759 213760
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 213760 13 213773
735 13 off += (uint64_t)n;
736 }
737 1 return 1;
738 }
739  
740 /* Opt-in (DATA_HISTOGRAM_SHARED_SPARSE=0): reserving commits memory this module otherwise fills lazily. */
741 11 static int hist_reserve(int fd, uint64_t size) {
742 11 const char *sp = getenv("DATA_HISTOGRAM_SHARED_SPARSE");
743 11 ▶ if (!sp || strcmp(sp, "0")) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (!sp || strcmp(sp, "0")) return 0; 0 11 11
if (!sp || strcmp(sp, "0")) return 0; 0 0 0
744 /* Before Linux 6.11 tmpfs gives up at any pending signal and undoes the allocation, so a
745 * periodic one could keep it from ever completing; SIGSTOP cannot be blocked. */
746 sigset_t all, old;
747 0 sigfillset(&all);
748 0 sigprocmask(SIG_BLOCK, &all, &old);
749 int e;
750 0 ▶ do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR);
Branch: 1 branch
BranchTrueFalseTotal
do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR); 0 0 0
751 0 sigprocmask(SIG_SETMASK, &old, NULL);
752 0 ▶ if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0;
Branch: 3 branches
BranchTrueFalseTotal
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 0 0
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 0 0
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 0 0
753 0 errno = e;
754 0 return -1;
755 }
756  
757 73 static HistHandle *hist_create(const char *path, int64_t lowest, int64_t highest,
758 int32_t sig_figs, mode_t mode, char *errbuf) {
759 HistGeometry g;
760 73 ▶ if (!hist_validate_create_args(lowest, highest, sig_figs, &g, errbuf)) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_create_args(lowest, highest, sig_figs, &g, errbuf)) return NULL; 5 68 73
761  
762 68 uint64_t total = hist_total_size(g.counts_len);
763 68 int anonymous = (path == NULL);
764 68 int fd = -1;
765 size_t map_size;
766 void *base;
767  
768 68 ▶ if (anonymous) {
Branch: 1 branch
BranchTrueFalseTotal
if (anonymous) { 50 18 68
769 50 map_size = (size_t)total;
770 50 base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
771 50 ▶ if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; } 0 50 50
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; } 0 0 0
772 } else {
773 18 fd = hist_secure_open(path, mode, errbuf);
774 28 ▶ if (fd < 0) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return NULL; 0 18 18
775 18 ▶ if (flock(fd, LOCK_EX) < 0) { HIST_ERR("flock: %s", strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (flock(fd, LOCK_EX) < 0) { HIST_ERR("flock: %s", strerror(errno)); close(fd); return NULL; } 0 18 18
if (flock(fd, LOCK_EX) < 0) { HIST_ERR("flock: %s", strerror(errno)); close(fd); return NULL; } 0 0 0
776 struct stat st;
777 18 ▶ if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 18 18
if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 0 0
778 18 int is_new = (st.st_size == 0);
779 18 ▶ if (!is_new && (uint64_t)st.st_size < sizeof(HistHeader)) {
Branch: 2 branches
BranchTrueFalseTotal
if (!is_new && (uint64_t)st.st_size < sizeof(HistHeader)) { 10 8 18
if (!is_new && (uint64_t)st.st_size < sizeof(HistHeader)) { 1 9 10
780 1 ▶ HIST_ERR("%s: file too small (%lld)", path, (long long)st.st_size);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: file too small (%lld)", path, (long long)st.st_size); 1 0 1
781 1 flock(fd, LOCK_UN); close(fd); return NULL;
782 }
783 17 ▶ if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) {
Branch: 3 branches
BranchTrueFalseTotal
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 8 9 17
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 8 0 8
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 0 8 8
784 0 ▶ HIST_ERR("%s: refusing to initialize file not owned by us", path);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: refusing to initialize file not owned by us", path); 0 0 0
785 0 flock(fd, LOCK_UN); close(fd); return NULL;
786 }
787 17 ▶ if (is_new && ftruncate(fd, (off_t)total) < 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (is_new && ftruncate(fd, (off_t)total) < 0) { 8 9 17
if (is_new && ftruncate(fd, (off_t)total) < 0) { 0 8 8
788 0 ▶ HIST_ERR("ftruncate: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("ftruncate: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; 0 0 0
789 }
790 17 ▶ if (is_new && hist_reserve(fd, total) < 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (is_new && hist_reserve(fd, total) < 0) { 8 9 17
if (is_new && hist_reserve(fd, total) < 0) { 0 8 8
791 0 ▶ HIST_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno)); 0 0 0
792 0 if (ftruncate(fd, 0) < 0) { /* best effort */ }
793 0 flock(fd, LOCK_UN); close(fd); return NULL;
794 }
795 17 ▶ map_size = is_new ? (size_t)total : (size_t)st.st_size;
Branch: 1 branch
BranchTrueFalseTotal
map_size = is_new ? (size_t)total : (size_t)st.st_size; 9 8 17
796 17 base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
797 17 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 17 17
798 0 ▶ HIST_ERR("mmap: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("mmap: %s", strerror(errno)); 0 0 0
799 0 ▶ if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ }
Branch: 1 branch
BranchTrueFalseTotal
if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ } 0 0 0
800 0 flock(fd, LOCK_UN); close(fd); return NULL;
801 }
802 17 ▶ if (!is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (!is_new) { 9 8 17
803 9 ▶ if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) { 4 5 9
804 /* Recover an abandoned mid-init file: a creator killed
805 * between the ftruncate and the header init leaves a
806 * full-size, all-zero (magic==0) file that would brick every
807 * future open of this path. Re-initialize ONLY when it is
808 * exactly our size, still uninitialized, and owned by us;
809 * anything else still errors. */
810 4 ▶ if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total
Branch: 2 branches
BranchTrueFalseTotal
if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total 3 1 4
if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total 2 1 3
811 2 ▶ && st.st_uid == geteuid() && hist_file_is_zero(fd, map_size)) {
Branch: 2 branches
BranchTrueFalseTotal
&& st.st_uid == geteuid() && hist_file_is_zero(fd, map_size)) { 2 0 2
&& st.st_uid == geteuid() && hist_file_is_zero(fd, map_size)) { 1 1 2
812 1 ▶ if (fchmod(fd, mode) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fchmod(fd, mode) < 0) { 0 1 1
813 0 ▶ HIST_ERR("%s: fchmod: %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: fchmod: %s", path, strerror(errno)); 0 0 0
814 0 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
815 }
816 1 ▶ if (hist_reserve(fd, total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (hist_reserve(fd, total) < 0) { 0 1 1
817 0 ▶ HIST_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno)); 0 0 0
818 0 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
819 }
820 1 hist_init_header(base, &g, total);
821 1 flock(fd, LOCK_UN); close(fd);
822 1 return hist_setup(base, map_size, path, -1);
823 }
824 3 ▶ if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total
Branch: 2 branches
BranchTrueFalseTotal
if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total 2 1 3
if (((HistHeader *)base)->magic == 0 && (uint64_t)st.st_size == total 1 1 2
825 1 ▶ && st.st_uid == geteuid()) {
Branch: 1 branch
BranchTrueFalseTotal
&& st.st_uid == geteuid()) { 1 0 1
826 1 ▶ HIST_ERR("%s: incomplete histogram file left by an interrupted create; remove it and retry", path);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: incomplete histogram file left by an interrupted create; remove it and retry", path); 1 0 1
827 1 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
828 }
829 2 ▶ HIST_ERR("invalid histogram file"); munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("invalid histogram file"); munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL; 2 0 2
830 }
831 5 ▶ if (__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) { 1 4 5
832 1 ▶ HIST_ERR("%s is frozen (read-only); open it with new_readonly", path);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s is frozen (read-only); open it with new_readonly", path); 1 0 1
833 1 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
834 }
835 4 flock(fd, LOCK_UN); close(fd);
836 4 return hist_setup(base, map_size, path, -1);
837 }
838 }
839 58 hist_init_header(base, &g, total);
840 58 ▶ if (fd >= 0) { flock(fd, LOCK_UN); close(fd); }
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { flock(fd, LOCK_UN); close(fd); } 8 50 58
841 58 return hist_setup(base, map_size, path, -1);
842 }
843  
844 4 static HistHandle *hist_create_memfd(const char *name, int64_t lowest, int64_t highest,
845 int32_t sig_figs, char *errbuf) {
846 HistGeometry g;
847 4 ▶ if (!hist_validate_create_args(lowest, highest, sig_figs, &g, errbuf)) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_create_args(lowest, highest, sig_figs, &g, errbuf)) return NULL; 2 2 4
848  
849 2 uint64_t total = hist_total_size(g.counts_len);
850 2 ▶ int fd = memfd_create(name ? name : "hist", MFD_CLOEXEC | MFD_ALLOW_SEALING);
Branch: 1 branch
BranchTrueFalseTotal
int fd = memfd_create(name ? name : "hist", MFD_CLOEXEC | MFD_ALLOW_SEALING); 1 1 2
851 2 ▶ if (fd < 0) { HIST_ERR("memfd_create: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fd < 0) { HIST_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 2 2
if (fd < 0) { HIST_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 0 0
852 2 ▶ if (ftruncate(fd, (off_t)total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)total) < 0) { 0 2 2
853 0 ▶ HIST_ERR("ftruncate: %s", strerror(errno)); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("ftruncate: %s", strerror(errno)); close(fd); return NULL; 0 0 0
854 }
855 2 ▶ if (hist_reserve(fd, total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (hist_reserve(fd, total) < 0) { 0 2 2
856 0 ▶ HIST_ERR("memfd: cannot reserve %llu bytes: %s", (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("memfd: cannot reserve %llu bytes: %s", (unsigned long long)total, strerror(errno)); 0 0 0
857 0 close(fd); return NULL;
858 }
859 2 (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);
860 2 void *base = mmap(NULL, (size_t)total, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
861 2 ▶ if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; } 0 2 2
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; } 0 0 0
862 2 hist_init_header(base, &g, total);
863 2 return hist_setup(base, (size_t)total, NULL, fd);
864 }
865  
866 2 static HistHandle *hist_open_fd(int fd, char *errbuf) {
867 2 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 2 0 2
868 struct stat st;
869 2 ▶ if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); return NULL; } 0 2 2
if (fstat(fd, &st) < 0) { HIST_ERR("fstat: %s", strerror(errno)); return NULL; } 0 0 0
870 2 ▶ if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("too small"); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("too small"); return NULL; } 1 1 2
if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("too small"); return NULL; } 1 0 1
871 /* before mmap: a creator may truncate the file until it has set magic */
872 uint32_t magic;
873 1 ▶ if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) { 1 0 1
if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) { 0 1 1
874 0 ▶ HIST_ERR("invalid histogram table"); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("invalid histogram table"); return NULL; 0 0 0
875 }
876 1 size_t ms = (size_t)st.st_size;
877 1 void *base = mmap(NULL, ms, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
878 1 ▶ if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; } 0 1 1
if (base == MAP_FAILED) { HIST_ERR("mmap: %s", strerror(errno)); return NULL; } 0 0 0
879 1 ▶ if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) { 0 1 1
880 0 ▶ HIST_ERR("invalid histogram table"); munmap(base, ms); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("invalid histogram table"); munmap(base, ms); return NULL; 0 0 0
881 }
882 1 ▶ if (__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) { 0 1 1
883 0 ▶ HIST_ERR("this histogram is frozen (read-only); open it with new_readonly");
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("this histogram is frozen (read-only); open it with new_readonly"); 0 0 0
884 0 munmap(base, ms); return NULL;
885 }
886 1 int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
887 1 ▶ if (myfd < 0) { HIST_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (myfd < 0) { HIST_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; } 0 1 1
if (myfd < 0) { HIST_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; } 0 0 0
888 1 return hist_setup(base, ms, NULL, myfd);
889 }
890  
891 70 static void hist_destroy(HistHandle *h) {
892 70 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 70 70
893 /* Release our reader slot on clean teardown (else short-lived-reader churn
894 * exhausts the slot table); skip if a read lock is still held (rdepth>0). */
895 70 ▶ if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid &&
Branch: 3 branches
BranchTrueFalseTotal
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 70 0 70
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 51 19 70
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 51 0 51
896 51 ▶ h->cached_fork_gen == __atomic_load_n(&hist_fork_gen, __ATOMIC_RELAXED) &&
Branch: 1 branch
BranchTrueFalseTotal
h->cached_fork_gen == __atomic_load_n(&hist_fork_gen, __ATOMIC_RELAXED) && 51 0 51
897 51 ▶ __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
__atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) { 51 0 51
898 /* Clear our occ bit BEFORE freeing the slot: we still own the pid so no
899 * claimant can take the slot mid-clear, and rdepth==0 so no writer needs
900 * to see us. (A crash skips this -> the bit is reclaimed lazily by a
901 * writer scan / re-claim, same as the pid.) */
902 51 hist_occ_clear(h, h->my_slot_idx);
903 51 uint32_t expected = h->cached_pid;
904 51 __atomic_compare_exchange_n(&h->reader_slots[h->my_slot_idx].pid,
905 &expected, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
906 }
907 70 ▶ if (h->backing_fd >= 0) close(h->backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (h->backing_fd >= 0) close(h->backing_fd); 3 67 70
908 70 ▶ if (h->base) munmap(h->base, h->mmap_size);
Branch: 1 branch
BranchTrueFalseTotal
if (h->base) munmap(h->base, h->mmap_size); 70 0 70
909 70 free(h->path);
910 70 free(h);
911 }
912  
913 4 static inline int hist_msync(HistHandle *h) {
914 4 ▶ if (!h || !h->base) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (!h || !h->base) return 0; 4 0 4
if (!h || !h->base) return 0; 0 4 4
915 4 return msync(h->base, h->mmap_size, MS_SYNC);
916 }
917  
918 /* Open a FROZEN (sealed) file read-only: O_RDONLY + PROT_READ, no lock ever.
919 * A sealed file is immutable and no read path writes the mapping, so queries
920 * take no reader-slot / rwlock traffic and any number of processes can share
921 * one PROT_READ mapping (same architecture; the magic rejects a wrong-endian
922 * file). */
923 6 static HistHandle *hist_open_readonly(const char *path, char *errbuf) {
924 6 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 6 0 6
925 6 int fd = open(path, O_RDONLY|O_NOFOLLOW|O_CLOEXEC);
926 6 ▶ if (fd < 0) { HIST_ERR("open %s: %s", path, strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fd < 0) { HIST_ERR("open %s: %s", path, strerror(errno)); return NULL; } 1 5 6
if (fd < 0) { HIST_ERR("open %s: %s", path, strerror(errno)); return NULL; } 1 0 1
927 struct stat st;
928 5 ▶ if (fstat(fd, &st) < 0) { HIST_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { HIST_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; } 0 5 5
if (fstat(fd, &st) < 0) { HIST_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; } 0 0 0
929 5 ▶ if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("%s: file too small", path); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("%s: file too small", path); close(fd); return NULL; } 0 5 5
if ((uint64_t)st.st_size < sizeof(HistHeader)) { HIST_ERR("%s: file too small", path); close(fd); return NULL; } 0 0 0
930 /* before mmap: a creator may truncate the file until it has set magic */
931 uint32_t magic;
932 5 ▶ if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) { 5 0 5
if (pread(fd, &magic, sizeof magic, offsetof(HistHeader, magic)) != (ssize_t)sizeof magic || magic != HIST_MAGIC) { 0 5 5
933 0 ▶ HIST_ERR("%s: invalid histogram file", path); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: invalid histogram file", path); close(fd); return NULL; 0 0 0
934 }
935 5 size_t ms = (size_t)st.st_size;
936 5 void *base = mmap(NULL, ms, PROT_READ, MAP_SHARED, fd, 0);
937 5 close(fd); /* the mapping keeps the file; a read-only view needs no fd (no msync/ftruncate) */
938 5 ▶ if (base == MAP_FAILED) { HIST_ERR("mmap %s: %s", path, strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { HIST_ERR("mmap %s: %s", path, strerror(errno)); return NULL; } 0 5 5
if (base == MAP_FAILED) { HIST_ERR("mmap %s: %s", path, strerror(errno)); return NULL; } 0 0 0
939 5 ▶ if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!hist_validate_header((HistHeader *)base, (uint64_t)st.st_size)) { 0 5 5
940 0 ▶ HIST_ERR("%s: invalid histogram file", path); munmap(base, ms); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s: invalid histogram file", path); munmap(base, ms); return NULL; 0 0 0
941 }
942 5 ▶ if (!__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_load_n(&((HistHeader *)base)->sealed, __ATOMIC_ACQUIRE)) { 1 4 5
943 1 ▶ HIST_ERR("%s is not frozen: call ->freeze on the producer before opening read-only", path);
Branch: 1 branch
BranchTrueFalseTotal
HIST_ERR("%s is not frozen: call ->freeze on the producer before opening read-only", path); 1 0 1
944 1 munmap(base, ms); return NULL;
945 }
946 4 HistHandle *h = hist_setup(base, ms, path, -1); /* munmaps on OOM */
947 4 ▶ if (!h) { HIST_ERR("out of memory"); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (!h) { HIST_ERR("out of memory"); return NULL; } 0 4 4
if (!h) { HIST_ERR("out of memory"); return NULL; } 0 0 0
948 4 h->readonly = 1;
949 4 return h;
950 }
951  
952 /* Seal a histogram: make it permanently immutable so it can be shipped and
953 * opened read-only. Takes the write lock so no record/record_many/merge/
954 * reset is in flight, publishes the seal, then flushes it (file/memfd-
955 * backed). Afterwards every mutator croaks and a read-write reopen is
956 * refused. */
957 2 static int hist_freeze(HistHandle *h) {
958 2 hist_rwlock_wrlock(h);
959 2 __atomic_store_n(&h->hdr->sealed, 1, __ATOMIC_RELEASE);
960 2 hist_rwlock_wrunlock(h);
961 2 ▶ if (h->path || h->backing_fd >= 0) return hist_msync(h); /* durability for file/memfd-backed */
Branch: 2 branches
BranchTrueFalseTotal
if (h->path || h->backing_fd >= 0) return hist_msync(h); /* durability for file/memfd-backed */ 0 2 2
if (h->path || h->backing_fd >= 0) return hist_msync(h); /* durability for file/memfd-backed */ 0 0 0
962 0 return 0; /* anonymous: the seal lives in shared memory (visible to forks); nothing to flush */
963 }
964  
965 /* ================================================================
966 * HdrHistogram index helpers (lock-free; pure functions of geometry)
967 *
968 * value | sub_bucket_mask is always >= 1 (sub_bucket_mask >= 1 since
969 * sub_bucket_count >= 2 and unit_magnitude >= 0), so __builtin_clzll is
970 * never called with 0.
971 * ================================================================ */
972  
973 688239 static inline int32_t hist_bucket_index(HistHandle *h, int64_t v) {
974 1376478 return (int32_t)((64 - __builtin_clzll((uint64_t)(v | h->hdr->sub_bucket_mask)))
975 688239 - h->hdr->unit_magnitude - (h->hdr->sub_bucket_half_count_magnitude + 1));
976 }
977  
978 688239 static inline int32_t hist_sub_bucket_index(HistHandle *h, int64_t v, int32_t bi) {
979 688239 return (int32_t)((uint64_t)v >> (bi + h->hdr->unit_magnitude));
980 }
981  
982 668641 static inline int64_t hist_counts_index(HistHandle *h, int32_t bi, int32_t sbi) {
983 668641 return ((int64_t)(bi + 1) << h->hdr->sub_bucket_half_count_magnitude)
984 668641 + (sbi - h->hdr->sub_bucket_half_count);
985 }
986  
987 344099 static inline int64_t hist_counts_index_for(HistHandle *h, int64_t v) {
988 344099 int32_t bi = hist_bucket_index(h, v);
989 344099 int32_t sbi = hist_sub_bucket_index(h, v, bi);
990 344099 return hist_counts_index(h, bi, sbi);
991 }
992  
993 /* reverse: lowest value stored at counts[index] */
994 39196 static inline int64_t hist_value_at_index(HistHandle *h, int64_t index) {
995 39196 int32_t bi = (int32_t)(index >> h->hdr->sub_bucket_half_count_magnitude) - 1;
996 39196 int32_t sbi = (int32_t)(index & (h->hdr->sub_bucket_half_count - 1)) + h->hdr->sub_bucket_half_count;
997 39196 ▶ if (bi < 0) { sbi -= h->hdr->sub_bucket_half_count; bi = 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (bi < 0) { sbi -= h->hdr->sub_bucket_half_count; bi = 0; } 19154 20042 39196
998 39196 return (int64_t)sbi << (bi + h->hdr->unit_magnitude);
999 }
1000  
1001 19598 static inline int64_t hist_size_of_equiv_range(HistHandle *h, int64_t v) {
1002 19598 int32_t bi = hist_bucket_index(h, v);
1003 19598 int32_t sbi = hist_sub_bucket_index(h, v, bi);
1004 19598 ▶ int32_t adj = (sbi >= h->hdr->sub_bucket_count) ? bi + 1 : bi;
Branch: 1 branch
BranchTrueFalseTotal
int32_t adj = (sbi >= h->hdr->sub_bucket_count) ? bi + 1 : bi; 0 19598 19598
1005 19598 return (int64_t)1 << (h->hdr->unit_magnitude + adj);
1006 }
1007  
1008 19598 static inline int64_t hist_lowest_equiv(HistHandle *h, int64_t v) {
1009 19598 return hist_value_at_index(h, hist_counts_index_for(h, v));
1010 }
1011  
1012 6049 static inline int64_t hist_highest_equiv(HistHandle *h, int64_t v) {
1013 6049 return hist_lowest_equiv(h, v) + hist_size_of_equiv_range(h, v) - 1;
1014 }
1015  
1016 13549 static inline int64_t hist_median_equiv(HistHandle *h, int64_t v) {
1017 13549 return hist_lowest_equiv(h, v) + (hist_size_of_equiv_range(h, v) >> 1);
1018 }
1019  
1020 /* Non-locking index resolver for the XS range-check before taking the lock.
1021 * Returns the counts index for v, or -1 if v falls outside the trackable
1022 * range (idx < 0 or idx >= counts_len). v must be >= 0. */
1023 324546 static inline int64_t hist_index_for(HistHandle *h, int64_t v) {
1024 324546 ▶ if (v > h->hdr->highest) return -1; /* documented croak contract: reject
Branch: 1 branch
BranchTrueFalseTotal
if (v > h->hdr->highest) return -1; /* documented croak contract: reject 4 324542 324546
1025 * values above highest that the last
1026 * bucket's pow2 span would else absorb */
1027 324542 int32_t bi = hist_bucket_index(h, v);
1028 324542 ▶ if (bi < 0 || bi >= h->hdr->bucket_count) return -1;
Branch: 2 branches
BranchTrueFalseTotal
if (bi < 0 || bi >= h->hdr->bucket_count) return -1; 324542 0 324542
if (bi < 0 || bi >= h->hdr->bucket_count) return -1; 0 324542 324542
1029 324542 int32_t sbi = hist_sub_bucket_index(h, v, bi);
1030 324542 int64_t idx = hist_counts_index(h, bi, sbi);
1031 324542 ▶ if (idx < 0 || idx >= h->hdr->counts_len) return -1;
Branch: 2 branches
BranchTrueFalseTotal
if (idx < 0 || idx >= h->hdr->counts_len) return -1; 324542 0 324542
if (idx < 0 || idx >= h->hdr->counts_len) return -1; 0 324542 324542
1032 324542 return idx;
1033 }
1034  
1035 /* ================================================================
1036 * HdrHistogram operations (callers hold the lock)
1037 * ================================================================ */
1038  
1039 /* Record `count` occurrences of `value`. The XS caller has ALREADY range-
1040 * checked 0 <= value <= highest and idx < counts_len before locking. */
1041 324501 static void hist_record_locked(HistHandle *h, int64_t value, int64_t count) {
1042 324501 int64_t idx = hist_counts_index_for(h, value);
1043 324501 ▶ if (idx < 0 || idx >= hist_counts_capacity(h)) return; /* Layer B: reject OOB idx (untrusted geometry) */
Branch: 2 branches
BranchTrueFalseTotal
if (idx < 0 || idx >= hist_counts_capacity(h)) return; /* Layer B: reject OOB idx (untrusted geometry) */ 324501 0 324501
if (idx < 0 || idx >= hist_counts_capacity(h)) return; /* Layer B: reject OOB idx (untrusted geometry) */ 0 324501 324501
1044 324501 int64_t *counts = hist_counts(h);
1045 324501 counts[idx] += count;
1046 324501 ▶ if (count != 0) { /* record(value, 0) records nothing -> no phantom min/max */
Branch: 1 branch
BranchTrueFalseTotal
if (count != 0) { /* record(value, 0) records nothing -> no phantom min/max */ 324501 0 324501
1047 324501 ▶ if (value < h->hdr->min_value) h->hdr->min_value = value;
Branch: 1 branch
BranchTrueFalseTotal
if (value < h->hdr->min_value) h->hdr->min_value = value; 45 324456 324501
1048 324501 ▶ if (value > h->hdr->max_value) h->hdr->max_value = value;
Branch: 1 branch
BranchTrueFalseTotal
if (value > h->hdr->max_value) h->hdr->max_value = value; 324471 30 324501
1049 }
1050 /* min() and the percentile walk trust the total, so it goes last. */
1051 324501 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1052 324501 h->hdr->total_count += count;
1053 }
1054  
1055 /* Highest equivalent value at or below which `p` percent of recorded values
1056 * lie. Returns 0 for an empty histogram. */
1057 6050 static int64_t hist_value_at_percentile_locked(HistHandle *h, double p) {
1058 6050 int64_t total = h->hdr->total_count;
1059 6050 ▶ if (total == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (total == 0) return 0; 1 6049 6050
1060 /* Allow for floating-point roundoff at integral ranks. */
1061 6049 double rank = p * (double)total / 100.0;
1062 6049 ▶ int64_t want = p >= 100.0 ? total : (int64_t)ceil(rank * (1.0 - DBL_EPSILON));
Branch: 1 branch
BranchTrueFalseTotal
int64_t want = p >= 100.0 ? total : (int64_t)ceil(rank * (1.0 - DBL_EPSILON)); 21 6028 6049
1063 6049 ▶ if (want < 1) want = 1;
Branch: 1 branch
BranchTrueFalseTotal
if (want < 1) want = 1; 6 6043 6049
1064 6049 ▶ if (want > total) want = total;
Branch: 1 branch
BranchTrueFalseTotal
if (want > total) want = total; 0 6049 6049
1065 6049 int64_t *counts = hist_counts(h);
1066 6049 int64_t running = 0;
1067 6049 int64_t len = hist_counts_len_safe(h); /* Layer B: never read past our mapping */
1068 1141036 ▶ for (int64_t idx = 0; idx < len; idx++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int64_t idx = 0; idx < len; idx++) { 1141036 0 1141036
1069 1141036 ▶ if (!counts[idx]) continue; /* skip empty cells (sparse); a 0 cell can never be the first to reach want */
Branch: 1 branch
BranchTrueFalseTotal
if (!counts[idx]) continue; /* skip empty cells (sparse); a 0 cell can never be the first to reach want */ 49188 1091848 1141036
1070 1091848 running += counts[idx];
1071 1091848 ▶ if (running >= want)
Branch: 1 branch
BranchTrueFalseTotal
if (running >= want) 6049 1085799 1091848
1072 6049 return hist_highest_equiv(h, hist_value_at_index(h, idx));
1073 }
1074 0 return 0;
1075 }
1076  
1077 /* Arithmetic mean of all recorded values (using each bucket's median-equivalent
1078 * value as the representative). Returns 0.0 for an empty histogram. */
1079 8 static double hist_mean_locked(HistHandle *h) {
1080 8 int64_t total = h->hdr->total_count;
1081 8 ▶ if (total == 0) return 0.0;
Branch: 1 branch
BranchTrueFalseTotal
if (total == 0) return 0.0; 0 8 8
1082 8 int64_t *counts = hist_counts(h);
1083 8 int64_t len = hist_counts_len_safe(h); /* Layer B: never read past our mapping */
1084 8 double sum = 0.0;
1085 90120 ▶ for (int64_t idx = 0; idx < len; idx++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int64_t idx = 0; idx < len; idx++) { 90112 8 90120
1086 90112 int64_t c = counts[idx];
1087 90112 ▶ if (c)
Branch: 1 branch
BranchTrueFalseTotal
if (c) 13549 76563 90112
1088 13549 sum += (double)c * (double)hist_median_equiv(h, hist_value_at_index(h, idx));
1089 }
1090 8 return sum / (double)total;
1091 }
1092  
1093 /* merge src counts into dst (caller guarantees equal geometry); cellwise add,
1094 * saturating at INT64_MAX on overflow (caller holds dst's write lock) */
1095 5 static void hist_merge_counts(int64_t *dst, const int64_t *src, int64_t counts_len) {
1096 56325 ▶ for (int64_t i = 0; i < counts_len; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int64_t i = 0; i < counts_len; i++) { 56320 5 56325
1097 56320 ▶ if (src[i] <= 0) continue; /* counts are non-negative; skip empty cells */
Branch: 1 branch
BranchTrueFalseTotal
if (src[i] <= 0) continue; /* counts are non-negative; skip empty cells */ 55238 1082 56320
1098 1082 ▶ if (dst[i] > INT64_MAX - src[i]) dst[i] = INT64_MAX; /* saturate */
Branch: 1 branch
BranchTrueFalseTotal
if (dst[i] > INT64_MAX - src[i]) dst[i] = INT64_MAX; /* saturate */ 0 1082 1082
1099 1082 else dst[i] += src[i];
1100 }
1101 5 }
1102  
1103 /* reset all counts to 0; reset total/min/max (caller holds the write lock) */
1104 1 static inline void hist_reset_locked(HistHandle *h) {
1105 1 int64_t len = hist_counts_len_safe(h); /* Layer B: never zero past our mapping */
1106 1 h->hdr->total_count = 0;
1107 1 h->hdr->min_value = INT64_MAX;
1108 1 h->hdr->max_value = 0;
1109 /* Empty the total first: a kill mid-memset leaves leftover counts, never a total they cannot reach. */
1110 1 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1111 1 memset(hist_counts(h), 0, (size_t)((uint64_t)len * sizeof(int64_t)));
1112 1 }
1113  
1114 #endif /* HIST_H */
« blib/lib/Data/Histogram/Shared.pm ↑ Summary Shared.xs »