sync.h

filter: Statementstmt 74.3% 736 / 990 Branchbran 53.6% 353 / 658 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 66.0% 1089 / 1648 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« Shared.xs ↑ Summary blib/lib/Data/Sync/Shared/Semaphore.pm »
1 /*
2 * sync.h -- Shared-memory synchronization primitives for Linux
3 *
4 * Five primitives:
5 * Semaphore -- bounded counter (CAS-based, cross-process resource limiting)
6 * Barrier -- N processes rendezvous at a point before proceeding
7 * RWLock -- reader-writer lock for external resources
8 * Condvar -- condition variable with futex wait/signal/broadcast
9 * Once -- one-time initialization gate (like pthread_once)
10 *
11 * All use file-backed mmap(MAP_SHARED) for cross-process sharing,
12 * futex for blocking wait, and PID-based stale lock recovery.
13 */
14  
15 #ifndef SYNC_H
16 #define SYNC_H
17  
18 #include <stddef.h>
19 #include <stdint.h>
20 #include <stdbool.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <fcntl.h>
25 #include <errno.h>
26 #include <time.h>
27 #include <limits.h>
28 #include <signal.h>
29 #include <sys/mman.h>
30 #include <sys/stat.h>
31 #include <sys/file.h>
32 #include <sys/syscall.h>
33 #include <linux/futex.h>
34 #include <sys/eventfd.h>
35 #include <pthread.h>
36  
37 /* ================================================================
38 * Constants
39 * ================================================================ */
40  
41 #define SYNC_MAGIC 0x53594E31U /* "SYN1" */
42 #define SYNC_VERSION 3 /* v3: on-disk format adds the RWLock reader-slot occupancy bitmap (layout change) */
43  
44 /* Primitive type IDs */
45 #define SYNC_TYPE_SEMAPHORE 0
46 #define SYNC_TYPE_BARRIER 1
47 #define SYNC_TYPE_RWLOCK 2
48 #define SYNC_TYPE_CONDVAR 3
49 #define SYNC_TYPE_ONCE 4
50  
51 #define SYNC_ERR_BUFLEN (PATH_MAX + 256)
52 #define SYNC_SPIN_LIMIT 32
53 #define SYNC_LOCK_TIMEOUT_SEC 2
54 #ifndef SYNC_READER_SLOTS
55 #define SYNC_READER_SLOTS 1024 /* per-process reader-counter mirror for RWLock */
56 #endif
57  
58 /* Occupancy bitmap (RWLock only): one bit per reader slot, set when a process
59 * claims its slot and cleared on clean release. A draining writer scans these
60 * SYNC_OCC_WORDS words to visit only OCCUPIED slots (O(words + live readers))
61 * instead of all SYNC_READER_SLOTS. Stored right after the reader-slot table. */
62 #define SYNC_OCC_WORDS (((SYNC_READER_SLOTS) + 63) / 64) /* 16 for 1024 slots */
63 #define SYNC_OCC_BYTES ((uint64_t)SYNC_OCC_WORDS * 8) /* 128 bytes */
64  
65 /* ================================================================
66 * Per-process reader-slot table (for RWLock dead-reader recovery)
67 *
68 * Allocated only when type == SYNC_TYPE_RWLOCK (Option A).
69 * ~16KB per RWLock (1024 slots * 16 bytes); zero overhead for other types.
70 *
71 * In the reader-slots-only rwlock a reader's ENTIRE contribution to the shared
72 * lock is `rdepth` in its OWN slot -- there is no separate shared reader counter
73 * to fall out of sync with it -- so a dead reader's contribution is exactly this
74 * one word, which a draining writer neutralises by clearing the slot's pid (the
75 * scan then ignores the slot). No orphaned counter can exist, so there is no
76 * quiescent force-reset and sustained readers cannot starve a writer. _rsv1/
77 * _rsv2 are kept only to preserve the 16-byte slot size across released builds.
78 * ================================================================ */
79  
80 typedef struct {
81 uint32_t pid; /* owning PID, 0 = free */
82 uint32_t rdepth; /* read-locks THIS process currently holds (recursion-safe) */
83 uint32_t _rsv1; /* reserved (was waiters_parked); unused, kept for layout size */
84 uint32_t _rsv2; /* reserved (was writers_parked); unused, kept for layout size */
85 } SyncReaderSlot;
86  
87 /* ================================================================
88 * Header (128 bytes = 2 cache lines, lives at start of mmap)
89 * ================================================================ */
90  
91 typedef struct {
92 /* ---- Cache line 0 (0-63): immutable after create ---- */
93 uint32_t magic; /* 0 */
94 uint32_t version; /* 4 */
95 uint32_t type; /* 8: SYNC_TYPE_* */
96 uint32_t param; /* 12: type-specific (sem max, barrier count, etc.) */
97 uint64_t total_size; /* 16: mmap size */
98 uint64_t reader_slots_off;/* 24: offset of SyncReaderSlot[SYNC_READER_SLOTS], 0 if not allocated (non-RWLock primitives) */
99 uint32_t slotless_rdepth; /* 32: RWLock live readers holding the lock with NO reader-slot
100 (claimed when the slot table was full). A draining writer
101 cannot attribute these to a pid, so it waits them out (the
102 documented slotless residual). Defaults to 0, so images from
103 before this field (was padding) stay compatible. */
104 uint8_t _pad0[28]; /* 36-63 */
105  
106 /* ---- Cache line 1 (64-127): mutable state ---- */
107  
108 /* Semaphore: value = current count, waiters = blocked acquirers */
109 /* Barrier: value = arrived count, waiters = blocked at barrier,
110 generation = increments each time barrier trips */
111 /* RWLock: value = WRITER word only (0=free, 0x80000000|pid=writer; readers
112 are NOT counted here -- each holds rdepth in its own reader-slot),
113 waiters = parked lockers (readers+writers) hint on the value futex */
114 /* Condvar: value = signal counter (futex word), waiters = blocked waiters,
115 mutex = associated mutex for predicate protection */
116 /* Once: value = state (0=INIT, 1=RUNNING|pid, 2=DONE),
117 waiters = blocked on completion */
118  
119 uint32_t value; /* 64: primary state word (futex target) */
120 uint32_t waiters; /* 68: waiter count */
121 uint32_t generation; /* 72: barrier generation / condvar epoch */
122 uint32_t mutex; /* 76: condvar mutex (0 or PID|0x80000000) */
123 uint32_t mutex_waiters; /* 80: condvar mutex / semaphore bulk waiter count */
124 uint32_t stat_recoveries;/* 84 */
125 uint64_t stat_acquires; /* 88 */
126 uint64_t stat_releases; /* 96 */
127 uint64_t stat_waits; /* 104 */
128 uint64_t stat_timeouts; /* 112 */
129 uint32_t stat_signals; /* 120 */
130 uint32_t drain_seq; /* 124: RWLock futex a releasing reader bumps to wake a
131 writer draining readers in wrlock Phase 2 */
132 } SyncHeader;
133  
134 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
135 _Static_assert(sizeof(SyncHeader) == 128, "SyncHeader must be 128 bytes");
136 #endif
137  
138 /* ================================================================
139 * Process-local handle
140 * ================================================================ */
141  
142 typedef struct {
143 SyncHeader *hdr;
144 size_t mmap_size;
145 char *path;
146 int notify_fd; /* eventfd, -1 if disabled */
147 int backing_fd; /* memfd fd, -1 for file-backed/anonymous */
148 SyncReaderSlot *reader_slots; /* in mmap, SYNC_READER_SLOTS entries; NULL if not RWLock */
149 uint64_t *occ; /* in mmap, SYNC_OCC_WORDS-word reader-slot occupancy bitmap; NULL if not RWLock (set alongside reader_slots) */
150 uint32_t my_slot_idx; /* UINT32_MAX = unclaimed; per-process slot index */
151 uint32_t cached_pid; /* getpid() at claim time */
152 uint32_t cached_fork_gen; /* fork-generation at claim time */
153 uint32_t slotless_held; /* rwlock read-locks this handle holds without a reader-slot */
154 } SyncHandle;
155  
156 /* ================================================================
157 * Utility
158 * ================================================================ */
159  
160 64 static inline void sync_spin_pause(void) {
161 #if defined(__x86_64__) || defined(__i386__)
162 64 __asm__ volatile("pause" ::: "memory");
163 #elif defined(__aarch64__)
164 __asm__ volatile("yield" ::: "memory");
165 #else
166 __asm__ volatile("" ::: "memory");
167 #endif
168 64 }
169  
170 /* A zombie (dead but not yet reaped) still answers kill(pid,0) as alive, so a
171 * process that crashed while holding the lock and lingers unreaped would never
172 * be recovered. Treat /proc/<pid>/stat state 'Z' as dead. Linux-only (as is
173 * this module); if /proc is unreadable we fall back to "alive" (safe: we never
174 * force-recover a possibly-live holder). */
175 69 static inline int sync_pid_is_zombie(uint32_t pid) {
176 char path[32], buf[256];
177 69 snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
178 69 int fd = open(path, O_RDONLY | O_CLOEXEC);
179 69 ▶ if (fd < 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return 0; 0 69 69
180 69 ssize_t n = read(fd, buf, sizeof(buf) - 1);
181 69 close(fd);
182 69 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 69 69
183 69 buf[n] = '\0';
184 /* "pid (comm) state ..."; comm may contain ')', so scan to the last one. */
185 69 char *rp = strrchr(buf, ')');
186 69 ▶ if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */
Branch: 2 branches
BranchTrueFalseTotal
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 69 0 69
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 0 69 69
187 69 ▶ return rp[1] == ' ' && rp[2] == 'Z';
Branch: 2 branches
BranchTrueFalseTotal
return rp[1] == ' ' && rp[2] == 'Z'; 69 0 69
return rp[1] == ' ' && rp[2] == 'Z'; 0 69 69
188 }
189 70 static inline int sync_pid_alive(uint32_t pid) {
190 70 ▶ if (pid == 0) return 0; /* no owner recorded, treat as dead */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) return 0; /* no owner recorded, treat as dead */ 0 70 70
191 70 ▶ if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */
Branch: 2 branches
BranchTrueFalseTotal
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 1 69 70
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 1 0 1
192 69 return !sync_pid_is_zombie(pid); /* kill() also succeeds for a zombie -> treat as dead */
193 }
194  
195 /* Convert timeout in seconds (double) to absolute deadline */
196 22 static inline void sync_make_deadline(double timeout, struct timespec *deadline) {
197 22 clock_gettime(CLOCK_MONOTONIC, deadline);
198 22 ▶ if (!(timeout < 1e9)) timeout = 1e9; /* clamp Inf/NaN/huge: avoid UB (time_t) cast -> instant spurious timeout */
Branch: 1 branch
BranchTrueFalseTotal
if (!(timeout < 1e9)) timeout = 1e9; /* clamp Inf/NaN/huge: avoid UB (time_t) cast -> instant spurious timeout */ 0 22 22
199 22 deadline->tv_sec += (time_t)timeout;
200 22 deadline->tv_nsec += (long)((timeout - (double)(time_t)timeout) * 1e9);
201 22 ▶ if (deadline->tv_nsec >= 1000000000L) {
Branch: 1 branch
BranchTrueFalseTotal
if (deadline->tv_nsec >= 1000000000L) { 5 17 22
202 5 deadline->tv_sec++;
203 5 deadline->tv_nsec -= 1000000000L;
204 }
205 22 }
206  
207 /* Compute remaining timespec from absolute deadline. Returns 0 if deadline passed. */
208 21 static inline int sync_remaining_time(const struct timespec *deadline,
209 struct timespec *remaining) {
210 struct timespec now;
211 21 clock_gettime(CLOCK_MONOTONIC, &now);
212 21 remaining->tv_sec = deadline->tv_sec - now.tv_sec;
213 21 remaining->tv_nsec = deadline->tv_nsec - now.tv_nsec;
214 21 ▶ if (remaining->tv_nsec < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (remaining->tv_nsec < 0) { 13 8 21
215 13 remaining->tv_sec--;
216 13 remaining->tv_nsec += 1000000000L;
217 }
218 21 return remaining->tv_sec >= 0;
219 }
220  
221 /* ================================================================
222 * Mutex helpers (for Condvar's internal mutex)
223 * ================================================================ */
224  
225 #define SYNC_MUTEX_WRITER_BIT 0x80000000U
226 #define SYNC_MUTEX_PID_MASK 0x7FFFFFFFU
227 #define SYNC_MUTEX_VAL(pid) (SYNC_MUTEX_WRITER_BIT | ((uint32_t)(pid) & SYNC_MUTEX_PID_MASK))
228  
229 static const struct timespec sync_lock_timeout = { SYNC_LOCK_TIMEOUT_SEC, 0 };
230  
231 0 static inline void sync_recover_stale_mutex(SyncHeader *hdr, uint32_t observed) {
232 0 ▶ if (!__atomic_compare_exchange_n(&hdr->mutex, &observed, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->mutex, &observed, 0, 0 0 0
233 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
234 0 return;
235 0 __atomic_add_fetch(&hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
236 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
237 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
238 0 ▶ if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0) 0 0 0
239 0 syscall(SYS_futex, &hdr->mutex, FUTEX_WAKE, 1, NULL, NULL, 0);
240 }
241  
242 17 static inline void sync_mutex_lock(SyncHeader *hdr) {
243 17 uint32_t mypid = SYNC_MUTEX_VAL((uint32_t)getpid());
244 17 for (int spin = 0; ; spin++) {
245 17 uint32_t expected = 0;
246 17 ▶ if (__atomic_compare_exchange_n(&hdr->mutex, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->mutex, &expected, mypid, 17 0 17
247 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
248 17 return;
249 0 ▶ if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) { 0 0 0
250 0 sync_spin_pause();
251 0 continue;
252 }
253 0 __atomic_add_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED);
254 /* StoreLoad: publish mutex_waiters++ before re-reading mutex, so an
255 * unlocker sees our registration or we see the unlock (cur==0 -> retry). */
256 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
257 0 uint32_t cur = __atomic_load_n(&hdr->mutex, __ATOMIC_RELAXED);
258 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
259 0 long rc = syscall(SYS_futex, &hdr->mutex, FUTEX_WAIT, cur,
260 &sync_lock_timeout, NULL, 0);
261 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
262 0 __atomic_sub_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED);
263 0 uint32_t val = __atomic_load_n(&hdr->mutex, __ATOMIC_RELAXED);
264 0 ▶ if (val >= SYNC_MUTEX_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= SYNC_MUTEX_WRITER_BIT) { 0 0 0
265 0 uint32_t pid = val & SYNC_MUTEX_PID_MASK;
266 0 ▶ if (!sync_pid_alive(pid))
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_pid_alive(pid)) 0 0 0
267 0 sync_recover_stale_mutex(hdr, val);
268 }
269 0 spin = 0;
270 0 continue;
271 }
272 }
273 0 __atomic_sub_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED);
274 0 spin = 0;
275 }
276 }
277  
278 18 static inline void sync_mutex_unlock(SyncHeader *hdr) {
279 18 __atomic_store_n(&hdr->mutex, 0, __ATOMIC_RELEASE);
280 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
281 18 __atomic_thread_fence(__ATOMIC_SEQ_CST);
282 18 ▶ if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0) 1 17 18
283 1 syscall(SYS_futex, &hdr->mutex, FUTEX_WAKE, 1, NULL, NULL, 0);
284 18 }
285  
286 /* ================================================================
287 * Futex-based write-preferring read-write lock (reader-slots-only)
288 * for SYNC_TYPE_RWLOCK, with dead-process recovery
289 *
290 * value == 0: unlocked (no writer)
291 * value == 0x80000000 | pid: write-locked by pid
292 *
293 * The reader count is NOT stored in `value`. It is DISTRIBUTED across
294 * per-process reader slots: each slot's `rdepth` is that process's entire
295 * contribution to the lock. A reader publishes its presence in its own slot and
296 * then re-checks `value`; a writer publishes `value` and then scans every slot
297 * until all live readers' rdepth reach 0. Sequentially-consistent store+load on
298 * each side (a Dekker handshake) gives mutual exclusion.
299 *
300 * Because a reader's whole contribution is ONE atomic word owned by ONE process,
301 * a crashed reader is recovered by clearing that one slot (CAS its pid to 0) --
302 * no second counter to strand, no orphaned +1, no quiescent force-reset. A
303 * reader killed anywhere in rdlock/rdunlock leaves at most `rdepth>0` in its dead
304 * slot, which the draining writer clears directly, so sustained read traffic can
305 * never starve a writer. Write-preference is inherent in the gate (new readers
306 * see value!=0 and yield).
307 * ================================================================ */
308  
309 #define SYNC_RWLOCK_WRITER_BIT 0x80000000U
310 #define SYNC_RWLOCK_PID_MASK 0x7FFFFFFFU
311 #define SYNC_RWLOCK_WR(pid) (SYNC_RWLOCK_WRITER_BIT | ((uint32_t)(pid) & SYNC_RWLOCK_PID_MASK))
312  
313 static inline int sync_rwlock_try_rdlock(SyncHandle *h);
314 static inline int sync_rwlock_try_wrlock(SyncHandle *h);
315  
316 /* Force-recover a stale WRITE lock left by a dead writer (held or mid-drain).
317 * A single CAS observed->0 wins the recovery race (a loser's CAS just fails);
318 * this module has no extra shared lock state to repair. Bumps stat_recoveries
319 * and wakes any lockers parked on the value futex. */
320 0 static inline void sync_recover_stale_rwlock(SyncHeader *hdr, uint32_t observed) {
321 0 ▶ if (!__atomic_compare_exchange_n(&hdr->value, &observed, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->value, &observed, 0, 0 0 0
322 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
323 0 return;
324 0 __atomic_add_fetch(&hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
325 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
326 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
327 0 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 0 0
328 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
329 }
330  
331 /* ---- Per-process reader-slot lifecycle (dead-reader recovery) ----
332 * Each process claims one SyncReaderSlot lazily on first rwlock op so that a
333 * SIGKILL'd reader's rdepth contribution can be neutralised by a draining
334 * writer. Only relevant for SYNC_TYPE_RWLOCK; non-RWLock primitives leave
335 * h->reader_slots == NULL and this helper is a no-op. */
336 static uint32_t sync_fork_gen = 0;
337 static pthread_once_t sync_atfork_once = PTHREAD_ONCE_INIT;
338 0 static void sync_on_fork_child(void) {
339 0 __atomic_add_fetch(&sync_fork_gen, 1, __ATOMIC_RELAXED);
340 0 }
341 2 static void sync_atfork_init(void) {
342 2 pthread_atfork(NULL, NULL, sync_on_fork_child);
343 2 }
344  
345 /* Occupancy bitmap: set a slot's bit when it is claimed, clear it on clean
346 * release. SEQ_CST so a set bit is ordered before that slot's rdepth can go
347 * non-zero (the bit is set in claim, which precedes any rdlock's rdepth++),
348 * letting a writer's SEQ_CST bitmap scan never miss a slot a committed reader
349 * holds. Callers guarantee h->occ != NULL (it is set alongside h->reader_slots,
350 * and every call site is under an `if (h->reader_slots)` guard). */
351 8 static inline void sync_occ_set(SyncHandle *h, uint32_t s) {
352 8 __atomic_fetch_or(&h->occ[s >> 6], (uint64_t)1 << (s & 63), __ATOMIC_SEQ_CST);
353 8 }
354 8 static inline void sync_occ_clear(SyncHandle *h, uint32_t s) {
355 8 __atomic_fetch_and(&h->occ[s >> 6], ~((uint64_t)1 << (s & 63)), __ATOMIC_SEQ_CST);
356 8 }
357  
358 26 static inline void sync_claim_reader_slot(SyncHandle *h) {
359 26 ▶ if (!h->reader_slots) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h->reader_slots) return; 0 26 26
360 26 pthread_once(&sync_atfork_once, sync_atfork_init);
361 26 uint32_t cur_gen = __atomic_load_n(&sync_fork_gen, __ATOMIC_RELAXED);
362 26 ▶ if (h->cached_fork_gen != cur_gen) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->cached_fork_gen != cur_gen) { 0 26 26
363 0 h->slotless_held = 0; /* fork: child holds none of the parent's slotless read locks */
364 0 h->cached_fork_gen = cur_gen;
365 0 h->my_slot_idx = UINT32_MAX;
366 }
367 26 ▶ if (h->my_slot_idx != UINT32_MAX) return;
Branch: 1 branch
BranchTrueFalseTotal
if (h->my_slot_idx != UINT32_MAX) return; 18 8 26
368 8 uint32_t now_pid = (uint32_t)getpid();
369 8 h->cached_pid = now_pid;
370 8 uint32_t start = now_pid % SYNC_READER_SLOTS;
371 /* Pass 1: take a free slot. */
372 8 ▶ for (uint32_t i = 0; i < SYNC_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < SYNC_READER_SLOTS; i++) { 8 0 8
373 8 uint32_t s = (start + i) % SYNC_READER_SLOTS;
374 8 uint32_t expected = 0;
375 8 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[s].pid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[s].pid, 8 0 8
376 &expected, now_pid, 0,
377 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
378 /* Fresh owner holds no read locks yet; clear any stale rdepth left by
379 * a dead predecessor (its contribution is dropped as we take over). */
380 8 __atomic_store_n(&h->reader_slots[s].rdepth, 0, __ATOMIC_RELAXED);
381 8 sync_occ_set(h, s); /* mark occupied BEFORE any rdlock can bump rdepth */
382 8 h->my_slot_idx = s;
383 8 return;
384 }
385 }
386 /* Pass 2: no free slot -- reclaim one whose owner is dead. Safe to take even
387 * if its rdepth>0: clearing pid drops the dead reader's entire contribution
388 * (a writer scan ignores rdepth when pid==0) and we reset rdepth to 0 as we
389 * claim it. */
390 0 ▶ for (uint32_t i = 0; i < SYNC_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < SYNC_READER_SLOTS; i++) { 0 0 0
391 0 uint32_t dpid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
392 0 ▶ if (dpid == 0 || dpid == now_pid || sync_pid_alive(dpid)) continue;
Branch: 3 branches
BranchTrueFalseTotal
if (dpid == 0 || dpid == now_pid || sync_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || sync_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || sync_pid_alive(dpid)) continue; 0 0 0
393 0 uint32_t expected = dpid;
394 0 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0, 0 0 0
395 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
396 0 __atomic_store_n(&h->reader_slots[i].rdepth, 0, __ATOMIC_RELAXED);
397 0 sync_occ_set(h, i);
398 0 h->my_slot_idx = i;
399 0 return;
400 }
401 }
402 /* Table full -- leave my_slot_idx = UINT32_MAX so this handle takes the
403 * slotless path (lock still works; recovery of THIS reader's death is the
404 * documented slotless limitation). */
405 }
406  
407 /* Inspect the writer word after a futex-wait timeout. If a dead writer holds
408 * it, force-recover. Dead READERS need no action here: only a writer that owns
409 * `value` drains readers, and it clears dead readers inline in its own scan. */
410 0 static inline void sync_recover_after_timeout(SyncHandle *h) {
411 0 SyncHeader *hdr = h->hdr;
412 0 uint32_t val = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
413 0 ▶ if (val >= SYNC_RWLOCK_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= SYNC_RWLOCK_WRITER_BIT) { 0 0 0
414 0 uint32_t pid = val & SYNC_RWLOCK_PID_MASK;
415 0 ▶ if (!sync_pid_alive(pid))
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_pid_alive(pid)) 0 0 0
416 0 sync_recover_stale_rwlock(hdr, val);
417 }
418 0 }
419  
420 /* Bump/drop the parked-waiter hint. Both readers (blocked at the gate) and
421 * writers (blocked acquiring `value`) wait on the value futex and use this, so
422 * wrunlock/recover know whether a FUTEX_WAKE is worth a syscall. A waiter
423 * SIGKILLed while parked leaves waiters over-counted -> at most a spurious wake
424 * (harmless); it can never under-count, so no wakeup is lost. */
425 2 static inline void sync_park(SyncHandle *h) {
426 2 __atomic_add_fetch(&h->hdr->waiters, 1, __ATOMIC_RELAXED);
427 2 }
428 2 static inline void sync_unpark(SyncHandle *h) {
429 2 __atomic_sub_fetch(&h->hdr->waiters, 1, __ATOMIC_RELAXED);
430 2 }
431  
432 /* Publish (inc) / retract (dec) this reader's presence -- its ENTIRE
433 * contribution to the lock. A slotted reader uses its slot's rdepth; a reader
434 * that could not claim a slot uses the global slotless_rdepth. inc() is SEQ_CST
435 * so the value re-check that follows it in rdlock forms a Dekker handshake with
436 * the writer's SEQ_CST value-store + rdepth-scan. dec() peels slotless first so
437 * a slot claimed mid-hold cannot misattribute the decrement. */
438 12 static inline void sync_rdepth_inc(SyncHandle *h) {
439 12 ▶ if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->my_slot_idx != UINT32_MAX) { 12 0 12
440 12 __atomic_add_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_SEQ_CST);
441 } else {
442 0 __atomic_add_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
443 0 h->slotless_held++;
444 }
445 12 }
446 12 static inline void sync_rdepth_dec(SyncHandle *h) {
447 12 ▶ if (h->slotless_held > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->slotless_held > 0) { 0 12 12
448 0 h->slotless_held--;
449 0 __atomic_sub_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_RELEASE);
450 12 ▶ } else if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (h->my_slot_idx != UINT32_MAX) { 12 0 12
451 12 __atomic_sub_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_RELEASE);
452 }
453 12 }
454  
455 /* Wake a writer that may be draining readers (it waits on drain_seq). Called
456 * after every rdepth decrement so a released read lock lets the writer re-scan
457 * promptly instead of waiting out its timeout. */
458 12 static inline void sync_reader_wake_drain(SyncHandle *h) {
459 12 ▶ if (__atomic_load_n(&h->hdr->value, __ATOMIC_ACQUIRE) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->hdr->value, __ATOMIC_ACQUIRE) != 0) { 0 12 12
460 0 __atomic_add_fetch(&h->hdr->drain_seq, 1, __ATOMIC_RELEASE);
461 0 syscall(SYS_futex, &h->hdr->drain_seq, FUTEX_WAKE, 1, NULL, NULL, 0);
462 }
463 12 }
464  
465 /* Give-up path for the timed/try readers: retract our published rdepth and wake
466 * a draining writer so it re-scans without our contribution. */
467 0 static inline void sync_reader_abort(SyncHandle *h) {
468 0 sync_rdepth_dec(h);
469 0 sync_reader_wake_drain(h);
470 0 }
471  
472 /* Writer-side scan of all reader slots: reclaim any dead reader (CAS its pid to
473 * 0) and report whether a LIVE reader still holds (rdepth>0), including the
474 * slotless residual. Shared by wrlock Phase 2, try_wrlock, and wrlock_timed. */
475 10 static inline int sync_rwlock_readers_busy(SyncHandle *h) {
476 10 int busy = 0;
477 10 ▶ if (h->reader_slots) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->reader_slots) { 10 0 10
478 /* Visit only OCCUPIED slots via the occupancy bitmap (SEQ_CST: a committed
479 * reader's bit -- set in claim, before its rdepth++ -- is ordered before
480 * this scan, so no held slot is skipped). O(SYNC_OCC_WORDS + live readers)
481 * instead of O(SYNC_READER_SLOTS). */
482 170 ▶ for (uint32_t w = 0; w < SYNC_OCC_WORDS; w++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t w = 0; w < SYNC_OCC_WORDS; w++) { 160 10 170
483 160 uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
484 171 ▶ while (word) {
Branch: 1 branch
BranchTrueFalseTotal
while (word) { 11 160 171
485 11 uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
486 11 word &= word - 1; /* consume this bit (local copy) */
487 11 uint32_t rd = __atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST);
488 11 ▶ if (rd == 0) continue; /* occupied but not read-locking now */
Branch: 1 branch
BranchTrueFalseTotal
if (rd == 0) continue; /* occupied but not read-locking now */ 10 1 11
489 1 uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
490 1 ▶ if (pid == 0) continue; /* no live owner -- stale rdepth, ignore */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) continue; /* no live owner -- stale rdepth, ignore */ 0 1 1
491 1 ▶ if (!sync_pid_alive(pid)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_pid_alive(pid)) { 0 1 1
492 /* Dead reader: clear its pid so the slot no longer counts (its
493 * whole contribution WAS this slot). Leave the occ bit SET
494 * (harmless -- a later scan hits pid==0 and skips, a re-claim
495 * re-sets it) to avoid racing a concurrent claimant. rdepth is
496 * left stale but is now ignored (pid==0) and zeroed by the next
497 * claimant. Count it as a recovery (gated on the CAS so it is
498 * tallied exactly once). */
499 0 uint32_t ep = pid;
500 0 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0, 0 0 0
501 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
502 0 __atomic_add_fetch(&h->hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
503 0 continue;
504 }
505 1 busy = 1; /* live reader still holding */
506 }
507 }
508 }
509 /* A live slotless reader keeps us waiting; a crashed slotless reader that
510 * cannot be attributed to a pid is the documented slotless limitation. */
511 10 ▶ if (__atomic_load_n(&h->hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0) 0 10 10
512 0 busy = 1;
513 10 return busy;
514 }
515  
516 7 static inline void sync_rwlock_rdlock(SyncHandle *h) {
517 7 sync_claim_reader_slot(h);
518 7 SyncHeader *hdr = h->hdr;
519 7 for (int spin = 0; ; spin++) {
520 7 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
521 7 ▶ if (cur == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) { 7 0 7
522 /* Optimistically take the read: publish rdepth, then re-check value.
523 * SEQ_CST inc + SEQ_CST load vs the writer's SEQ_CST value CAS +
524 * SEQ_CST rdepth scan: by the single total order of SEQ_CST ops the
525 * two sides cannot both miss each other, so we never hold
526 * concurrently with a writer. */
527 7 sync_rdepth_inc(h);
528 7 ▶ if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) == 0) 7 0 7
529 7 return; /* no writer after our publish -> we hold */
530 /* A writer appeared during our publish -- yield to it (write-preferring). */
531 0 sync_reader_abort(h); /* retract rdepth + wake the draining writer */
532 0 spin = 0;
533 0 continue;
534 }
535 /* value != 0: a writer holds or is acquiring. Recover if it is dead. */
536 0 if (cur >= SYNC_RWLOCK_WRITER_BIT &&
537 0 !sync_pid_alive(cur & SYNC_RWLOCK_PID_MASK)) {
538 0 sync_recover_stale_rwlock(hdr, cur);
539 0 spin = 0;
540 0 continue;
541 }
542 0 ▶ if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) { 0 0 0
543 0 sync_spin_pause();
544 0 continue;
545 }
546 0 sync_park(h);
547 /* StoreLoad: publish our parked registration before re-reading value
548 * (weak-memory lost-wakeup guard). */
549 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
550 0 cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
551 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
552 0 long rc = syscall(SYS_futex, &hdr->value, FUTEX_WAIT, cur,
553 &sync_lock_timeout, NULL, 0);
554 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
555 0 sync_unpark(h);
556 0 sync_recover_after_timeout(h);
557 0 spin = 0;
558 0 continue;
559 }
560 }
561 0 sync_unpark(h);
562 0 spin = 0;
563 }
564 }
565  
566 12 static inline void sync_rwlock_rdunlock(SyncHandle *h) {
567 12 sync_rdepth_dec(h); /* RELEASE: drop our entire contribution */
568 12 sync_reader_wake_drain(h); /* if a writer is draining, wake it to re-scan */
569 12 }
570  
571 /* try_rdlock: one-shot, non-blocking. Returns 1 on success, 0 on failure. */
572 3 static inline int sync_rwlock_try_rdlock(SyncHandle *h) {
573 3 sync_claim_reader_slot(h);
574 3 SyncHeader *hdr = h->hdr;
575 3 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
576 3 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 1 2 3
577 /* A writer holds or is acquiring. Recover once if it is dead, re-read. */
578 2 if (cur >= SYNC_RWLOCK_WRITER_BIT &&
579 1 !sync_pid_alive(cur & SYNC_RWLOCK_PID_MASK)) {
580 0 sync_recover_stale_rwlock(hdr, cur);
581 0 cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
582 }
583 1 ▶ if (cur != 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) return 0; 1 0 1
584 }
585 2 sync_rdepth_inc(h);
586 2 ▶ if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) != 0) { 0 2 2
587 /* A writer appeared during our publish -- yield (write-preferring). */
588 0 sync_reader_abort(h);
589 0 return 0;
590 }
591 2 return 1;
592 }
593  
594 /* Timed rdlock: returns 1 on success, 0 on timeout. timeout<0 = infinite,
595 * timeout==0 = one-shot try. Same optimistic gate as sync_rwlock_rdlock; the
596 * gate FUTEX_WAIT is bounded by an absolute deadline computed from `timeout`.
597 * On give-up we hold NOTHING (every 0-return has already retracted rdepth). */
598 3 static inline int sync_rwlock_rdlock_timed(SyncHandle *h, double timeout) {
599 3 ▶ if (timeout == 0) return sync_rwlock_try_rdlock(h);
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return sync_rwlock_try_rdlock(h); 0 3 3
600 3 sync_claim_reader_slot(h);
601 3 SyncHeader *hdr = h->hdr;
602 struct timespec deadline, remaining;
603 3 int has_deadline = (timeout > 0);
604 3 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 3 0 3
605 35 for (int spin = 0; ; spin++) {
606 35 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
607 35 ▶ if (cur == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) { 2 33 35
608 /* Optimistic publish + Dekker re-check (see sync_rwlock_rdlock). */
609 2 sync_rdepth_inc(h);
610 2 ▶ if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->value, __ATOMIC_SEQ_CST) == 0) 2 0 2
611 2 return 1;
612 0 sync_reader_abort(h); /* retract rdepth + wake the draining writer */
613 0 ▶ if (has_deadline && !sync_remaining_time(&deadline, &remaining))
Branch: 2 branches
BranchTrueFalseTotal
if (has_deadline && !sync_remaining_time(&deadline, &remaining)) 0 0 0
if (has_deadline && !sync_remaining_time(&deadline, &remaining)) 0 0 0
614 0 return 0; /* deadline passed; we hold nothing */
615 0 spin = 0;
616 0 continue;
617 }
618 66 if (cur >= SYNC_RWLOCK_WRITER_BIT &&
619 33 !sync_pid_alive(cur & SYNC_RWLOCK_PID_MASK)) {
620 0 sync_recover_stale_rwlock(hdr, cur);
621 0 spin = 0;
622 0 continue;
623 }
624 33 ▶ if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) { 32 1 33
625 32 sync_spin_pause();
626 32 continue;
627 }
628 1 sync_park(h);
629 /* StoreLoad: publish our parked registration before re-reading value. */
630 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
631 1 cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
632 1 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 1 0 1
633 /* Cap the wait at SYNC_LOCK_TIMEOUT_SEC so stale-holder recovery runs
634 * periodically even with a longer user deadline. */
635 1 struct timespec *pts = (struct timespec *)&sync_lock_timeout;
636 1 int capped = 1;
637 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
638 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 1 1
639 0 sync_unpark(h);
640 0 return 0; /* deadline passed; nothing held */
641 }
642 1 ▶ if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) { pts = &remaining; capped = 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) { pts = &remaining; capped = 0; } 1 0 1
643 }
644 1 long rc = syscall(SYS_futex, &hdr->value, FUTEX_WAIT, cur, pts, NULL, 0);
645 1 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 1 0 1
if (rc == -1 && errno == ETIMEDOUT) { 1 0 1
646 1 sync_unpark(h);
647 1 ▶ if (!capped) return 0; /* user deadline expired; nothing held */
Branch: 1 branch
BranchTrueFalseTotal
if (!capped) return 0; /* user deadline expired; nothing held */ 1 0 1
648 0 sync_recover_after_timeout(h);
649 0 spin = 0;
650 0 continue;
651 }
652 }
653 0 sync_unpark(h);
654 0 spin = 0;
655 }
656 }
657  
658 5 static inline void sync_rwlock_wrlock(SyncHandle *h) {
659 5 sync_claim_reader_slot(h); /* refresh cached_pid + own a slot across fork */
660 5 SyncHeader *hdr = h->hdr;
661 /* Encode PID in the value word itself (0x80000000 | pid) to eliminate any
662 * crash window between acquiring the lock and storing the owner. */
663 5 uint32_t mypid = SYNC_RWLOCK_WR((uint32_t)getpid());
664 /* Phase 1: acquire the writer word (mutual exclusion among writers). */
665 5 for (int spin = 0; ; spin++) {
666 5 uint32_t expected = 0;
667 5 ▶ if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid, 5 0 5
668 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
669 5 break;
670 /* Contended: expected now holds the current value. Recover a dead writer. */
671 0 if (expected >= SYNC_RWLOCK_WRITER_BIT &&
672 0 !sync_pid_alive(expected & SYNC_RWLOCK_PID_MASK)) {
673 0 sync_recover_stale_rwlock(hdr, expected);
674 0 spin = 0;
675 0 continue;
676 }
677 0 ▶ if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) { 0 0 0
678 0 sync_spin_pause();
679 0 continue;
680 }
681 0 sync_park(h);
682 /* StoreLoad: see the rdlock park path (weak-memory lost-wakeup guard). */
683 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
684 0 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
685 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
686 0 long rc = syscall(SYS_futex, &hdr->value, FUTEX_WAIT, cur,
687 &sync_lock_timeout, NULL, 0);
688 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
689 0 sync_unpark(h);
690 0 sync_recover_after_timeout(h);
691 0 spin = 0;
692 0 continue;
693 }
694 }
695 0 sync_unpark(h);
696 0 spin = 0;
697 }
698 /* Phase 2: we own `value`, so no NEW reader can join (they see value!=0 and
699 * yield). Drain the readers that were already holding when we won the CAS.
700 * The SEQ_CST CAS above + the SEQ_CST rdepth loads below are the writer side
701 * of the Dekker handshake. */
702 0 for (;;) {
703 5 uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_ACQUIRE); /* snapshot BEFORE scan */
704 5 ▶ if (!sync_rwlock_readers_busy(h))
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_rwlock_readers_busy(h)) 5 0 5
705 5 return; /* exclusive: value held + every rdepth 0 */
706 /* Wait for a reader to release (drain_seq bump) or time out to re-scan
707 * (which reclaims any newly-dead slotted reader). */
708 0 syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, &sync_lock_timeout, NULL, 0);
709 }
710 }
711  
712 8 static inline void sync_rwlock_wrunlock(SyncHandle *h) {
713 8 SyncHeader *hdr = h->hdr;
714 8 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
715 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
716 8 __atomic_thread_fence(__ATOMIC_SEQ_CST);
717 8 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 8 8
718 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
719 8 }
720  
721 /* try_wrlock: one-shot. Returns 1 iff we obtain exclusive ownership, 0
722 * otherwise. A try must NOT wait for readers to drain. */
723 3 static inline int sync_rwlock_try_wrlock(SyncHandle *h) {
724 3 sync_claim_reader_slot(h);
725 3 SyncHeader *hdr = h->hdr;
726 3 uint32_t mypid = SYNC_RWLOCK_WR((uint32_t)getpid());
727 3 uint32_t expected = 0;
728 3 ▶ if (!__atomic_compare_exchange_n(&hdr->value, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->value, &expected, mypid, 1 2 3
729 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED)) {
730 /* Contended. Recover a dead writer and retry once; else fail. */
731 2 if (expected >= SYNC_RWLOCK_WRITER_BIT &&
732 1 !sync_pid_alive(expected & SYNC_RWLOCK_PID_MASK)) {
733 0 sync_recover_stale_rwlock(hdr, expected);
734 0 expected = 0;
735 0 ▶ if (!__atomic_compare_exchange_n(&hdr->value, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->value, &expected, mypid, 0 0 0
736 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
737 0 return 0;
738 } else {
739 1 return 0;
740 }
741 }
742 /* We hold `value`. Dekker: our SEQ_CST CAS + the SEQ_CST rdepth loads in the
743 * scan cannot both miss a reader whose SEQ_CST publish + value re-check ran. */
744 2 __atomic_thread_fence(__ATOMIC_SEQ_CST);
745 2 ▶ if (sync_rwlock_readers_busy(h)) {
Branch: 1 branch
BranchTrueFalseTotal
if (sync_rwlock_readers_busy(h)) { 1 1 2
746 /* Readers still hold -- release the writer word and fail (no waiting). */
747 1 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
748 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
749 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 1 1
750 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
751 1 return 0;
752 }
753 1 return 1; /* exclusive: value held + every rdepth 0 */
754 }
755  
756 /* Timed wrlock: returns 1 on success, 0 on timeout. timeout<0 = infinite,
757 * timeout==0 = one-shot try. BOTH the Phase-1 value acquire and the Phase-2
758 * drain wait are bounded by an absolute deadline; on a Phase-2 give-up we own
759 * `value` and MUST release it before returning 0. */
760 4 static inline int sync_rwlock_wrlock_timed(SyncHandle *h, double timeout) {
761 4 ▶ if (timeout == 0) return sync_rwlock_try_wrlock(h);
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return sync_rwlock_try_wrlock(h); 0 4 4
762 4 sync_claim_reader_slot(h);
763 4 SyncHeader *hdr = h->hdr;
764 4 uint32_t mypid = SYNC_RWLOCK_WR((uint32_t)getpid());
765 struct timespec deadline, remaining;
766 4 int has_deadline = (timeout > 0);
767 4 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 4 0 4
768 /* Phase 1: acquire the writer word, bounded by the deadline. */
769 36 for (int spin = 0; ; spin++) {
770 36 uint32_t expected = 0;
771 36 ▶ if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid, 3 33 36
772 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
773 3 break;
774 66 if (expected >= SYNC_RWLOCK_WRITER_BIT &&
775 33 !sync_pid_alive(expected & SYNC_RWLOCK_PID_MASK)) {
776 0 sync_recover_stale_rwlock(hdr, expected);
777 0 spin = 0;
778 32 continue;
779 }
780 33 ▶ if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < SYNC_SPIN_LIMIT, 1)) { 32 1 33
781 32 sync_spin_pause();
782 32 continue;
783 }
784 1 sync_park(h);
785 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
786 1 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
787 1 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 1 0 1
788 1 struct timespec *pts = (struct timespec *)&sync_lock_timeout;
789 1 int capped = 1;
790 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
791 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 1 1
792 0 sync_unpark(h);
793 1 return 0; /* deadline passed; no lock held */
794 }
795 1 ▶ if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) { pts = &remaining; capped = 0; }
Branch: 1 branch
BranchTrueFalseTotal
if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) { pts = &remaining; capped = 0; } 1 0 1
796 }
797 1 long rc = syscall(SYS_futex, &hdr->value, FUTEX_WAIT, cur, pts, NULL, 0);
798 1 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 1 0 1
if (rc == -1 && errno == ETIMEDOUT) { 1 0 1
799 1 sync_unpark(h);
800 1 ▶ if (!capped) return 0; /* user deadline expired; no lock held */
Branch: 1 branch
BranchTrueFalseTotal
if (!capped) return 0; /* user deadline expired; no lock held */ 1 0 1
801 0 sync_recover_after_timeout(h);
802 0 spin = 0;
803 0 continue;
804 }
805 }
806 0 sync_unpark(h);
807 0 spin = 0;
808 }
809 /* Phase 2: we own `value`; drain existing readers, bounded by the deadline.
810 * On a deadline give-up we must release the writer word we hold. */
811 0 for (;;) {
812 3 uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_ACQUIRE);
813 3 ▶ if (!sync_rwlock_readers_busy(h))
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_rwlock_readers_busy(h)) 3 0 3
814 3 return 1; /* exclusive */
815 0 struct timespec *pts = (struct timespec *)&sync_lock_timeout;
816 0 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 0 0 0
817 0 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 0 0
818 /* Deadline hit while draining -- release the value word we own. */
819 0 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
820 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
821 0 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 0 0
822 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
823 0 return 0;
824 }
825 0 ▶ if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) pts = &remaining;
Branch: 1 branch
BranchTrueFalseTotal
if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) pts = &remaining; 0 0 0
826 }
827 0 syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, pts, NULL, 0);
828 }
829 }
830  
831 /* Downgrade: convert a held write lock (value == WRITER|pid) to a read lock.
832 * Publish our reader rdepth FIRST (SEQ_CST) so a writer that later CASes `value`
833 * sees our rdepth in its drain scan, THEN release the writer word. */
834 1 static inline void sync_rwlock_downgrade(SyncHandle *h) {
835 1 sync_claim_reader_slot(h);
836 1 SyncHeader *hdr = h->hdr;
837 1 sync_rdepth_inc(h); /* publish as a reader before releasing */
838 1 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
839 /* StoreLoad: publish the release before reading the waiter count (weak-memory lost-wakeup guard). */
840 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
841 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 1 1
842 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
843 1 }
844  
845 /* ================================================================
846 * Semaphore operations
847 *
848 * value = current count (0..param where param=max)
849 * CAS-based acquire/release, futex wait when 0
850 * ================================================================ */
851  
852 31 static inline int sync_sem_try_acquire(SyncHandle *h) {
853 31 SyncHeader *hdr = h->hdr;
854 0 for (;;) {
855 31 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
856 57 ▶ if (cur == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) return 0; 5 26 31
857 26 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - 1,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - 1, 26 0 26
858 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
859 26 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
860 26 return 1;
861 }
862 }
863 }
864  
865 10 static inline int sync_sem_try_acquire_n(SyncHandle *h, uint32_t n) {
866 10 ▶ if (n == 0) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (n == 0) return 1; 1 9 10
867 9 SyncHeader *hdr = h->hdr;
868 0 for (;;) {
869 9 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
870 14 ▶ if (cur < n) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (cur < n) return 0; 4 5 9
871 5 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - n,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - n, 5 0 5
872 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
873 5 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
874 5 return 1;
875 }
876 }
877 }
878  
879 3 static inline int sync_sem_acquire_n(SyncHandle *h, uint32_t n, double timeout) {
880 3 ▶ if (n == 0) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (n == 0) return 1; 0 3 3
881 3 ▶ if (sync_sem_try_acquire_n(h, n)) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (sync_sem_try_acquire_n(h, n)) return 1; 2 1 3
882 1 ▶ if (timeout == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return 0; 0 1 1
883  
884 1 SyncHeader *hdr = h->hdr;
885 struct timespec deadline, remaining;
886 1 int has_deadline = (timeout > 0);
887 1 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 1 0 1
888  
889 1 ▶ if (n > 1) __atomic_add_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED);
Branch: 1 branch
BranchTrueFalseTotal
if (n > 1) __atomic_add_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED); 1 0 1
890 1 __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);
891 1 int acquired = 0;
892  
893 0 for (;;) {
894 1 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
895 1 ▶ if (cur >= n) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur >= n) { 0 1 1
896 0 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - n,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - n, 0 0 0
897 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
898 0 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
899 0 acquired = 1;
900 0 break;
901 }
902 0 continue;
903 }
904  
905 1 __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
906  
907 1 struct timespec *pts = NULL;
908 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
909 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 1 1
910 0 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
911 0 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
912 0 break;
913 }
914 1 pts = &remaining;
915 }
916  
917 1 syscall(SYS_futex, &hdr->value, FUTEX_WAIT, cur, pts, NULL, 0);
918 1 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
919  
920 1 ▶ if (sync_sem_try_acquire_n(h, n)) {
Branch: 1 branch
BranchTrueFalseTotal
if (sync_sem_try_acquire_n(h, n)) { 0 1 1
921 0 acquired = 1;
922 0 break;
923 }
924  
925 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
926 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 1 0 1
927 1 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
928 1 break;
929 }
930 }
931 }
932 1 ▶ if (n > 1) __atomic_sub_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED);
Branch: 1 branch
BranchTrueFalseTotal
if (n > 1) __atomic_sub_fetch(&hdr->mutex_waiters, 1, __ATOMIC_RELAXED); 1 0 1
933 1 return acquired;
934 }
935  
936 4 static inline int sync_sem_acquire(SyncHandle *h, double timeout) {
937 4 ▶ if (sync_sem_try_acquire(h)) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (sync_sem_try_acquire(h)) return 1; 2 2 4
938 2 ▶ if (timeout == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return 0; 1 1 2
939  
940 1 SyncHeader *hdr = h->hdr;
941 struct timespec deadline, remaining;
942 1 int has_deadline = (timeout > 0);
943 1 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 1 0 1
944  
945 1 __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);
946  
947 0 for (;;) {
948 1 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
949 1 ▶ if (cur > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur > 0) { 0 1 1
950 0 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - 1,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, cur - 1, 0 0 0
951 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
952 0 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
953 1 return 1;
954 }
955 0 continue;
956 }
957  
958 1 __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
959  
960 1 struct timespec *pts = NULL;
961 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
962 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 1 1
963 0 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
964 0 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
965 0 return 0;
966 }
967 1 pts = &remaining;
968 }
969  
970 1 syscall(SYS_futex, &hdr->value, FUTEX_WAIT, 0, pts, NULL, 0);
971 1 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
972  
973 /* Retry acquire after wakeup */
974 1 ▶ if (sync_sem_try_acquire(h)) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (sync_sem_try_acquire(h)) return 1; 0 1 1
975  
976 1 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 1 0 1
977 1 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 1 0 1
978 1 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
979 1 return 0;
980 }
981 }
982 }
983 }
984  
985 8 static inline void sync_sem_release(SyncHandle *h) {
986 8 SyncHeader *hdr = h->hdr;
987 8 uint32_t max_val = hdr->param;
988 0 for (;;) {
989 8 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
990 8 uint32_t next = cur + 1;
991 8 ▶ if (next > max_val) next = max_val; /* clamp at max */
Branch: 1 branch
BranchTrueFalseTotal
if (next > max_val) next = max_val; /* clamp at max */ 1 7 8
992 8 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, next,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, next, 8 0 8
993 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
994 8 __atomic_add_fetch(&hdr->stat_releases, 1, __ATOMIC_RELAXED);
995 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
996 8 __atomic_thread_fence(__ATOMIC_SEQ_CST);
997 8 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_ACQUIRE) > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_ACQUIRE) > 0) { 1 7 8
998 /* A bulk request may be ineligible while a smaller waiter can run. */
999 1 ▶ int wake = __atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0 ? INT_MAX : 1;
Branch: 1 branch
BranchTrueFalseTotal
int wake = __atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0 ? INT_MAX : 1; 1 0 1
1000 1 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, wake, NULL, NULL, 0);
1001 }
1002 8 return;
1003 }
1004 }
1005 }
1006  
1007 15 static inline void sync_sem_release_n(SyncHandle *h, uint32_t n) {
1008 15 ▶ if (n == 0) return;
Branch: 1 branch
BranchTrueFalseTotal
if (n == 0) return; 1 14 15
1009 14 SyncHeader *hdr = h->hdr;
1010 14 uint32_t max_val = hdr->param;
1011 0 for (;;) {
1012 14 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_RELAXED);
1013 14 ▶ uint32_t next = (n > max_val - cur) ? max_val : cur + n;
Branch: 1 branch
BranchTrueFalseTotal
uint32_t next = (n > max_val - cur) ? max_val : cur + n; 10 4 14
1014 14 ▶ if (__atomic_compare_exchange_n(&hdr->value, &cur, next,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &cur, next, 14 0 14
1015 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
1016 14 __atomic_add_fetch(&hdr->stat_releases, 1, __ATOMIC_RELAXED);
1017 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1018 14 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1019 14 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_ACQUIRE) > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_ACQUIRE) > 0) { 5 9 14
1020 5 uint32_t wake = n < (uint32_t)INT_MAX ? n : INT_MAX;
1021 5 ▶ if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0) wake = INT_MAX;
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->mutex_waiters, __ATOMIC_RELAXED) > 0) wake = INT_MAX; 5 0 5
1022 5 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, wake, NULL, NULL, 0);
1023 }
1024 14 return;
1025 }
1026 }
1027 }
1028  
1029 /* Drain: acquire all available permits at once, return count acquired */
1030 4 static inline uint32_t sync_sem_drain(SyncHandle *h) {
1031 4 SyncHeader *hdr = h->hdr;
1032 4 uint32_t cur = __atomic_exchange_n(&hdr->value, 0, __ATOMIC_ACQUIRE);
1033 4 ▶ if (cur == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) return 0; 1 3 4
1034 3 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
1035 3 return cur;
1036 }
1037  
1038 61 static inline uint32_t sync_sem_value(SyncHandle *h) {
1039 61 return __atomic_load_n(&h->hdr->value, __ATOMIC_RELAXED);
1040 }
1041  
1042 /* ================================================================
1043 * Barrier operations
1044 *
1045 * param = number of parties
1046 * value = arrived count (0..param)
1047 * generation = bit 31: "broken" flag (set on timeout)
1048 * bits 0..30: generation counter (bumped each trip/reset)
1049 *
1050 * Timeout breaks the barrier permanently; all waiters return -1 and all
1051 * future wait() calls also return -1 until sync_barrier_reset() is called.
1052 * This avoids racing a timed-out waiter's reset with new-generation arrivals.
1053 * ================================================================ */
1054  
1055 #define SYNC_BARRIER_BROKEN_BIT 0x80000000U
1056 #define SYNC_BARRIER_GEN_MASK 0x7FFFFFFFU
1057  
1058 11 static inline int sync_barrier_wait(SyncHandle *h, double timeout) {
1059 11 SyncHeader *hdr = h->hdr;
1060 11 uint32_t parties = hdr->param;
1061  
1062 11 ▶ if (timeout == 0) return -1; /* non-blocking probe: can't rendezvous instantly */
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return -1; /* non-blocking probe: can't rendezvous instantly */ 1 10 11
1063  
1064 10 uint32_t gen_raw = __atomic_load_n(&hdr->generation, __ATOMIC_ACQUIRE);
1065 10 ▶ if (gen_raw & SYNC_BARRIER_BROKEN_BIT) return -1; /* already broken */
Branch: 1 branch
BranchTrueFalseTotal
if (gen_raw & SYNC_BARRIER_BROKEN_BIT) return -1; /* already broken */ 1 9 10
1066  
1067 9 uint32_t arrived = __atomic_add_fetch(&hdr->value, 1, __ATOMIC_ACQ_REL);
1068  
1069 9 ▶ if (arrived == parties) {
Branch: 1 branch
BranchTrueFalseTotal
if (arrived == parties) { 1 8 9
1070 /* Last to arrive -- trip the barrier. CAS preserves broken bit invariant. */
1071 1 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
1072 0 for (;;) {
1073 1 uint32_t old_g = __atomic_load_n(&hdr->generation, __ATOMIC_RELAXED);
1074 1 ▶ if (old_g & SYNC_BARRIER_BROKEN_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (old_g & SYNC_BARRIER_BROKEN_BIT) { 0 1 1
1075 0 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
1076 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1077 0 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1078 0 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 0 0
1079 0 syscall(SYS_futex, &hdr->generation, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1080 0 return -1;
1081 }
1082 1 uint32_t new_g = (old_g + 1) & SYNC_BARRIER_GEN_MASK;
1083 1 ▶ if (__atomic_compare_exchange_n(&hdr->generation, &old_g, new_g,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->generation, &old_g, new_g, 1 0 1
1084 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
1085 1 break;
1086 }
1087 1 __atomic_add_fetch(&hdr->stat_releases, 1, __ATOMIC_RELAXED);
1088 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1089 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1090 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 1 0 1
1091 1 syscall(SYS_futex, &hdr->generation, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1092 1 return 1; /* leader */
1093 }
1094  
1095 /* Not last -- wait for generation to change or broken bit to appear */
1096 8 __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);
1097  
1098 struct timespec deadline, remaining;
1099 8 int has_deadline = (timeout > 0);
1100 8 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 8 0 8
1101  
1102 8 for (;;) {
1103 16 uint32_t cur_raw = __atomic_load_n(&hdr->generation, __ATOMIC_ACQUIRE);
1104 16 ▶ if ((cur_raw & SYNC_BARRIER_GEN_MASK) != gen_raw) return 0; /* our cohort tripped */
Branch: 1 branch
BranchTrueFalseTotal
if ((cur_raw & SYNC_BARRIER_GEN_MASK) != gen_raw) return 0; /* our cohort tripped */ 5 11 16
1105 11 ▶ if (cur_raw & SYNC_BARRIER_BROKEN_BIT) return -1; /* broken */
Branch: 1 branch
BranchTrueFalseTotal
if (cur_raw & SYNC_BARRIER_BROKEN_BIT) return -1; /* broken */ 0 11 11
1106  
1107 11 __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1108  
1109 11 struct timespec *pts = NULL;
1110 11 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 11 0 11
1111 11 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 3 8 11
1112 3 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1113 /* A later cohort's timeout cannot break our completed cohort. */
1114 3 uint32_t g = gen_raw;
1115 3 if (!__atomic_compare_exchange_n(&hdr->generation, &g,
1116 3 ▶ gen_raw | SYNC_BARRIER_BROKEN_BIT,
Branch: 1 branch
BranchTrueFalseTotal
gen_raw | SYNC_BARRIER_BROKEN_BIT, 0 3 3
1117 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)
1118 0 ▶ && (g & SYNC_BARRIER_GEN_MASK) != gen_raw) {
Branch: 1 branch
BranchTrueFalseTotal
&& (g & SYNC_BARRIER_GEN_MASK) != gen_raw) { 0 0 0
1119 0 return 0;
1120 }
1121 3 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
1122 3 syscall(SYS_futex, &hdr->generation, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1123 3 return -1;
1124 }
1125 8 pts = &remaining;
1126 }
1127  
1128 8 syscall(SYS_futex, &hdr->generation, FUTEX_WAIT, gen_raw, pts, NULL, 0);
1129 8 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1130 }
1131 }
1132  
1133 10 static inline uint32_t sync_barrier_generation(SyncHandle *h) {
1134 10 return __atomic_load_n(&h->hdr->generation, __ATOMIC_RELAXED) & SYNC_BARRIER_GEN_MASK;
1135 }
1136  
1137 8 static inline uint32_t sync_barrier_arrived(SyncHandle *h) {
1138 8 return __atomic_load_n(&h->hdr->value, __ATOMIC_RELAXED);
1139 }
1140  
1141 3 static inline int sync_barrier_is_broken(SyncHandle *h) {
1142 3 return (__atomic_load_n(&h->hdr->generation, __ATOMIC_RELAXED)
1143 3 & SYNC_BARRIER_BROKEN_BIT) != 0;
1144 }
1145  
1146 2 static inline void sync_barrier_reset(SyncHandle *h) {
1147 2 SyncHeader *hdr = h->hdr;
1148 2 __atomic_store_n(&hdr->value, 0, __ATOMIC_RELEASE);
1149 /* Bump gen and clear broken bit in one CAS */
1150 0 for (;;) {
1151 2 uint32_t old_g = __atomic_load_n(&hdr->generation, __ATOMIC_RELAXED);
1152 2 uint32_t new_g = ((old_g & SYNC_BARRIER_GEN_MASK) + 1) & SYNC_BARRIER_GEN_MASK;
1153 2 ▶ if (__atomic_compare_exchange_n(&hdr->generation, &old_g, new_g,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->generation, &old_g, new_g, 2 0 2
1154 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
1155 2 break;
1156 }
1157 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1158 2 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1159 2 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 2 2
1160 0 syscall(SYS_futex, &hdr->generation, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1161 2 }
1162  
1163 /* ================================================================
1164 * Condvar operations
1165 *
1166 * Uses the internal mutex (hdr->mutex) to protect the predicate.
1167 * value = signal counter (futex word)
1168 * generation = broadcast epoch
1169 * ================================================================ */
1170  
1171 13 static inline void sync_condvar_lock(SyncHandle *h) {
1172 13 sync_mutex_lock(h->hdr);
1173 13 __atomic_add_fetch(&h->hdr->stat_acquires, 1, __ATOMIC_RELAXED);
1174 13 }
1175  
1176 14 static inline void sync_condvar_unlock(SyncHandle *h) {
1177 14 sync_mutex_unlock(h->hdr);
1178 14 __atomic_add_fetch(&h->hdr->stat_releases, 1, __ATOMIC_RELAXED);
1179 14 }
1180  
1181 1 static inline int sync_condvar_try_lock(SyncHandle *h) {
1182 1 SyncHeader *hdr = h->hdr;
1183 1 uint32_t mypid = SYNC_MUTEX_VAL((uint32_t)getpid());
1184 1 uint32_t expected = 0;
1185 1 ▶ if (__atomic_compare_exchange_n(&hdr->mutex, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->mutex, &expected, mypid, 1 0 1
1186 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
1187 1 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
1188 1 return 1;
1189 }
1190 0 return 0;
1191 }
1192  
1193 /* Wait: atomically unlock mutex, wait on futex, re-lock mutex.
1194 * Returns 1 on signal/broadcast, 0 on timeout. */
1195 5 static inline int sync_condvar_wait(SyncHandle *h, double timeout) {
1196 5 SyncHeader *hdr = h->hdr;
1197  
1198 5 ▶ if (timeout == 0) return 0; /* non-blocking: no wait */
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return 0; /* non-blocking: no wait */ 1 4 5
1199  
1200 4 uint32_t seq = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
1201  
1202 4 __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);
1203  
1204 4 sync_mutex_unlock(hdr);
1205  
1206 struct timespec deadline, remaining;
1207 4 int has_deadline = (timeout > 0);
1208 4 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 4 0 4
1209  
1210 4 int signaled = 0;
1211 0 for (;;) {
1212 4 uint32_t cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
1213 4 ▶ if (cur != seq) { signaled = 1; break; }
Branch: 1 branch
BranchTrueFalseTotal
if (cur != seq) { signaled = 1; break; } 0 4 4
1214  
1215 4 __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1216  
1217 4 struct timespec *pts = NULL;
1218 4 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 4 0 4
1219 4 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 4 4
1220 0 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1221 0 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
1222 0 break;
1223 }
1224 4 pts = &remaining;
1225 }
1226  
1227 4 long rc = syscall(SYS_futex, &hdr->value, FUTEX_WAIT, seq, pts, NULL, 0);
1228 4 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1229  
1230 4 cur = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
1231 4 ▶ if (cur != seq) { signaled = 1; break; }
Branch: 1 branch
BranchTrueFalseTotal
if (cur != seq) { signaled = 1; break; } 1 3 4
1232  
1233 3 ▶ if (rc == -1 && errno == ETIMEDOUT && has_deadline) {
Branch: 3 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT && has_deadline) { 3 0 3
if (rc == -1 && errno == ETIMEDOUT && has_deadline) { 3 0 3
if (rc == -1 && errno == ETIMEDOUT && has_deadline) { 3 0 3
1234 3 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
1235 3 break;
1236 }
1237 }
1238  
1239 4 sync_mutex_lock(hdr);
1240 4 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
1241 4 return signaled;
1242 }
1243  
1244 1 static inline void sync_condvar_signal(SyncHandle *h) {
1245 1 SyncHeader *hdr = h->hdr;
1246 1 __atomic_add_fetch(&hdr->value, 1, __ATOMIC_RELEASE);
1247 1 __atomic_add_fetch(&hdr->stat_signals, 1, __ATOMIC_RELAXED);
1248 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1249 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1250 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 1 0 1
1251 1 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, 1, NULL, NULL, 0);
1252 1 }
1253  
1254 1 static inline void sync_condvar_broadcast(SyncHandle *h) {
1255 1 SyncHeader *hdr = h->hdr;
1256 1 __atomic_add_fetch(&hdr->value, 1, __ATOMIC_RELEASE);
1257 1 __atomic_add_fetch(&hdr->stat_signals, 1, __ATOMIC_RELAXED);
1258 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1259 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1260 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 1 0 1
1261 1 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1262 1 }
1263  
1264 /* ================================================================
1265 * Once operations
1266 *
1267 * value states: 0=INIT, (SYNC_MUTEX_WRITER_BIT|pid)=RUNNING, 1=DONE
1268 * ================================================================ */
1269  
1270 #define SYNC_ONCE_INIT 0
1271 #define SYNC_ONCE_DONE 1
1272 /* RUNNING = SYNC_MUTEX_WRITER_BIT | pid */
1273  
1274 8 static inline int sync_once_is_done(SyncHandle *h) {
1275 8 return __atomic_load_n(&h->hdr->value, __ATOMIC_ACQUIRE) == SYNC_ONCE_DONE;
1276 }
1277  
1278 /* Try to become the initializer. Returns:
1279 * 1 = you are the initializer, call once_done() when finished
1280 * 0 = already done
1281 * -1 = another process is initializing (wait with once_wait) */
1282 8 static inline int sync_once_try(SyncHandle *h) {
1283 8 SyncHeader *hdr = h->hdr;
1284 8 uint32_t mypid = SYNC_MUTEX_VAL((uint32_t)getpid());
1285  
1286 8 uint32_t expected = SYNC_ONCE_INIT;
1287 8 ▶ if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &expected, mypid, 4 4 8
1288 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
1289 4 __atomic_add_fetch(&hdr->stat_acquires, 1, __ATOMIC_RELAXED);
1290 4 return 1;
1291 }
1292 4 ▶ if (expected == SYNC_ONCE_DONE) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (expected == SYNC_ONCE_DONE) return 0; 1 3 4
1293 3 return -1;
1294 }
1295  
1296 /* Call/wait combo: try to become initializer, or wait for completion.
1297 * Returns 1 if caller is the initializer, 0 if already done or waited. */
1298 6 static inline int sync_once_enter(SyncHandle *h, double timeout) {
1299 6 SyncHeader *hdr = h->hdr;
1300  
1301 /* Non-blocking probe: just try, don't wait */
1302 6 int r = sync_once_try(h);
1303 6 ▶ if (r == 1) return 1;
Branch: 1 branch
BranchTrueFalseTotal
if (r == 1) return 1; 3 3 6
1304 3 ▶ if (r == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (r == 0) return 0; 1 2 3
1305 2 ▶ if (timeout == 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (timeout == 0) return 0; 1 1 2
1306  
1307 struct timespec deadline, remaining;
1308 1 int has_deadline = (timeout > 0);
1309 1 ▶ if (has_deadline) sync_make_deadline(timeout, &deadline);
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) sync_make_deadline(timeout, &deadline); 1 0 1
1310  
1311 1 __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);
1312  
1313 1 for (;;) {
1314 2 r = sync_once_try(h);
1315 2 ▶ if (r == 1) return 1; /* caller is initializer */
Branch: 1 branch
BranchTrueFalseTotal
if (r == 1) return 1; /* caller is initializer */ 1 1 2
1316 1 ▶ if (r == 0) return 0; /* already done */
Branch: 1 branch
BranchTrueFalseTotal
if (r == 0) return 0; /* already done */ 0 1 1
1317  
1318 /* r == -1: someone else is running. Wait or detect stale. */
1319 1 uint32_t val = __atomic_load_n(&hdr->value, __ATOMIC_ACQUIRE);
1320 1 ▶ if (val == SYNC_ONCE_DONE) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (val == SYNC_ONCE_DONE) return 0; 0 1 1
1321 2 ▶ if (val == SYNC_ONCE_INIT) continue; /* race: was reset, retry */
Branch: 1 branch
BranchTrueFalseTotal
if (val == SYNC_ONCE_INIT) continue; /* race: was reset, retry */ 0 1 1
1322  
1323 /* Check stale initializer */
1324 1 ▶ if (val >= SYNC_MUTEX_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= SYNC_MUTEX_WRITER_BIT) { 1 0 1
1325 1 uint32_t pid = val & SYNC_MUTEX_PID_MASK;
1326 1 ▶ if (!sync_pid_alive(pid)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_pid_alive(pid)) { 1 0 1
1327 1 ▶ if (__atomic_compare_exchange_n(&hdr->value, &val, SYNC_ONCE_INIT,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->value, &val, SYNC_ONCE_INIT, 1 0 1
1328 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) {
1329 1 __atomic_add_fetch(&hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
1330 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1331 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1332 1 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 1 1
1333 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1334 }
1335 1 continue;
1336 }
1337 }
1338  
1339 0 __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1340  
1341 /* Always cap at SYNC_LOCK_TIMEOUT_SEC so stale-initializer recovery
1342 * runs periodically even when the caller specifies infinite timeout. */
1343 0 struct timespec *pts = (struct timespec *)&sync_lock_timeout;
1344 0 ▶ if (has_deadline) {
Branch: 1 branch
BranchTrueFalseTotal
if (has_deadline) { 0 0 0
1345 0 ▶ if (!sync_remaining_time(&deadline, &remaining)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!sync_remaining_time(&deadline, &remaining)) { 0 0 0
1346 0 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1347 0 __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
1348 0 return 0;
1349 }
1350 0 ▶ if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC)
Branch: 1 branch
BranchTrueFalseTotal
if (remaining.tv_sec < SYNC_LOCK_TIMEOUT_SEC) 0 0 0
1351 0 pts = &remaining;
1352 }
1353  
1354 0 syscall(SYS_futex, &hdr->value, FUTEX_WAIT, val, pts, NULL, 0);
1355 0 __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
1356 }
1357 }
1358  
1359 4 static inline void sync_once_done(SyncHandle *h) {
1360 4 SyncHeader *hdr = h->hdr;
1361 4 __atomic_store_n(&hdr->value, SYNC_ONCE_DONE, __ATOMIC_RELEASE);
1362 4 __atomic_add_fetch(&hdr->stat_releases, 1, __ATOMIC_RELAXED);
1363 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1364 4 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1365 4 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 1 3 4
1366 1 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1367 4 }
1368  
1369 2 static inline void sync_once_reset(SyncHandle *h) {
1370 2 SyncHeader *hdr = h->hdr;
1371 2 __atomic_store_n(&hdr->value, SYNC_ONCE_INIT, __ATOMIC_RELEASE);
1372 /* StoreLoad: publish the state change before reading the waiter count (weak-memory lost-wakeup guard). */
1373 2 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1374 2 ▶ if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) 0 2 2
1375 0 syscall(SYS_futex, &hdr->value, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
1376 2 }
1377  
1378 /* ================================================================
1379 * Create / Open / Close
1380 *
1381 * Layout:
1382 * [0..127] : SyncHeader
1383 * [128 .. 128+SLOTS_SIZE-1] : SyncReaderSlot[SYNC_READER_SLOTS] (RWLock only)
1384 * [128+SLOTS_SIZE .. +OCC_BYTES-1] : occupancy bitmap, SYNC_OCC_WORDS words (RWLock only)
1385 *
1386 * Non-RWLock primitives keep total_size = sizeof(SyncHeader) (Option A:
1387 * pay-for-what-you-use, ~16KB only when needed).
1388 * ================================================================ */
1389  
1390 #define SYNC_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, SYNC_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while(0)
1391  
1392 87 static inline uint64_t sync_layout_total_size(uint32_t type) {
1393 87 uint64_t sz = sizeof(SyncHeader);
1394 87 ▶ if (type == SYNC_TYPE_RWLOCK)
Branch: 1 branch
BranchTrueFalseTotal
if (type == SYNC_TYPE_RWLOCK) 15 72 87
1395 15 sz += (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot) + SYNC_OCC_BYTES;
1396 87 return sz;
1397 }
1398  
1399 87 static inline uint64_t sync_layout_slots_off(uint32_t type) {
1400 87 ▶ return (type == SYNC_TYPE_RWLOCK) ? sizeof(SyncHeader) : 0;
Branch: 1 branch
BranchTrueFalseTotal
return (type == SYNC_TYPE_RWLOCK) ? sizeof(SyncHeader) : 0; 15 72 87
1401 }
1402  
1403 /* Securely obtain a fd: create exclusively (O_CREAT|O_EXCL|O_NOFOLLOW at mode,
1404 * default 0600), or attach an existing file (O_RDWR|O_NOFOLLOW, no O_CREAT). */
1405 23 static int sync_secure_open(const char *path, mode_t mode, char *errbuf) {
1406 23 ▶ for (int attempt = 0; attempt < 100; attempt++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int attempt = 0; attempt < 100; attempt++) { 23 0 23
1407 23 int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, mode);
1408 23 ▶ if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */ 9 14 23
1409 14 ▶ if (errno != EEXIST) { SYNC_ERR("create %s: %s", path, strerror(errno)); return -1; }
Branch: 2 branches
BranchTrueFalseTotal
if (errno != EEXIST) { SYNC_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 14 14
if (errno != EEXIST) { SYNC_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 0 0
1410 14 fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
1411 14 ▶ if (fd >= 0) return fd;
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) return fd; 14 0 14
1412 0 ▶ if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */
Branch: 1 branch
BranchTrueFalseTotal
if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */ 0 0 0
1413 0 ▶ SYNC_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */ 0 0 0
1414 0 return -1;
1415 }
1416 0 ▶ SYNC_ERR("open %s: create/attach kept racing", path);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("open %s: create/attach kept racing", path); 0 0 0
1417 0 return -1;
1418 }
1419  
1420 /* True iff the whole file is zero -- what an abandoned mid-init
1421 creator leaves. Lets recovery re-init only a provably-empty file, never
1422 one that merely starts with a zero word. Cold path, so a byte scan is
1423 fine. Read, not mapped: a read fault on a tmpfs hole allocates the page. */
1424 2 static int sync_file_is_zero(int fd, uint64_t size) {
1425 char buf[16384];
1426 3 ▶ for (uint64_t off = 0; off < size; ) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint64_t off = 0; off < size; ) { 2 1 3
1427 2 ssize_t n = pread(fd, buf, size - off < sizeof buf ? (size_t)(size - off) : sizeof buf, (off_t)off);
1428 2 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 2 2
1429 257 ▶ for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0;
Branch: 2 branches
BranchTrueFalseTotal
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 1 255 256
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 256 1 257
1430 1 off += (uint64_t)n;
1431 }
1432 1 return 1;
1433 }
1434  
1435 86 static SyncHandle *sync_create(const char *path, uint32_t type, uint32_t param,
1436 uint32_t initial, mode_t mode, char *errbuf) {
1437 86 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 86 0 86
1438  
1439 86 ▶ if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; } 0 86 86
if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; } 0 0 0
1440 86 ▶ if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 35 51 86
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 1 34 35
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 1 0 1
1441 85 ▶ if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 34 51 85
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 2 32 34
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 2 0 2
1442 83 ▶ if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 14 69 83
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 2 12 14
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 2 0 2
1443  
1444 81 uint64_t total_size = sync_layout_total_size(type);
1445 81 uint64_t slots_off = sync_layout_slots_off(type);
1446 81 int anonymous = (path == NULL);
1447 size_t map_size;
1448 void *base;
1449  
1450 81 ▶ if (anonymous) {
Branch: 1 branch
BranchTrueFalseTotal
if (anonymous) { 58 23 81
1451 58 map_size = (size_t)total_size;
1452 58 base = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
1453 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1454 58 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 58 58
1455 0 ▶ SYNC_ERR("mmap(anonymous): %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("mmap(anonymous): %s", strerror(errno)); 0 0 0
1456 0 return NULL;
1457 }
1458 58 SyncHeader *hdr = (SyncHeader *)base;
1459 58 memset(hdr, 0, sizeof(SyncHeader));
1460 58 hdr->version = SYNC_VERSION;
1461 58 hdr->type = type;
1462 58 hdr->param = param;
1463 58 hdr->total_size = total_size;
1464 58 hdr->reader_slots_off = slots_off;
1465 58 ▶ if (type == SYNC_TYPE_SEMAPHORE)
Branch: 1 branch
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE) 20 38 58
1466 20 hdr->value = initial;
1467 /* MAP_ANONYMOUS already zero-fills the reader_slots + occ region. */
1468 /* Publish magic LAST, as a release store: it is the commit point, so
1469 a creator killed before it leaves magic==0 and never a file
1470 mistaken for a valid one. A kill during the field stores leaves
1471 one to remove by hand. */
1472 58 __atomic_store_n(&hdr->magic, SYNC_MAGIC, __ATOMIC_RELEASE);
1473 58 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1474 58 goto setup_handle;
1475 } else {
1476 23 int fd = sync_secure_open(path, mode, errbuf);
1477 29 ▶ if (fd < 0) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return NULL; 0 23 23
1478  
1479 23 ▶ if (flock(fd, LOCK_EX) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (flock(fd, LOCK_EX) < 0) { 0 23 23
1480 0 ▶ SYNC_ERR("flock(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("flock(%s): %s", path, strerror(errno)); 0 0 0
1481 0 close(fd); return NULL;
1482 }
1483  
1484 struct stat st;
1485 23 ▶ if (fstat(fd, &st) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { 0 23 23
1486 0 ▶ SYNC_ERR("fstat(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fstat(%s): %s", path, strerror(errno)); 0 0 0
1487 0 flock(fd, LOCK_UN); close(fd); return NULL;
1488 }
1489  
1490 23 int is_new = (st.st_size == 0);
1491  
1492 23 ▶ if (!is_new && (uint64_t)st.st_size < sizeof(SyncHeader)) {
Branch: 2 branches
BranchTrueFalseTotal
if (!is_new && (uint64_t)st.st_size < sizeof(SyncHeader)) { 14 9 23
if (!is_new && (uint64_t)st.st_size < sizeof(SyncHeader)) { 0 14 14
1493 0 ▶ SYNC_ERR("%s: file too small (%lld)", path, (long long)st.st_size);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("%s: file too small (%lld)", path, (long long)st.st_size); 0 0 0
1494 0 flock(fd, LOCK_UN); close(fd); return NULL;
1495 }
1496  
1497 23 ▶ if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) {
Branch: 3 branches
BranchTrueFalseTotal
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 9 14 23
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 9 0 9
if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 0 9 9
1498 0 ▶ SYNC_ERR("%s: refusing to initialize file not owned by us", path);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("%s: refusing to initialize file not owned by us", path); 0 0 0
1499 0 flock(fd, LOCK_UN); close(fd); return NULL;
1500 }
1501 23 ▶ if (is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (is_new) { 9 14 23
1502 9 ▶ if (ftruncate(fd, (off_t)total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)total_size) < 0) { 0 9 9
1503 0 ▶ SYNC_ERR("ftruncate(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("ftruncate(%s): %s", path, strerror(errno)); 0 0 0
1504 0 flock(fd, LOCK_UN); close(fd); return NULL;
1505 }
1506 }
1507  
1508 23 ▶ map_size = is_new ? (size_t)total_size : (size_t)st.st_size;
Branch: 1 branch
BranchTrueFalseTotal
map_size = is_new ? (size_t)total_size : (size_t)st.st_size; 14 9 23
1509 23 base = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1510 23 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 23 23
1511 0 ▶ SYNC_ERR("mmap(%s): %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("mmap(%s): %s", path, strerror(errno)); 0 0 0
1512 0 ▶ if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ }
Branch: 1 branch
BranchTrueFalseTotal
if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ } 0 0 0
1513 0 flock(fd, LOCK_UN); close(fd); return NULL;
1514 }
1515  
1516 23 SyncHeader *hdr = (SyncHeader *)base;
1517  
1518 23 ▶ if (!is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (!is_new) { 14 9 23
1519 38 int valid = (hdr->magic == SYNC_MAGIC &&
1520 10 ▶ hdr->version == SYNC_VERSION &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->version == SYNC_VERSION && 10 0 10
1521 31 ▶ hdr->type == type &&
Branch: 2 branches
BranchTrueFalseTotal
hdr->type == type && 10 4 14
hdr->type == type && 7 3 10
1522 7 ▶ hdr->total_size == (uint64_t)st.st_size);
Branch: 1 branch
BranchTrueFalseTotal
hdr->total_size == (uint64_t)st.st_size); 7 0 7
1523 14 ▶ if (valid && type == SYNC_TYPE_RWLOCK) {
Branch: 2 branches
BranchTrueFalseTotal
if (valid && type == SYNC_TYPE_RWLOCK) { 7 7 14
if (valid && type == SYNC_TYPE_RWLOCK) { 1 6 7
1524 /* reader_slots_off must point to a valid region, and the file must
1525 * hold the slot table AND the occupancy bitmap that follows it. */
1526 1 uint64_t need = sizeof(SyncHeader) +
1527 (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot) +
1528 SYNC_OCC_BYTES;
1529 1 ▶ if (hdr->reader_slots_off != sizeof(SyncHeader) ||
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->reader_slots_off != sizeof(SyncHeader) || 1 0 1
1530 1 ▶ (uint64_t)st.st_size < need)
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)st.st_size < need) 0 1 1
1531 0 valid = 0;
1532 }
1533 /* Recover an abandoned mid-init file: a creator killed between
1534 * the ftruncate and the header init leaves a full-size, all-zero
1535 * (magic==0) file that would brick every future open of this
1536 * path. Re-initialize ONLY when it is exactly our size, still
1537 * uninitialized, and owned by us; anything else still errors. */
1538 14 ▶ if (hdr->magic == 0 && (uint64_t)st.st_size == total_size
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->magic == 0 && (uint64_t)st.st_size == total_size 3 11 14
if (hdr->magic == 0 && (uint64_t)st.st_size == total_size 2 1 3
1539 2 ▶ && st.st_uid == geteuid() && sync_file_is_zero(fd, map_size)) {
Branch: 2 branches
BranchTrueFalseTotal
&& st.st_uid == geteuid() && sync_file_is_zero(fd, map_size)) { 2 0 2
&& st.st_uid == geteuid() && sync_file_is_zero(fd, map_size)) { 1 1 2
1540 1 ▶ if (fchmod(fd, mode) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fchmod(fd, mode) < 0) { 0 1 1
1541 0 ▶ SYNC_ERR("%s: fchmod: %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("%s: fchmod: %s", path, strerror(errno)); 0 0 0
1542 0 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
1543 }
1544 1 hdr->version = SYNC_VERSION;
1545 1 hdr->type = type;
1546 1 hdr->param = param;
1547 1 hdr->total_size = total_size;
1548 1 hdr->reader_slots_off = slots_off;
1549 1 ▶ if (type == SYNC_TYPE_SEMAPHORE)
Branch: 1 branch
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE) 1 0 1
1550 1 hdr->value = initial;
1551 /* Publish magic LAST, as a release store: it is the commit
1552 point, so a creator killed before it leaves magic==0 and
1553 never a file mistaken for a valid one. A kill during the
1554 field stores leaves one to remove by hand. */
1555 1 __atomic_store_n(&hdr->magic, SYNC_MAGIC, __ATOMIC_RELEASE);
1556 1 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1557 1 flock(fd, LOCK_UN); close(fd);
1558 8 goto setup_handle;
1559 }
1560 13 ▶ if (!valid) {
Branch: 1 branch
BranchTrueFalseTotal
if (!valid) { 6 7 13
1561 6 ▶ if (hdr->magic == 0 && (uint64_t)st.st_size == total_size
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->magic == 0 && (uint64_t)st.st_size == total_size 2 4 6
if (hdr->magic == 0 && (uint64_t)st.st_size == total_size 1 1 2
1562 1 ▶ && st.st_uid == geteuid())
Branch: 1 branch
BranchTrueFalseTotal
&& st.st_uid == geteuid()) 1 0 1
1563 1 ▶ SYNC_ERR("%s: incomplete sync file left by an interrupted create; remove it and retry", path);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("%s: incomplete sync file left by an interrupted create; remove it and retry", path); 1 0 1
1564 else
1565 5 ▶ SYNC_ERR("%s: invalid or incompatible sync file", path);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("%s: invalid or incompatible sync file", path); 5 0 5
1566 6 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
1567 }
1568 7 flock(fd, LOCK_UN);
1569 7 close(fd);
1570 7 goto setup_handle;
1571 }
1572  
1573 /* Initialize while holding the flock */
1574 9 memset(base, 0, (size_t)total_size); /* zero header + reader_slots + occ region */
1575 9 hdr->version = SYNC_VERSION;
1576 9 hdr->type = type;
1577 9 hdr->param = param;
1578 9 hdr->total_size = total_size;
1579 9 hdr->reader_slots_off = slots_off;
1580 9 ▶ if (type == SYNC_TYPE_SEMAPHORE)
Branch: 1 branch
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE) 5 4 9
1581 5 hdr->value = initial;
1582 /* Publish magic LAST, as a release store: it is the commit point, so
1583 a creator killed before it leaves magic==0 and never a file
1584 mistaken for a valid one. A kill during the field stores leaves
1585 one to remove by hand. */
1586 9 __atomic_store_n(&hdr->magic, SYNC_MAGIC, __ATOMIC_RELEASE);
1587 9 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1588  
1589 9 flock(fd, LOCK_UN);
1590 9 close(fd);
1591 } /* end file-backed */
1592  
1593 75 setup_handle:;
1594 {
1595 75 SyncHeader *hdr = (SyncHeader *)base;
1596 75 SyncHandle *h = (SyncHandle *)calloc(1, sizeof(SyncHandle));
1597 75 ▶ if (!h) { munmap(base, map_size); return NULL; }
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { munmap(base, map_size); return NULL; } 0 75 75
1598  
1599 75 h->hdr = hdr;
1600 75 h->mmap_size = map_size;
1601 75 ▶ h->path = path ? strdup(path) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->path = path ? strdup(path) : NULL; 17 58 75
1602 75 h->notify_fd = -1;
1603 75 h->backing_fd = -1;
1604 /* Layer B: reader_slots_off is read from the (attacker-writable) mmap image
1605 * and used as a pointer offset; bound it against the mapping size so a
1606 * poisoned offset yields NULL instead of an out-of-bounds pointer. Valid
1607 * images use 0 (non-RWLock) or sizeof(SyncHeader) (RWLock), both in range. */
1608 {
1609 75 uint64_t rso = hdr->reader_slots_off;
1610 75 uint64_t need_slots = (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot);
1611 75 ▶ if (rso > 0 && rso <= (uint64_t)map_size &&
Branch: 2 branches
BranchTrueFalseTotal
if (rso > 0 && rso <= (uint64_t)map_size && 13 62 75
if (rso > 0 && rso <= (uint64_t)map_size && 13 0 13
1612 13 ▶ (uint64_t)map_size - rso >= need_slots &&
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)map_size - rso >= need_slots && 13 0 13
1613 26 ▶ (rso & (uint64_t)(_Alignof(SyncReaderSlot) - 1)) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
(rso & (uint64_t)(_Alignof(SyncReaderSlot) - 1)) == 0) { 13 0 13
1614 13 h->reader_slots = (SyncReaderSlot *)((char *)base + rso);
1615 /* Occupancy bitmap follows the slot table; bound it within the mapping
1616 * too. On a poisoned or short map NULL BOTH -- occ is used only under an
1617 * `if (h->reader_slots)` guard, so the two must be non-NULL together. */
1618 13 uint64_t occ_off = rso + need_slots;
1619 13 ▶ if (occ_off <= (uint64_t)map_size &&
Branch: 1 branch
BranchTrueFalseTotal
if (occ_off <= (uint64_t)map_size && 13 0 13
1620 13 ▶ (uint64_t)map_size - occ_off >= SYNC_OCC_BYTES) {
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)map_size - occ_off >= SYNC_OCC_BYTES) { 13 0 13
1621 13 h->occ = (uint64_t *)((char *)base + occ_off);
1622 } else {
1623 0 h->reader_slots = NULL;
1624 0 h->occ = NULL;
1625 }
1626 } else {
1627 62 h->reader_slots = NULL;
1628 62 h->occ = NULL;
1629 }
1630 }
1631 75 h->my_slot_idx = UINT32_MAX;
1632  
1633 75 return h;
1634 }
1635 }
1636  
1637 6 static SyncHandle *sync_create_memfd(const char *name, uint32_t type,
1638 uint32_t param, uint32_t initial,
1639 char *errbuf) {
1640 6 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 6 0 6
1641  
1642 6 ▶ if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; } 0 6 6
if (type > SYNC_TYPE_ONCE) { SYNC_ERR("unknown type %u", type); return NULL; } 0 0 0
1643 6 ▶ if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 2 4 6
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 0 2 2
if (type == SYNC_TYPE_SEMAPHORE && param == 0) { SYNC_ERR("semaphore max must be > 0"); return NULL; } 0 0 0
1644 6 ▶ if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 2 4 6
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 0 2 2
if (type == SYNC_TYPE_SEMAPHORE && initial > param) { SYNC_ERR("initial (%u) > max (%u)", initial, param); return NULL; } 0 0 0
1645 6 ▶ if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; }
Branch: 3 branches
BranchTrueFalseTotal
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 1 5 6
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 0 1 1
if (type == SYNC_TYPE_BARRIER && param < 2) { SYNC_ERR("barrier count must be >= 2"); return NULL; } 0 0 0
1646  
1647 6 uint64_t total_size = sync_layout_total_size(type);
1648 6 uint64_t slots_off = sync_layout_slots_off(type);
1649  
1650 6 ▶ int fd = memfd_create(name ? name : "sync", MFD_CLOEXEC | MFD_ALLOW_SEALING);
Branch: 1 branch
BranchTrueFalseTotal
int fd = memfd_create(name ? name : "sync", MFD_CLOEXEC | MFD_ALLOW_SEALING); 6 0 6
1651 6 ▶ if (fd < 0) { SYNC_ERR("memfd_create: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fd < 0) { SYNC_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 6 6
if (fd < 0) { SYNC_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 0 0
1652  
1653 6 ▶ if (ftruncate(fd, (off_t)total_size) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)total_size) < 0) { 0 6 6
1654 0 ▶ SYNC_ERR("ftruncate(memfd): %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("ftruncate(memfd): %s", strerror(errno)); 0 0 0
1655 0 close(fd); return NULL;
1656 }
1657 6 (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);
1658  
1659 6 void *base = mmap(NULL, (size_t)total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1660 6 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 6 6
1661 0 ▶ SYNC_ERR("mmap(memfd): %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("mmap(memfd): %s", strerror(errno)); 0 0 0
1662 0 close(fd); return NULL;
1663 }
1664  
1665 6 SyncHeader *hdr = (SyncHeader *)base;
1666 6 memset(hdr, 0, (size_t)total_size); /* zero header + reader_slots + occ region */
1667 6 hdr->version = SYNC_VERSION;
1668 6 hdr->type = type;
1669 6 hdr->param = param;
1670 6 hdr->total_size = total_size;
1671 6 hdr->reader_slots_off = slots_off;
1672  
1673 6 ▶ if (type == SYNC_TYPE_SEMAPHORE)
Branch: 1 branch
BranchTrueFalseTotal
if (type == SYNC_TYPE_SEMAPHORE) 2 4 6
1674 2 hdr->value = initial;
1675  
1676 /* Publish magic LAST, as a release store: it is the commit point, so a
1677 creator killed before it leaves magic==0 and never a file mistaken for
1678 a valid one. A kill during the field stores leaves one to remove by
1679 hand. */
1680 6 __atomic_store_n(&hdr->magic, SYNC_MAGIC, __ATOMIC_RELEASE);
1681 6 __atomic_thread_fence(__ATOMIC_SEQ_CST);
1682  
1683 6 SyncHandle *h = (SyncHandle *)calloc(1, sizeof(SyncHandle));
1684 6 ▶ if (!h) { munmap(base, (size_t)total_size); close(fd); return NULL; }
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { munmap(base, (size_t)total_size); close(fd); return NULL; } 0 6 6
1685  
1686 6 h->hdr = hdr;
1687 6 h->mmap_size = (size_t)total_size;
1688 6 h->path = NULL;
1689 6 h->notify_fd = -1;
1690 6 h->backing_fd = fd;
1691 6 ▶ if (slots_off > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (slots_off > 0) { 1 5 6
1692 1 h->reader_slots = (SyncReaderSlot *)((char *)base + slots_off);
1693 /* Occupancy bitmap follows the slot table (total_size, and the sealed
1694 * memfd mapping, both cover it -- trusted create, so no bound check). */
1695 1 h->occ = (uint64_t *)((char *)base + slots_off +
1696 (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot));
1697 } else {
1698 5 h->reader_slots = NULL;
1699 5 h->occ = NULL;
1700 }
1701 6 h->my_slot_idx = UINT32_MAX;
1702  
1703 6 return h;
1704 }
1705  
1706 5 static SyncHandle *sync_open_fd(int fd, uint32_t type, char *errbuf) {
1707 5 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 5 0 5
1708  
1709 struct stat st;
1710 5 ▶ if (fstat(fd, &st) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { 1 4 5
1711 1 ▶ SYNC_ERR("fstat(fd=%d): %s", fd, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fstat(fd=%d): %s", fd, strerror(errno)); 1 0 1
1712 1 return NULL;
1713 }
1714  
1715 4 ▶ if ((uint64_t)st.st_size < sizeof(SyncHeader)) {
Branch: 1 branch
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(SyncHeader)) { 0 4 4
1716 0 ▶ SYNC_ERR("fd %d: too small (%lld)", fd, (long long)st.st_size);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fd %d: too small (%lld)", fd, (long long)st.st_size); 0 0 0
1717 0 return NULL;
1718 }
1719  
1720 /* before mmap: a creator may truncate the file until it has set magic */
1721 uint32_t magic;
1722 4 ▶ if (pread(fd, &magic, sizeof magic, offsetof(SyncHeader, magic)) != (ssize_t)sizeof magic || magic != SYNC_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (pread(fd, &magic, sizeof magic, offsetof(SyncHeader, magic)) != (ssize_t)sizeof magic || magic != SYNC_MAGIC) { 4 0 4
if (pread(fd, &magic, sizeof magic, offsetof(SyncHeader, magic)) != (ssize_t)sizeof magic || magic != SYNC_MAGIC) { 0 4 4
1723 0 ▶ SYNC_ERR("fd %d: invalid or incompatible sync", fd);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fd %d: invalid or incompatible sync", fd); 0 0 0
1724 0 return NULL;
1725 }
1726  
1727 4 size_t map_size = (size_t)st.st_size;
1728 4 void *base = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1729 4 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 4 4
1730 0 ▶ SYNC_ERR("mmap(fd=%d): %s", fd, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("mmap(fd=%d): %s", fd, strerror(errno)); 0 0 0
1731 0 return NULL;
1732 }
1733  
1734 4 SyncHeader *hdr = (SyncHeader *)base;
1735 12 int valid = (hdr->magic == SYNC_MAGIC &&
1736 4 ▶ hdr->version == SYNC_VERSION &&
Branch: 1 branch
BranchTrueFalseTotal
hdr->version == SYNC_VERSION && 4 0 4
1737 11 ▶ hdr->type == type &&
Branch: 2 branches
BranchTrueFalseTotal
hdr->type == type && 4 0 4
hdr->type == type && 3 1 4
1738 3 ▶ hdr->total_size == (uint64_t)st.st_size);
Branch: 1 branch
BranchTrueFalseTotal
hdr->total_size == (uint64_t)st.st_size); 3 0 3
1739 4 ▶ if (valid && type == SYNC_TYPE_RWLOCK) {
Branch: 2 branches
BranchTrueFalseTotal
if (valid && type == SYNC_TYPE_RWLOCK) { 3 1 4
if (valid && type == SYNC_TYPE_RWLOCK) { 1 2 3
1740 1 uint64_t need = sizeof(SyncHeader) +
1741 (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot) +
1742 SYNC_OCC_BYTES;
1743 1 ▶ if (hdr->reader_slots_off != sizeof(SyncHeader) ||
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->reader_slots_off != sizeof(SyncHeader) || 1 0 1
1744 1 ▶ (uint64_t)st.st_size < need)
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)st.st_size < need) 0 1 1
1745 0 valid = 0;
1746 }
1747 4 ▶ if (!valid) {
Branch: 1 branch
BranchTrueFalseTotal
if (!valid) { 1 3 4
1748 1 ▶ SYNC_ERR("fd %d: invalid or incompatible sync", fd);
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fd %d: invalid or incompatible sync", fd); 1 0 1
1749 1 munmap(base, map_size);
1750 1 return NULL;
1751 }
1752  
1753 3 int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
1754 3 ▶ if (myfd < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (myfd < 0) { 0 3 3
1755 0 ▶ SYNC_ERR("fcntl(F_DUPFD_CLOEXEC): %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
SYNC_ERR("fcntl(F_DUPFD_CLOEXEC): %s", strerror(errno)); 0 0 0
1756 0 munmap(base, map_size);
1757 0 return NULL;
1758 }
1759  
1760 3 SyncHandle *h = (SyncHandle *)calloc(1, sizeof(SyncHandle));
1761 3 ▶ if (!h) { munmap(base, map_size); close(myfd); return NULL; }
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { munmap(base, map_size); close(myfd); return NULL; } 0 3 3
1762  
1763 3 h->hdr = hdr;
1764 3 h->mmap_size = map_size;
1765 3 h->path = NULL;
1766 3 h->notify_fd = -1;
1767 3 h->backing_fd = myfd;
1768 /* Layer B: bound the mmap-supplied reader_slots_off against the mapping
1769 * size before using it as a pointer offset (see sync_create). */
1770 {
1771 3 uint64_t rso = hdr->reader_slots_off;
1772 3 uint64_t need_slots = (uint64_t)SYNC_READER_SLOTS * sizeof(SyncReaderSlot);
1773 3 ▶ if (rso > 0 && rso <= (uint64_t)map_size &&
Branch: 2 branches
BranchTrueFalseTotal
if (rso > 0 && rso <= (uint64_t)map_size && 1 2 3
if (rso > 0 && rso <= (uint64_t)map_size && 1 0 1
1774 1 ▶ (uint64_t)map_size - rso >= need_slots &&
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)map_size - rso >= need_slots && 1 0 1
1775 2 ▶ (rso & (uint64_t)(_Alignof(SyncReaderSlot) - 1)) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
(rso & (uint64_t)(_Alignof(SyncReaderSlot) - 1)) == 0) { 1 0 1
1776 1 h->reader_slots = (SyncReaderSlot *)((char *)base + rso);
1777 /* Occupancy bitmap follows the slot table; bound it within the mapping
1778 * too. On a poisoned or short map NULL BOTH -- occ is used only under an
1779 * `if (h->reader_slots)` guard, so the two must be non-NULL together. */
1780 1 uint64_t occ_off = rso + need_slots;
1781 1 ▶ if (occ_off <= (uint64_t)map_size &&
Branch: 1 branch
BranchTrueFalseTotal
if (occ_off <= (uint64_t)map_size && 1 0 1
1782 1 ▶ (uint64_t)map_size - occ_off >= SYNC_OCC_BYTES) {
Branch: 1 branch
BranchTrueFalseTotal
(uint64_t)map_size - occ_off >= SYNC_OCC_BYTES) { 1 0 1
1783 1 h->occ = (uint64_t *)((char *)base + occ_off);
1784 } else {
1785 0 h->reader_slots = NULL;
1786 0 h->occ = NULL;
1787 }
1788 } else {
1789 2 h->reader_slots = NULL;
1790 2 h->occ = NULL;
1791 }
1792 }
1793 3 h->my_slot_idx = UINT32_MAX;
1794  
1795 3 return h;
1796 }
1797  
1798 84 static void sync_destroy(SyncHandle *h) {
1799 84 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 84 84
1800 /* Release reader slot -- only if we still own it AND no fork has happened
1801 * since we claimed it. A forked child that inherits the handle but never
1802 * acquired the lock itself must NOT clear the parent's slot. */
1803 84 ▶ if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid &&
Branch: 3 branches
BranchTrueFalseTotal
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 15 69 84
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 8 7 15
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 8 0 8
1804 8 ▶ h->cached_fork_gen == __atomic_load_n(&sync_fork_gen, __ATOMIC_RELAXED) &&
Branch: 1 branch
BranchTrueFalseTotal
h->cached_fork_gen == __atomic_load_n(&sync_fork_gen, __ATOMIC_RELAXED) && 8 0 8
1805 8 ▶ __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
__atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) { 8 0 8
1806 /* rdepth==0: a still-held read lock's slot must survive for recovery */
1807 /* Clear our occ bit BEFORE releasing the slot: we still own the pid so no
1808 * claimant can take the slot mid-clear, and rdepth==0 so no writer needs
1809 * to see us. (A crash skips this -> the bit is reclaimed lazily by a
1810 * writer scan / re-claim, same as the pid.) */
1811 8 sync_occ_clear(h, h->my_slot_idx);
1812 8 uint32_t expected = h->cached_pid;
1813 /* CAS pid -> 0; do NOT clear rdepth -- between the CAS and a follow-up
1814 * store, a new process could claim the slot, and our store would clobber
1815 * its state. sync_claim_reader_slot zeros rdepth on every claim, so
1816 * leaving a stale value is safe. */
1817 8 __atomic_compare_exchange_n(&h->reader_slots[h->my_slot_idx].pid,
1818 &expected, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
1819 }
1820 84 ▶ if (h->notify_fd >= 0) close(h->notify_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (h->notify_fd >= 0) close(h->notify_fd); 11 73 84
1821 84 ▶ if (h->backing_fd >= 0) close(h->backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (h->backing_fd >= 0) close(h->backing_fd); 9 75 84
1822 84 ▶ if (h->hdr) munmap(h->hdr, h->mmap_size);
Branch: 1 branch
BranchTrueFalseTotal
if (h->hdr) munmap(h->hdr, h->mmap_size); 84 0 84
1823 84 free(h->path);
1824 84 free(h);
1825 }
1826  
1827 /* ================================================================
1828 * Eventfd integration
1829 * ================================================================ */
1830  
1831 11 static int sync_create_eventfd(SyncHandle *h) {
1832 11 ▶ if (h->notify_fd >= 0) return h->notify_fd;
Branch: 1 branch
BranchTrueFalseTotal
if (h->notify_fd >= 0) return h->notify_fd; 0 11 11
1833 11 int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
1834 11 ▶ if (efd < 0) return -1;
Branch: 1 branch
BranchTrueFalseTotal
if (efd < 0) return -1; 0 11 11
1835 11 h->notify_fd = efd;
1836 11 return efd;
1837 }
1838  
1839 10 static int sync_notify(SyncHandle *h) {
1840 10 ▶ if (h->notify_fd < 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (h->notify_fd < 0) return 0; 0 10 10
1841 10 uint64_t val = 1;
1842 10 return write(h->notify_fd, &val, sizeof(val)) == sizeof(val);
1843 }
1844  
1845 11 static int64_t sync_eventfd_consume(SyncHandle *h) {
1846 11 ▶ if (h->notify_fd < 0) return -1;
Branch: 1 branch
BranchTrueFalseTotal
if (h->notify_fd < 0) return -1; 0 11 11
1847 11 uint64_t val = 0;
1848 11 ▶ if (read(h->notify_fd, &val, sizeof(val)) != sizeof(val)) return -1;
Branch: 1 branch
BranchTrueFalseTotal
if (read(h->notify_fd, &val, sizeof(val)) != sizeof(val)) return -1; 5 6 11
1849 6 return (int64_t)val;
1850 }
1851  
1852 /* ================================================================
1853 * Misc
1854 * ================================================================ */
1855  
1856 0 static int sync_msync(SyncHandle *h) {
1857 0 return msync(h->hdr, h->mmap_size, MS_SYNC);
1858 }
1859  
1860 #endif /* SYNC_H */
« Shared.xs ↑ Summary blib/lib/Data/Sync/Shared/Semaphore.pm »