ndarray.h

filter: Statementstmt 67.9% 365 / 537 Branchbran 48.5% 228 / 470 Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 58.8% 593 / 1007 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« blib/lib/Data/NDArray/Shared.pm ↑ Summary Shared.xs »
1 /*
2 * ndarray.h -- Shared-memory typed dense N-dimensional numeric array for Linux
3 *
4 * Holds one contiguous, row-major numeric tensor of a fixed dtype (one of
5 * f64/f32/i64/i32/i16/i8/u64/u32/u16/u8), a fixed shape (1..8 dimensions) and
6 * the matching row-major strides, in a shared mapping so several processes
7 * share one array. Element access is by multi-index or by flat index; the
8 * whole array supports fill/zero, reshape (no copy), reductions (sum/min/max/
9 * mean) and in-place scalar and element-wise array arithmetic. A
10 * write-preferring futex rwlock with reader-slot dead-process recovery guards
11 * mutation; immutable header fields (dtype/ndim/shape/strides/size/itemsize)
12 * are read lock-free.
13 *
14 * Layout: Header -> reader_slots[1024] -> occ bitmap -> data[size * itemsize]
15 */
16  
17 #ifndef NDARRAY_H
18 #define NDARRAY_H
19  
20 #include <stddef.h>
21 #include <stdint.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <unistd.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <time.h>
28 #include <limits.h>
29 #include <signal.h>
30 #include <stdio.h>
31 #include <sys/mman.h>
32 #include <sys/stat.h>
33 #include <sys/file.h>
34 #include <sys/syscall.h>
35 #include <sys/random.h>
36 #include <linux/futex.h>
37 #include <pthread.h>
38  
39 #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
40 #error "ndarray.h: requires little-endian architecture"
41 #endif
42  
43  
44 /* ================================================================
45 * Constants
46 * ================================================================ */
47  
48 #define NDA_MAGIC 0x4144444EU /* "NDDA" (little-endian) */
49 #define NDA_VERSION 2 /* 2: added the occupancy bitmap region (layout change) */
50 #define NDA_ERR_BUFLEN (PATH_MAX + 256)
51 #define NDA_READER_SLOTS 1024 /* max concurrent reader processes for dead-process recovery */
52  
53 /* Occupancy bitmap: one bit per reader slot, set when a process claims a slot and
54 * cleared on clean release. A writer scans these NDA_OCC_WORDS words to visit
55 * only OCCUPIED slots (O(words + live readers)) instead of all NDA_READER_SLOTS. */
56 #define NDA_OCC_WORDS (((NDA_READER_SLOTS) + 63) / 64) /* 16 for 1024 slots */
57 #define NDA_OCC_BYTES ((uint64_t)NDA_OCC_WORDS * 8) /* 128 bytes */
58 #define NDA_MAX_DIMS 8
59 #define NDA_MAX_BYTES ((uint64_t)1 << 40) /* 1 TiB cap on the data buffer */
60  
61 #define NDA_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, NDA_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while (0)
62  
63 /* ================================================================
64 * dtypes
65 * ================================================================ */
66  
67 enum NdaDtype {
68 NDA_F64, NDA_F32, NDA_I64, NDA_I32, NDA_I16,
69 NDA_I8, NDA_U64, NDA_U32, NDA_U16, NDA_U8,
70 NDA_NTYPES
71 };
72  
73 static const uint32_t nda_itemsize_tab[NDA_NTYPES] = { 8,4,8,4,2,1,8,4,2,1 };
74 static const char *const nda_name_tab[NDA_NTYPES] =
75 { "f64","f32","i64","i32","i16","i8","u64","u32","u16","u8" };
76  
77 /* Classify a dtype. Float = F64,F32; signed-int = I64..I8; rest unsigned. */
78 32 ▶static inline int nda_is_float(uint32_t dt) { return dt == NDA_F64 || dt == NDA_F32; }
Branch: 2 branches
BranchTrueFalseTotal
static inline int nda_is_float(uint32_t dt) { return dt == NDA_F64 || dt == NDA_F32; } 24 8 32
static inline int nda_is_float(uint32_t dt) { return dt == NDA_F64 || dt == NDA_F32; } 6 18 24
79 18 ▶static inline int nda_is_signed(uint32_t dt) { return dt == NDA_I64 || dt == NDA_I32 || dt == NDA_I16 || dt == NDA_I8; }
Branch: 4 branches
BranchTrueFalseTotal
static inline int nda_is_signed(uint32_t dt) { return dt == NDA_I64 || dt == NDA_I32 || dt == NDA_I16 || dt == NDA_I8; } 10 8 18
static inline int nda_is_signed(uint32_t dt) { return dt == NDA_I64 || dt == NDA_I32 || dt == NDA_I16 || dt == NDA_I8; } 2 8 10
static inline int nda_is_signed(uint32_t dt) { return dt == NDA_I64 || dt == NDA_I32 || dt == NDA_I16 || dt == NDA_I8; } 2 0 2
static inline int nda_is_signed(uint32_t dt) { return dt == NDA_I64 || dt == NDA_I32 || dt == NDA_I16 || dt == NDA_I8; } 0 2 2
80  
81 /* Parse a dtype name string -> enum, or -1 if unknown. */
82 142 static inline int nda_dtype_from_name(const char *s, size_t len) {
83 555 ▶ for (int i = 0; i < NDA_NTYPES; i++)
Branch: 1 branch
BranchTrueFalseTotal
for (int i = 0; i < NDA_NTYPES; i++) 553 2 555
84 553 ▶ if (strlen(nda_name_tab[i]) == len && memcmp(nda_name_tab[i], s, len) == 0)
Branch: 2 branches
BranchTrueFalseTotal
if (strlen(nda_name_tab[i]) == len && memcmp(nda_name_tab[i], s, len) == 0) 382 171 553
if (strlen(nda_name_tab[i]) == len && memcmp(nda_name_tab[i], s, len) == 0) 140 242 382
85 140 return i;
86 2 return -1;
87 }
88  
89 /* ================================================================
90 * Structs
91 * ================================================================ */
92  
93 /* Per-process slot for dead-process recovery. In the reader-slots-only rwlock a
94 * reader's ENTIRE contribution to the shared lock is `rdepth` in its OWN slot --
95 * there is no separate shared reader counter to fall out of sync with it -- so a
96 * dead reader's contribution is exactly this one word, which a draining writer
97 * neutralises by clearing the slot's pid (the scan then ignores the slot). No
98 * orphaned counter can exist, so there is no quiescent force-reset and sustained
99 * readers cannot starve a writer. _rsv1/_rsv2 are kept only to preserve the
100 * 16-byte slot size across the already-released builds. */
101 typedef struct {
102 uint32_t pid; /* 0 = unclaimed */
103 uint32_t rdepth; /* read-locks THIS process currently holds (recursion-safe) */
104 uint32_t _rsv1; /* reserved (was waiters_parked); unused, kept for layout size */
105 uint32_t _rsv2; /* reserved (was writers_parked); unused, kept for layout size */
106 } NdaReaderSlot;
107  
108 struct NdaHeader {
109 uint32_t magic, version; /* 0,4 */
110 uint32_t dtype; /* 8 enum NdaDtype */
111 uint32_t ndim; /* 12 1..NDA_MAX_DIMS */
112  
113 /* ---- shape / strides (in ELEMENTS, row-major) ---- */
114 uint64_t shape[NDA_MAX_DIMS]; /* 16 .. 79 */
115 uint64_t strides[NDA_MAX_DIMS]; /* 80 .. 143 */
116 uint64_t size; /* 144 product of shape = total elements */
117 uint32_t itemsize; /* 152 bytes per element */
118 uint32_t _pad0; /* 156 */
119  
120 /* ---- offsets / sizes ---- */
121 uint64_t total_size; /* 160 */
122 uint64_t reader_slots_off; /* 168 */
123 uint64_t data_off; /* 176 */
124 uint64_t array_id; /* 184 stable identity for set-op lock ordering */
125  
126 /* ---- lock + stats ---- */
127 uint32_t wlock; /* 192 WRITER word ONLY: 0 (free) or WRITER_BIT|pid. NOT a reader count. */
128 uint32_t rwait; /* 196 parked-waiter hint (readers+writers blocked on wlock); over-count-safe */
129 uint32_t drain_seq; /* 200 futex bumped by a reader releasing under a draining writer (wakes it) */
130 uint32_t slotless_rdepth; /* 204 readers holding with no reader-slot (documented residual) */
131 uint64_t stat_ops; /* 208 */
132 uint8_t sealed; /* 216 0 = mutable, 1 = frozen (read-only; lock-free reads) */
133 uint8_t _pad[39]; /* 217..255 */
134 };
135 typedef struct NdaHeader NdaHeader;
136  
137 _Static_assert(sizeof(struct NdaHeader) == 256, "NdaHeader must be 256 bytes");
138  
139 /* ---- Process-local handle ---- */
140  
141 typedef struct NdaHandle {
142 NdaHeader *hdr;
143 NdaReaderSlot *reader_slots; /* NDA_READER_SLOTS entries */
144 uint64_t *occ; /* NDA_OCC_WORDS-word slot-occupancy bitmap (trusted layout offset) */
145 void *base; /* mmap base */
146 /* ---- immutable geometry cached at attach (Layer B: never re-read the
147 * peer-writable header for a loop bound / byte extent / data base; a
148 * lock-violating peer that corrupts hdr->{size,itemsize,data_off} after we
149 * attached must not drive an OOB access) ---- */
150 char *data; /* data base = trusted layout, NOT peer hdr->data_off */
151 uint64_t size; /* total element count (immutable after create) */
152 uint32_t dtype; /* enum NdaDtype (immutable after create); every element-width / element-type decision uses THIS, never peer hdr->dtype */
153 uint32_t itemsize; /* bytes per element (immutable after create) */
154 size_t mmap_size;
155 char *path; /* backing file path (strdup'd) */
156 int backing_fd; /* memfd or reopened-fd to close on destroy, -1 for file/anon */
157 uint32_t my_slot_idx; /* UINT32_MAX if all slots taken (no recovery for this handle) */
158 uint32_t cached_pid; /* getpid() cached at last slot claim */
159 uint32_t cached_fork_gen; /* nda_fork_gen value at last slot claim */
160 uint32_t slotless_held; /* read-locks this process holds with no reader-slot */
161 int readonly; /* 1 = frozen O_RDONLY/PROT_READ view: lock-free reads, mutation croaks */
162 dev_t device;
163 ino_t inode; /* backing identity; a copied file has its own lock */
164 } NdaHandle;
165  
166 /* ================================================================
167 * Futex-based write-preferring read-write lock (reader-slots-only)
168 * with dead-process recovery
169 *
170 * The reader count is NOT stored in a shared counter. It is DISTRIBUTED across
171 * per-process reader slots: each slot's `rdepth` is that process's entire
172 * contribution to the lock. A reader publishes its presence in its own slot and
173 * then re-checks the writer word; a writer publishes the writer word and then
174 * scans every slot until all live readers' rdepth reach 0. Sequentially-
175 * consistent store+load on each side (a Dekker handshake) gives mutual exclusion.
176 *
177 * Because a reader's whole contribution is ONE atomic word owned by ONE process,
178 * a crashed reader is recovered by clearing that one slot (CAS its pid to 0) --
179 * there is no second counter to strand, no orphaned +1, and therefore no
180 * quiescent force-reset. A reader killed anywhere in rdlock/rdunlock leaves at
181 * most `rdepth>0` in its dead slot, which the draining writer clears directly, so
182 * sustained read traffic can never starve a writer. Write-preference is inherent
183 * in the gate (new readers see wlock!=0 and yield), so there is no reader-count
184 * yield hack.
185 * ================================================================ */
186  
187 #define NDA_RWLOCK_SPIN_LIMIT 32
188 #define NDA_LOCK_TIMEOUT_SEC 2 /* FUTEX_WAIT timeout for stale-lock detection / drain re-scan */
189  
190 0 static inline void nda_rwlock_spin_pause(void) {
191 #if defined(__x86_64__) || defined(__i386__)
192 0 __asm__ volatile("pause" ::: "memory");
193 #elif defined(__aarch64__)
194 __asm__ volatile("yield" ::: "memory");
195 #else
196 __asm__ volatile("" ::: "memory");
197 #endif
198 0 }
199  
200 /* Writer word encoding: WRITER_BIT|pid when write-locked, 0 when free. */
201 #define NDA_RWLOCK_WRITER_BIT 0x80000000U
202 #define NDA_RWLOCK_PID_MASK 0x7FFFFFFFU
203 #define NDA_RWLOCK_WR(pid) (NDA_RWLOCK_WRITER_BIT | ((uint32_t)(pid) & NDA_RWLOCK_PID_MASK))
204  
205 /* A zombie (dead but not yet reaped) still answers kill(pid,0) as alive, so a
206 * process that crashed while holding the lock and lingers unreaped would never
207 * be recovered. Treat /proc/<pid>/stat state 'Z' as dead. Linux-only (as is
208 * this module); if /proc is unreadable we fall back to "alive" (safe: we never
209 * force-recover a possibly-live holder). */
210 0 static inline int nda_pid_is_zombie(uint32_t pid) {
211 char path[32], buf[256];
212 0 snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
213 0 int fd = open(path, O_RDONLY | O_CLOEXEC);
214 0 ▶ if (fd < 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return 0; 0 0 0
215 0 ssize_t n = read(fd, buf, sizeof(buf) - 1);
216 0 close(fd);
217 0 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 0 0
218 0 buf[n] = '\0';
219 /* "pid (comm) state ..."; comm may contain ')', so scan to the last one. */
220 0 char *rp = strrchr(buf, ')');
221 0 ▶ if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */
Branch: 2 branches
BranchTrueFalseTotal
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 0 0 0
if (!rp || rp + 2 >= buf + n) return 0; /* need ") X" within the bytes read */ 0 0 0
222 0 ▶ return rp[1] == ' ' && rp[2] == 'Z';
Branch: 2 branches
BranchTrueFalseTotal
return rp[1] == ' ' && rp[2] == 'Z'; 0 0 0
return rp[1] == ' ' && rp[2] == 'Z'; 0 0 0
223 }
224 /* 1 if alive or unknown, 0 if definitely dead. Cannot detect PID reuse: a
225 * recycled PID reports "alive" and the slot is not reclaimed until that
226 * process exits. See "Crash Safety" in the POD. */
227 0 static inline int nda_pid_alive(uint32_t pid) {
228 0 ▶ if (pid == 0) return 0; /* no owner recorded, treat as dead */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) return 0; /* no owner recorded, treat as dead */ 0 0 0
229 0 ▶ if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */
Branch: 2 branches
BranchTrueFalseTotal
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 0 0 0
if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */ 0 0 0
230 0 return !nda_pid_is_zombie(pid); /* kill() also succeeds for a zombie -> treat as dead */
231 }
232  
233 /* 1 if ndim/shape/strides describe a row-major array of exactly `size` elements. */
234 20 static inline int nda_shape_ok(const NdaHeader *hdr, uint64_t size) {
235 20 uint32_t nd = hdr->ndim;
236 20 ▶ if (nd < 1 || nd > NDA_MAX_DIMS) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (nd < 1 || nd > NDA_MAX_DIMS) return 0; 20 0 20
if (nd < 1 || nd > NDA_MAX_DIMS) return 0; 0 20 20
237 20 uint64_t n = 1;
238 49 ▶ for (uint32_t d = 0; d < nd; d++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t d = 0; d < nd; d++) { 29 20 49
239 29 ▶ if (hdr->shape[d] < 1 || hdr->shape[d] > UINT64_MAX / n) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (hdr->shape[d] < 1 || hdr->shape[d] > UINT64_MAX / n) return 0; 29 0 29
if (hdr->shape[d] < 1 || hdr->shape[d] > UINT64_MAX / n) return 0; 0 29 29
240 29 n *= hdr->shape[d];
241 }
242 20 ▶ if (n != size) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n != size) return 0; 0 20 20
243 20 uint64_t st = 1;
244 49 ▶ for (int d = (int)nd - 1; d >= 0; d--) {
Branch: 1 branch
BranchTrueFalseTotal
for (int d = (int)nd - 1; d >= 0; d--) { 29 20 49
245 29 ▶ if (hdr->strides[d] != st) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->strides[d] != st) return 0; 0 29 29
246 29 st *= hdr->shape[d];
247 }
248 20 return 1;
249 }
250  
251 /* A reshape killed part-way leaves ndim/shape/strides mixed; the element count
252 * never changes, so fall back to its 1-D view. */
253 0 static inline void nda_repair_shape_locked(NdaHandle *h) {
254 0 ▶ if (nda_shape_ok(h->hdr, h->size)) return;
Branch: 1 branch
BranchTrueFalseTotal
if (nda_shape_ok(h->hdr, h->size)) return; 0 0 0
255 0 h->hdr->ndim = 1;
256 0 h->hdr->shape[0] = h->size;
257 0 h->hdr->strides[0] = 1;
258 }
259  
260 /* Force-recover a stale WRITE lock left by a dead writer (held or mid-drain).
261 * CAS to OUR pid to hold the lock while fixing shared state, then release.
262 * Using our pid (not a bare WRITER_BIT sentinel) means a subsequent recovering
263 * process can detect and re-recover if we crash mid-recovery. */
264 0 static inline void nda_recover_stale_lock(NdaHandle *h, uint32_t observed_wlock) {
265 0 NdaHeader *hdr = h->hdr;
266 0 uint32_t mypid = NDA_RWLOCK_WR((uint32_t)getpid());
267 0 ▶ if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock,
Branch: 1 branch
BranchTrueFalseTotal
if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock, 0 0 0
268 mypid, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
269 0 return;
270 /* A torn shape implies the dead writer had drained every reader. */
271 0 nda_repair_shape_locked(h);
272 0 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
273 0 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 0 0 0
274 0 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
275 }
276  
277 static const struct timespec nda_lock_timeout = { NDA_LOCK_TIMEOUT_SEC, 0 };
278  
279 /* Process-global fork-generation counter. Incremented in the pthread_atfork
280 * child callback so every open handle detects a fork transition on the next
281 * lock call without paying a getpid() syscall on the hot path. */
282 static uint32_t nda_fork_gen = 1;
283 static pthread_once_t nda_atfork_once = PTHREAD_ONCE_INIT;
284 0 static void nda_on_fork_child(void) {
285 0 __atomic_add_fetch(&nda_fork_gen, 1, __ATOMIC_RELAXED);
286 0 }
287 7 static void nda_atfork_init(void) {
288 7 pthread_atfork(NULL, NULL, nda_on_fork_child);
289 7 }
290  
291 /* Occupancy bitmap: set a slot's bit when it is claimed, clear it on clean
292 * release. SEQ_CST so a set bit is ordered before the slot's rdepth can go
293 * non-zero (bit set in claim, which precedes any rdlock), letting a writer's
294 * SEQ_CST bitmap scan never miss a slot a committed reader holds. */
295 117 static inline void nda_occ_set(NdaHandle *h, uint32_t s) {
296 117 __atomic_fetch_or(&h->occ[s >> 6], (uint64_t)1 << (s & 63), __ATOMIC_SEQ_CST);
297 117 }
298 117 static inline void nda_occ_clear(NdaHandle *h, uint32_t s) {
299 117 __atomic_fetch_and(&h->occ[s >> 6], ~((uint64_t)1 << (s & 63)), __ATOMIC_SEQ_CST);
300 117 }
301  
302 /* Ensure this process owns a reader slot. Called from the lock helpers so that
303 * fork()'d children pick up their own slot lazily instead of sharing the
304 * parent's. Hot-path is a single relaxed load + compare; only on a
305 * fork-generation mismatch do we touch getpid() and scan slots. */
306 1823 static inline void nda_claim_reader_slot(NdaHandle *h) {
307 1823 uint32_t cur_gen = __atomic_load_n(&nda_fork_gen, __ATOMIC_RELAXED);
308 1823 ▶ if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1))
Branch: 2 branches
BranchTrueFalseTotal
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 1706 117 1823
if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1)) 1706 0 1706
309 1706 return;
310 /* Cold path -- register the atfork hook once per process, then claim. */
311 117 pthread_once(&nda_atfork_once, nda_atfork_init);
312 /* Re-read after pthread_once: nda_on_fork_child may have bumped it. */
313 117 cur_gen = __atomic_load_n(&nda_fork_gen, __ATOMIC_RELAXED);
314 117 uint32_t now_pid = (uint32_t)getpid();
315 117 h->cached_pid = now_pid;
316 117 ▶ if (cur_gen != h->cached_fork_gen) h->slotless_held = 0; /* fork: child holds none of the parent's slotless read locks */
Branch: 1 branch
BranchTrueFalseTotal
if (cur_gen != h->cached_fork_gen) h->slotless_held = 0; /* fork: child holds none of the parent's slotless read locks */ 117 0 117
317 117 h->cached_fork_gen = cur_gen;
318 117 h->my_slot_idx = UINT32_MAX;
319 117 uint32_t start = now_pid % NDA_READER_SLOTS;
320 /* Pass 1: take a free slot. */
321 124 ▶ for (uint32_t i = 0; i < NDA_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < NDA_READER_SLOTS; i++) { 124 0 124
322 124 uint32_t s = (start + i) % NDA_READER_SLOTS;
323 124 uint32_t expected = 0;
324 124 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[s].pid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[s].pid, 117 7 124
325 &expected, now_pid, 0,
326 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
327 /* Fresh owner holds no read locks yet; clear any stale rdepth left by
328 * a dead predecessor (its contribution is dropped as we take over). */
329 117 __atomic_store_n(&h->reader_slots[s].rdepth, 0, __ATOMIC_RELAXED);
330 117 nda_occ_set(h, s); /* mark occupied BEFORE any rdlock can bump rdepth */
331 117 h->my_slot_idx = s;
332 117 return;
333 }
334 }
335 /* Pass 2: no free slot -- reclaim one whose owner is dead. Safe to take even
336 * if its rdepth>0: clearing pid drops the dead reader's entire contribution
337 * (a writer scan ignores rdepth when pid==0) and we reset rdepth to 0 as we
338 * claim it. */
339 0 ▶ for (uint32_t i = 0; i < NDA_READER_SLOTS; i++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t i = 0; i < NDA_READER_SLOTS; i++) { 0 0 0
340 0 uint32_t dpid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
341 0 ▶ if (dpid == 0 || dpid == now_pid || nda_pid_alive(dpid)) continue;
Branch: 3 branches
BranchTrueFalseTotal
if (dpid == 0 || dpid == now_pid || nda_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || nda_pid_alive(dpid)) continue; 0 0 0
if (dpid == 0 || dpid == now_pid || nda_pid_alive(dpid)) continue; 0 0 0
342 0 uint32_t expected = dpid;
343 0 ▶ if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0, 0 0 0
344 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
345 0 __atomic_store_n(&h->reader_slots[i].rdepth, 0, __ATOMIC_RELAXED);
346 0 nda_occ_set(h, i);
347 0 h->my_slot_idx = i;
348 0 return;
349 }
350 }
351 /* Table full -- leave my_slot_idx = UINT32_MAX so this handle takes the
352 * slotless path (lock still works; recovery of THIS reader's death is the
353 * documented slotless limitation). */
354 }
355  
356 /* Inspect the writer word after a futex-wait timeout. If a dead writer holds
357 * it, force-recover. Dead READERS need no action here: only a writer that owns
358 * wlock drains readers, and it clears dead readers inline in its own scan. */
359 0 static inline void nda_recover_after_timeout(NdaHandle *h) {
360 0 uint32_t val = __atomic_load_n(&h->hdr->wlock, __ATOMIC_RELAXED);
361 0 ▶ if (val >= NDA_RWLOCK_WRITER_BIT) {
Branch: 1 branch
BranchTrueFalseTotal
if (val >= NDA_RWLOCK_WRITER_BIT) { 0 0 0
362 0 uint32_t pid = val & NDA_RWLOCK_PID_MASK;
363 0 ▶ if (!nda_pid_alive(pid))
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_pid_alive(pid)) 0 0 0
364 0 nda_recover_stale_lock(h, val);
365 }
366 0 }
367  
368 /* Bump/drop the parked-waiter hint. Both readers (blocked at the gate) and
369 * writers (blocked acquiring wlock) wait on the wlock futex and use this, so
370 * wrunlock/recover know whether a FUTEX_WAKE is worth a syscall. A waiter
371 * SIGKILLed while parked leaves rwait over-counted -> at most a spurious wake
372 * (harmless); it can never under-count, so no wakeup is lost. */
373 0 static inline void nda_park(NdaHandle *h) {
374 0 __atomic_add_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
375 0 }
376 0 static inline void nda_unpark(NdaHandle *h) {
377 0 __atomic_sub_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
378 0 }
379  
380 /* Publish (inc) / retract (dec) this reader's presence -- its ENTIRE
381 * contribution to the lock. A slotted reader uses its slot's rdepth; a reader
382 * that could not claim a slot uses the global slotless_rdepth. inc() is SEQ_CST
383 * so the wlock re-check that follows it in rdlock forms a Dekker handshake with
384 * the writer's SEQ_CST wlock-store + rdepth-scan. leave() peels slotless first
385 * so a slot claimed mid-hold cannot misattribute the decrement. */
386 1285 static inline void nda_rdepth_inc(NdaHandle *h) {
387 1285 ▶ if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->my_slot_idx != UINT32_MAX) { 1285 0 1285
388 1285 __atomic_add_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_SEQ_CST);
389 } else {
390 0 __atomic_add_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
391 0 h->slotless_held++;
392 }
393 1285 }
394 1285 static inline void nda_rdepth_dec(NdaHandle *h) {
395 1285 ▶ if (h->slotless_held > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (h->slotless_held > 0) { 0 1285 1285
396 0 h->slotless_held--;
397 0 __atomic_sub_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_RELEASE);
398 1285 ▶ } else if (h->my_slot_idx != UINT32_MAX) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (h->my_slot_idx != UINT32_MAX) { 1285 0 1285
399 1285 __atomic_sub_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_RELEASE);
400 }
401 1285 }
402  
403 /* Wake a writer that may be draining readers (it waits on drain_seq). Called
404 * after every rdepth decrement so a released read lock lets the writer re-scan
405 * promptly instead of waiting out its timeout. */
406 1285 static inline void nda_reader_wake_drain(NdaHandle *h) {
407 1285 ▶ if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_ACQUIRE) != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_ACQUIRE) != 0) { 0 1285 1285
408 0 __atomic_add_fetch(&h->hdr->drain_seq, 1, __ATOMIC_RELEASE);
409 0 syscall(SYS_futex, &h->hdr->drain_seq, FUTEX_WAKE, 1, NULL, NULL, 0);
410 }
411 1285 }
412  
413 1285 static inline void nda_rwlock_rdlock(NdaHandle *h) {
414 1285 nda_claim_reader_slot(h);
415 1285 NdaHeader *hdr = h->hdr;
416 1285 for (int spin = 0; ; spin++) {
417 1285 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
418 1285 ▶ if (cur == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur == 0) { 1285 0 1285
419 /* Optimistically take the read: publish rdepth, then re-check wlock.
420 * SEQ_CST inc + SEQ_CST load vs the writer's SEQ_CST wlock CAS +
421 * SEQ_CST rdepth scan: by the single total order of SEQ_CST ops the
422 * two sides cannot both miss each other, so we never hold
423 * concurrently with a writer. */
424 1285 nda_rdepth_inc(h);
425 1285 ▶ if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0) 1285 0 1285
426 1285 return; /* no writer after our publish -> we hold the read lock */
427 /* A writer appeared during our publish -- yield to it (write-preferring). */
428 0 nda_rdepth_dec(h);
429 0 nda_reader_wake_drain(h); /* let the draining writer see rdepth drop */
430 0 spin = 0;
431 0 continue;
432 }
433 /* wlock != 0: a writer holds or is acquiring. Recover if it is dead. */
434 0 if (cur >= NDA_RWLOCK_WRITER_BIT &&
435 0 !nda_pid_alive(cur & NDA_RWLOCK_PID_MASK)) {
436 0 nda_recover_stale_lock(h, cur);
437 0 spin = 0;
438 0 continue;
439 }
440 0 ▶ if (__builtin_expect(spin < NDA_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < NDA_RWLOCK_SPIN_LIMIT, 1)) { 0 0 0
441 0 nda_rwlock_spin_pause();
442 0 continue;
443 }
444 0 nda_park(h);
445 0 cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
446 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
447 0 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
448 &nda_lock_timeout, NULL, 0);
449 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
450 0 nda_unpark(h);
451 0 nda_recover_after_timeout(h);
452 0 spin = 0;
453 0 continue;
454 }
455 }
456 0 nda_unpark(h);
457 0 spin = 0;
458 }
459 }
460  
461 1285 static inline void nda_rwlock_rdunlock(NdaHandle *h) {
462 1285 nda_rdepth_dec(h); /* RELEASE: drop our entire contribution */
463 1285 nda_reader_wake_drain(h); /* if a writer is draining, wake it to re-scan */
464 1285 }
465  
466 538 static inline void nda_rwlock_wrlock(NdaHandle *h) {
467 538 nda_claim_reader_slot(h); /* refresh cached_pid across fork */
468 538 NdaHeader *hdr = h->hdr;
469 /* Encode PID in the wlock word itself (0x80000000 | pid) to eliminate any
470 * crash window between acquiring the lock and storing the owner. */
471 538 uint32_t mypid = NDA_RWLOCK_WR(h->cached_pid);
472 /* Phase 1: acquire the writer word (mutual exclusion among writers). */
473 538 for (int spin = 0; ; spin++) {
474 538 uint32_t expected = 0;
475 538 ▶ if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid,
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid, 538 0 538
476 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
477 538 break;
478 /* Contended: expected now holds the current wlock value. */
479 0 if (expected >= NDA_RWLOCK_WRITER_BIT &&
480 0 !nda_pid_alive(expected & NDA_RWLOCK_PID_MASK)) {
481 0 nda_recover_stale_lock(h, expected);
482 0 spin = 0;
483 0 continue;
484 }
485 0 ▶ if (__builtin_expect(spin < NDA_RWLOCK_SPIN_LIMIT, 1)) {
Branch: 1 branch
BranchTrueFalseTotal
if (__builtin_expect(spin < NDA_RWLOCK_SPIN_LIMIT, 1)) { 0 0 0
486 0 nda_rwlock_spin_pause();
487 0 continue;
488 }
489 0 nda_park(h);
490 0 uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
491 0 ▶ if (cur != 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (cur != 0) { 0 0 0
492 0 long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
493 &nda_lock_timeout, NULL, 0);
494 0 ▶ if (rc == -1 && errno == ETIMEDOUT) {
Branch: 2 branches
BranchTrueFalseTotal
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
if (rc == -1 && errno == ETIMEDOUT) { 0 0 0
495 0 nda_unpark(h);
496 0 nda_recover_after_timeout(h);
497 0 spin = 0;
498 0 continue;
499 }
500 }
501 0 nda_unpark(h);
502 0 spin = 0;
503 }
504 /* Phase 2: we own wlock, so no NEW reader can join (they see wlock!=0 and
505 * yield). Drain the readers that were already holding when we won the CAS.
506 * The SEQ_CST CAS above + the SEQ_CST rdepth loads below are the writer side
507 * of the Dekker handshake. */
508 0 for (;;) {
509 538 uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_ACQUIRE); /* snapshot BEFORE scan */
510 538 int busy = 0;
511 /* Visit only OCCUPIED slots via the occupancy bitmap (SEQ_CST: a committed
512 * reader's bit -- set in claim, before its rdepth++ -- is ordered before
513 * this scan, so no held slot is skipped). O(NDA_OCC_WORDS + live readers)
514 * instead of O(NDA_READER_SLOTS). */
515 9146 ▶ for (uint32_t w = 0; w < NDA_OCC_WORDS; w++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t w = 0; w < NDA_OCC_WORDS; w++) { 8608 538 9146
516 8608 uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
517 9151 ▶ while (word) {
Branch: 1 branch
BranchTrueFalseTotal
while (word) { 543 8608 9151
518 543 uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
519 543 word &= word - 1; /* consume this bit (local copy) */
520 543 uint32_t rd = __atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST);
521 543 ▶ if (rd == 0) continue; /* occupied but not read-locking now */
Branch: 1 branch
BranchTrueFalseTotal
if (rd == 0) continue; /* occupied but not read-locking now */ 543 0 543
522 0 uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
523 0 ▶ if (pid == 0) continue; /* stale rdepth on a freed slot */
Branch: 1 branch
BranchTrueFalseTotal
if (pid == 0) continue; /* stale rdepth on a freed slot */ 0 0 0
524 0 ▶ if (!nda_pid_alive(pid)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_pid_alive(pid)) { 0 0 0
525 /* Dead reader: drop its pid so the slot no longer counts. Leave
526 * the occ bit set (harmless -- a later scan hits pid==0 and skips,
527 * a re-claim re-sets it) to avoid racing a concurrent claimant. */
528 0 uint32_t ep = pid;
529 0 __atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
530 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
531 0 continue;
532 }
533 0 busy = 1; /* live reader still holding */
534 }
535 }
536 /* A live slotless reader keeps us waiting; a crashed slotless reader that
537 * cannot be attributed to a pid is the documented slotless limitation. */
538 538 ▶ if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0) 0 538 538
539 0 busy = 1;
540 538 ▶ if (!busy)
Branch: 1 branch
BranchTrueFalseTotal
if (!busy) 538 0 538
541 538 return; /* exclusive: wlock held + every rdepth 0 */
542 /* Wait for a reader to release (drain_seq bump) or time out to re-scan
543 * (which reclaims any newly-dead slotted reader). */
544 0 syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, &nda_lock_timeout, NULL, 0);
545 }
546 }
547  
548 538 static inline void nda_rwlock_wrunlock(NdaHandle *h) {
549 538 NdaHeader *hdr = h->hdr;
550 538 __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
551 538 ▶ if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
Branch: 1 branch
BranchTrueFalseTotal
if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0) 0 538 538
552 0 syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
553 538 }
554  
555 /* ================================================================
556 * Layout math + data access
557 *
558 * Layout: Header -> reader_slots[1024] -> occ bitmap -> data[size * itemsize]
559 * The reader-slot region is a multiple of 16 bytes, the occ bitmap is 128
560 * bytes, and the header is 256 bytes, so data_off is 16-byte aligned (good for
561 * any element width).
562 * ================================================================ */
563  
564 typedef struct { uint64_t reader_slots, occ, data; } NdaLayout;
565  
566 411 static inline NdaLayout nda_layout(void) {
567 NdaLayout L;
568 411 L.reader_slots = sizeof(struct NdaHeader);
569 411 L.occ = L.reader_slots + (uint64_t)NDA_READER_SLOTS * sizeof(NdaReaderSlot);
570 411 L.data = L.occ + NDA_OCC_BYTES;
571 411 return L;
572 }
573  
574 135 static inline uint64_t nda_total_size(uint64_t data_bytes) {
575 135 NdaLayout L = nda_layout();
576 135 return L.data + data_bytes;
577 }
578  
579 2849 static inline char *nda_data(NdaHandle *h) {
580 2849 return h->data; /* cached trusted layout base; never re-read peer hdr->data_off */
581 }
582  
583 /* Flat element index for a multi-index (caller has bounds-checked each dim). */
584 461 static inline uint64_t nda_flat_offset(NdaHandle *h, const uint64_t *idx, uint32_t ndim) {
585 461 const uint64_t *st = h->hdr->strides;
586 461 uint64_t off = 0;
587 1803 ▶ for (uint32_t d = 0; d < ndim; d++) off += idx[d] * st[d];
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t d = 0; d < ndim; d++) off += idx[d] * st[d]; 1342 461 1803
588 461 return off;
589 }
590  
591 /* ================================================================
592 * Typed element load (callers hold a lock). Read element e as a double.
593 * ================================================================ */
594  
595 1010 static inline double nda_load_nv(NdaHandle *h, uint64_t e) {
596 1010 char *base = nda_data(h);
597 1010 switch (h->dtype) { /* cached: peer hdr->dtype must not resize the element */
598 984 case NDA_F64: { double v; memcpy(&v, base + e*8, 8); return v; }
599 16 case NDA_F32: { float v; memcpy(&v, base + e*4, 4); return (double)v; }
600 0 case NDA_I64: { int64_t v; memcpy(&v, base + e*8, 8); return (double)v; }
601 10 case NDA_I32: { int32_t v; memcpy(&v, base + e*4, 4); return (double)v; }
602 0 case NDA_I16: { int16_t v; memcpy(&v, base + e*2, 2); return (double)v; }
603 0 case NDA_I8: { int8_t v; v = (int8_t)base[e]; return (double)v; }
604 0 case NDA_U64: { uint64_t v; memcpy(&v, base + e*8, 8); return (double)v; }
605 0 case NDA_U32: { uint32_t v; memcpy(&v, base + e*4, 4); return (double)v; }
606 0 case NDA_U16: { uint16_t v; memcpy(&v, base + e*2, 2); return (double)v; }
607 0 case NDA_U8: { uint8_t v; v = (uint8_t)base[e]; return (double)v; }
608 }
609 0 return 0.0;
610 }
611  
612 /* Load element e of a SIGNED-int dtype widened to int64_t (caller holds a
613 * lock; dtype must be one of I64/I32/I16/I8). */
614 26 static inline int64_t nda_load_i64(NdaHandle *h, uint64_t e) {
615 26 char *base = nda_data(h);
616 26 switch (h->dtype) { /* cached: peer hdr->dtype must not resize the element */
617 16 case NDA_I64: { int64_t v; memcpy(&v, base + e*8, 8); return v; }
618 10 case NDA_I32: { int32_t v; memcpy(&v, base + e*4, 4); return (int64_t)v; }
619 0 case NDA_I16: { int16_t v; memcpy(&v, base + e*2, 2); return (int64_t)v; }
620 0 case NDA_I8: { int8_t v = (int8_t)base[e]; return (int64_t)v; }
621 0 default: return 0;
622 }
623 }
624  
625 /* Load element e of an UNSIGNED-int dtype widened to uint64_t (caller holds a
626 * lock; dtype must be one of U64/U32/U16/U8). */
627 4 static inline uint64_t nda_load_u64(NdaHandle *h, uint64_t e) {
628 4 char *base = nda_data(h);
629 4 switch (h->dtype) { /* cached: peer hdr->dtype must not resize the element */
630 4 case NDA_U64: { uint64_t v; memcpy(&v, base + e*8, 8); return v; }
631 0 case NDA_U32: { uint32_t v; memcpy(&v, base + e*4, 4); return (uint64_t)v; }
632 0 case NDA_U16: { uint16_t v; memcpy(&v, base + e*2, 2); return (uint64_t)v; }
633 0 case NDA_U8: { uint8_t v = (uint8_t)base[e]; return (uint64_t)v; }
634 0 default: return 0;
635 }
636 }
637  
638 /* Sum every element as a double (caller holds the read lock). */
639 18 static inline double nda_sum_locked(NdaHandle *h) {
640 18 uint64_t size = h->size, e; /* cached immutable count, not peer hdr->size */
641 18 double acc = 0.0;
642 564 ▶ for (e = 0; e < size; e++) acc += nda_load_nv(h, e);
Branch: 1 branch
BranchTrueFalseTotal
for (e = 0; e < size; e++) acc += nda_load_nv(h, e); 546 18 564
643 18 return acc;
644 }
645  
646 /* Find the flat index of the min (want_max=0) or max (want_max=1) element,
647 * comparing in the element's NATIVE type so that i64/u64 values above 2^53
648 * (which collapse/mis-order as doubles) are ranked exactly. Float dtypes
649 * compare as double. Caller holds the read lock; size >= 1 always. */
650 18 static inline uint64_t nda_argextreme_locked(NdaHandle *h, int want_max) {
651 18 uint64_t size = h->size, e, best = 0; /* cached immutable count, not peer hdr->size */
652 18 uint32_t dt = h->dtype; /* cached: peer hdr->dtype must not resize the element */
653 18 ▶ if (nda_is_float(dt)) {
Branch: 1 branch
BranchTrueFalseTotal
if (nda_is_float(dt)) { 8 10 18
654 8 double bestv = nda_load_nv(h, 0);
655 464 ▶ for (e = 1; e < size; e++) {
Branch: 1 branch
BranchTrueFalseTotal
for (e = 1; e < size; e++) { 456 8 464
656 456 double v = nda_load_nv(h, e);
657 456 ▶ if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; }
Branch: 2 branches
BranchTrueFalseTotal
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 228 228 456
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 218 238 456
658 }
659 10 ▶ } else if (nda_is_signed(dt)) {
Branch: 1 branch
BranchTrueFalseTotal
} else if (nda_is_signed(dt)) { 8 2 10
660 8 int64_t bestv = nda_load_i64(h, 0);
661 26 ▶ for (e = 1; e < size; e++) {
Branch: 1 branch
BranchTrueFalseTotal
for (e = 1; e < size; e++) { 18 8 26
662 18 int64_t v = nda_load_i64(h, e);
663 18 ▶ if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; }
Branch: 2 branches
BranchTrueFalseTotal
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 9 9 18
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 6 12 18
664 }
665 } else {
666 2 uint64_t bestv = nda_load_u64(h, 0);
667 4 ▶ for (e = 1; e < size; e++) {
Branch: 1 branch
BranchTrueFalseTotal
for (e = 1; e < size; e++) { 2 2 4
668 2 uint64_t v = nda_load_u64(h, e);
669 2 ▶ if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; }
Branch: 2 branches
BranchTrueFalseTotal
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 1 1 2
if (want_max ? (v > bestv) : (v < bestv)) { bestv = v; best = e; } 1 1 2
670 }
671 }
672 18 return best;
673 }
674  
675 /* ================================================================
676 * Validate create args + header init / setup / open / destroy
677 * ================================================================ */
678  
679 /* Generate a non-zero per-array identity, used ONLY at create time to order
680 * element-wise set-op lock acquisition consistently across unrelated
681 * processes. Prefers getrandom(); on any failure/short read falls back to a
682 * non-zero mix. Never returns 0. */
683 118 static inline uint64_t nda_gen_array_id(const void *hdr_addr) {
684 static uint32_t nda_id_counter = 0;
685 118 uint64_t id = 0;
686 118 ssize_t r = getrandom(&id, sizeof id, 0);
687 118 ▶ if (r != (ssize_t)sizeof id) {
Branch: 1 branch
BranchTrueFalseTotal
if (r != (ssize_t)sizeof id) { 0 118 118
688 0 uint32_t c = __atomic_add_fetch(&nda_id_counter, 1, __ATOMIC_RELAXED);
689 0 id = ((uint64_t)(uint32_t)getpid() << 32)
690 0 ^ ((uint64_t)c * 0x9E3779B97F4A7C15ull)
691 0 ^ (uint64_t)(uintptr_t)hdr_addr;
692 }
693 118 ▶ if (id == 0) id = 0x9E3779B97F4A7C15ull; /* never 0 */
Branch: 1 branch
BranchTrueFalseTotal
if (id == 0) id = 0x9E3779B97F4A7C15ull; /* never 0 */ 0 118 118
694 118 return id;
695 }
696  
697 /* Validate create args + compute derived shape/strides/size/itemsize.
698 * Single source of truth: the XS layer does NOT duplicate these checks.
699 * On success fills *out_* and returns 1; on failure writes errbuf, returns 0. */
700 136 static int nda_validate_create_args(int dtype, const uint64_t *shape, uint32_t ndim,
701 uint64_t *out_size, uint64_t out_strides[NDA_MAX_DIMS],
702 uint64_t *out_data_bytes, char *errbuf) {
703 136 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 136 0 136
704 136 ▶ if (dtype < 0 || dtype >= NDA_NTYPES) { NDA_ERR("unknown dtype"); return 0; }
Branch: 3 branches
BranchTrueFalseTotal
if (dtype < 0 || dtype >= NDA_NTYPES) { NDA_ERR("unknown dtype"); return 0; } 136 0 136
if (dtype < 0 || dtype >= NDA_NTYPES) { NDA_ERR("unknown dtype"); return 0; } 0 136 136
if (dtype < 0 || dtype >= NDA_NTYPES) { NDA_ERR("unknown dtype"); return 0; } 0 0 0
705 136 ▶ if (ndim < 1) { NDA_ERR("ndim must be >= 1"); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (ndim < 1) { NDA_ERR("ndim must be >= 1"); return 0; } 0 136 136
if (ndim < 1) { NDA_ERR("ndim must be >= 1"); return 0; } 0 0 0
706 136 ▶ if (ndim > NDA_MAX_DIMS) { NDA_ERR("ndim must be <= %d", NDA_MAX_DIMS); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (ndim > NDA_MAX_DIMS) { NDA_ERR("ndim must be <= %d", NDA_MAX_DIMS); return 0; } 0 136 136
if (ndim > NDA_MAX_DIMS) { NDA_ERR("ndim must be <= %d", NDA_MAX_DIMS); return 0; } 0 0 0
707 136 uint64_t size = 1;
708 317 ▶ for (uint32_t d = 0; d < ndim; d++) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t d = 0; d < ndim; d++) { 181 136 317
709 181 ▶ if (shape[d] < 1) { NDA_ERR("shape[%u] must be >= 1", d); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (shape[d] < 1) { NDA_ERR("shape[%u] must be >= 1", d); return 0; } 0 181 181
if (shape[d] < 1) { NDA_ERR("shape[%u] must be >= 1", d); return 0; } 0 0 0
710 /* size *= shape[d] with overflow guard */
711 181 ▶ if (shape[d] > UINT64_MAX / size) { NDA_ERR("shape too large"); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (shape[d] > UINT64_MAX / size) { NDA_ERR("shape too large"); return 0; } 0 181 181
if (shape[d] > UINT64_MAX / size) { NDA_ERR("shape too large"); return 0; } 0 0 0
712 181 size *= shape[d];
713 }
714 136 uint32_t itemsize = nda_itemsize_tab[dtype];
715 136 ▶ if (size > NDA_MAX_BYTES / itemsize) { NDA_ERR("shape too large"); return 0; }
Branch: 2 branches
BranchTrueFalseTotal
if (size > NDA_MAX_BYTES / itemsize) { NDA_ERR("shape too large"); return 0; } 1 135 136
if (size > NDA_MAX_BYTES / itemsize) { NDA_ERR("shape too large"); return 0; } 1 0 1
716 135 uint64_t data_bytes = size * itemsize;
717 /* row-major strides */
718 135 out_strides[ndim - 1] = 1;
719 180 ▶ for (int d = (int)ndim - 2; d >= 0; d--)
Branch: 1 branch
BranchTrueFalseTotal
for (int d = (int)ndim - 2; d >= 0; d--) 45 135 180
720 45 out_strides[d] = out_strides[d + 1] * shape[d + 1];
721 135 *out_size = size;
722 135 *out_data_bytes = data_bytes;
723 135 return 1;
724 }
725  
726 118 static inline void nda_init_header(void *base, int dtype, const uint64_t *shape,
727 uint32_t ndim, uint64_t size,
728 const uint64_t *strides,
729 uint64_t total_size) {
730 118 NdaLayout L = nda_layout();
731 118 NdaHeader *hdr = (NdaHeader *)base;
732 /* Zero the header + reader-slot region (lock-recovery state) and the data
733 buffer (fresh array starts all-zero). */
734 118 memset(base, 0, (size_t)total_size);
735 118 hdr->version = NDA_VERSION;
736 118 hdr->dtype = (uint32_t)dtype;
737 118 hdr->ndim = ndim;
738 272 ▶ for (uint32_t d = 0; d < ndim; d++) { hdr->shape[d] = shape[d]; hdr->strides[d] = strides[d]; }
Branch: 1 branch
BranchTrueFalseTotal
for (uint32_t d = 0; d < ndim; d++) { hdr->shape[d] = shape[d]; hdr->strides[d] = strides[d]; } 154 118 272
739 118 hdr->size = size;
740 118 hdr->itemsize = nda_itemsize_tab[dtype];
741 118 hdr->total_size = total_size;
742 118 hdr->reader_slots_off = L.reader_slots;
743 118 hdr->data_off = L.data;
744 118 hdr->array_id = nda_gen_array_id(base);
745 /* Publish magic LAST, as a release store: it is the commit point, so a
746 creator killed before it leaves magic==0 and never a file mistaken for
747 a valid one. A kill during the field stores leaves one to remove by
748 hand. */
749 118 __atomic_store_n(&hdr->magic, NDA_MAGIC, __ATOMIC_RELEASE);
750 118 __atomic_thread_fence(__ATOMIC_SEQ_CST);
751 118 }
752  
753 137 static inline NdaHandle *nda_setup(void *base, size_t map_size,
754 const char *path, int backing_fd, const struct stat *st) {
755 137 NdaHeader *hdr = (NdaHeader *)base;
756 137 NdaHandle *h = (NdaHandle *)calloc(1, sizeof(NdaHandle));
757 137 ▶ if (!h) {
Branch: 1 branch
BranchTrueFalseTotal
if (!h) { 0 137 137
758 0 munmap(base, map_size);
759 0 ▶ if (backing_fd >= 0) close(backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (backing_fd >= 0) close(backing_fd); 0 0 0
760 0 return NULL;
761 }
762 137 NdaLayout L = nda_layout();
763 137 h->hdr = hdr;
764 137 h->base = base;
765 137 h->reader_slots = (NdaReaderSlot *)((uint8_t *)base + sizeof(NdaHeader)); /* trusted layout, not the peer-writable header offset */
766 137 h->occ = (uint64_t *)((uint8_t *)base + L.occ); /* trusted layout offset */
767 137 h->data = (char *)base + L.data; /* trusted layout, not the peer-writable hdr->data_off */
768 137 h->size = hdr->size; /* cache immutable geometry (validated at attach) */
769 /* Cache dtype too: a lock-violating peer that corrupts hdr->dtype after we
770 attached must not change the per-element WIDTH (nb) while size/data stay
771 fixed -> OOB. Validate it in range exactly like the other cached
772 geometry (every create/attach path has already validated it; clamp
773 defensively so an out-of-range value can never index a typed table). */
774 137 ▶ h->dtype = (hdr->dtype < NDA_NTYPES) ? hdr->dtype : NDA_F64;
Branch: 1 branch
BranchTrueFalseTotal
h->dtype = (hdr->dtype < NDA_NTYPES) ? hdr->dtype : NDA_F64; 137 0 137
775 137 h->itemsize = hdr->itemsize;
776 137 h->mmap_size = map_size;
777 137 ▶ h->path = path ? strdup(path) : NULL;
Branch: 1 branch
BranchTrueFalseTotal
h->path = path ? strdup(path) : NULL; 34 103 137
778 137 h->backing_fd = backing_fd;
779 137 h->my_slot_idx = UINT32_MAX;
780 137 ▶ if (st) { h->device = st->st_dev; h->inode = st->st_ino; }
Branch: 1 branch
BranchTrueFalseTotal
if (st) { h->device = st->st_dev; h->inode = st->st_ino; } 37 100 137
781 137 return h;
782 }
783  
784 /* Validate a mapped header (shared by reopen and open_fd). Stored
785 * dtype/shape/strides/size win on reopen; require self-consistency and the
786 * file size to match. */
787 25 static inline int nda_validate_fixed(const NdaHeader *hdr, uint64_t file_size) {
788 25 ▶ if (hdr->magic != NDA_MAGIC) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->magic != NDA_MAGIC) return 0; 4 21 25
789 21 ▶ if (hdr->version != NDA_VERSION) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->version != NDA_VERSION) return 0; 0 21 21
790 21 ▶ if (hdr->dtype >= NDA_NTYPES) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->dtype >= NDA_NTYPES) return 0; 0 21 21
791 21 ▶ if (hdr->itemsize != nda_itemsize_tab[hdr->dtype]) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->itemsize != nda_itemsize_tab[hdr->dtype]) return 0; 0 21 21
792 21 uint64_t size = hdr->size;
793 21 ▶ if (size < 1 || size > NDA_MAX_BYTES / hdr->itemsize) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (size < 1 || size > NDA_MAX_BYTES / hdr->itemsize) return 0; 21 0 21
if (size < 1 || size > NDA_MAX_BYTES / hdr->itemsize) return 0; 0 21 21
794 21 uint64_t data_bytes = size * hdr->itemsize;
795 21 NdaLayout L = nda_layout();
796 21 ▶ if (hdr->reader_slots_off != L.reader_slots) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->reader_slots_off != L.reader_slots) return 0; 0 21 21
797 21 ▶ if (hdr->data_off != L.data) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->data_off != L.data) return 0; 0 21 21
798 21 ▶ if (hdr->total_size != L.data + data_bytes) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->total_size != L.data + data_bytes) return 0; 0 21 21
799 21 ▶ if (hdr->total_size != file_size) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (hdr->total_size != file_size) return 0; 0 21 21
800 21 return 1;
801 }
802  
803 5 static inline int nda_validate_header(const NdaHeader *hdr, uint64_t file_size) {
804 5 ▶ return nda_validate_fixed(hdr, file_size) && nda_shape_ok(hdr, hdr->size);
Branch: 2 branches
BranchTrueFalseTotal
return nda_validate_fixed(hdr, file_size) && nda_shape_ok(hdr, hdr->size); 5 0 5
return nda_validate_fixed(hdr, file_size) && nda_shape_ok(hdr, hdr->size); 5 0 5
805 }
806  
807 /* Read-write attach: a shape torn by a killed reshape is repaired under the
808 * write lock (which also recovers the dead writer) instead of refusing the file. */
809 15 static NdaHandle *nda_setup_rw(void *base, size_t map_size, const char *path, int backing_fd,
810 const struct stat *st) {
811 15 NdaHandle *h = nda_setup(base, map_size, path, backing_fd, st);
812 15 ▶ if (h && !nda_shape_ok(h->hdr, h->size)) {
Branch: 2 branches
BranchTrueFalseTotal
if (h && !nda_shape_ok(h->hdr, h->size)) { 15 0 15
if (h && !nda_shape_ok(h->hdr, h->size)) { 0 15 15
813 0 nda_rwlock_wrlock(h);
814 0 nda_repair_shape_locked(h);
815 0 nda_rwlock_wrunlock(h);
816 }
817 15 return h;
818 }
819  
820 /* Securely obtain a fd: create exclusively (O_CREAT|O_EXCL|O_NOFOLLOW at mode,
821 * default 0600), or attach an existing file (O_RDWR|O_NOFOLLOW, no O_CREAT). */
822 35 static int nda_secure_open(const char *path, mode_t mode, char *errbuf) {
823 35 ▶ for (int attempt = 0; attempt < 100; attempt++) {
Branch: 1 branch
BranchTrueFalseTotal
for (int attempt = 0; attempt < 100; attempt++) { 35 0 35
824 35 int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, mode);
825 35 ▶ if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { (void)fchmod(fd, mode); return fd; } /* exact mode: umask narrowed the O_EXCL create */ 17 18 35
826 18 ▶ if (errno != EEXIST) { NDA_ERR("create %s: %s", path, strerror(errno)); return -1; }
Branch: 2 branches
BranchTrueFalseTotal
if (errno != EEXIST) { NDA_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 18 18
if (errno != EEXIST) { NDA_ERR("create %s: %s", path, strerror(errno)); return -1; } 0 0 0
827 18 fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
828 18 ▶ if (fd >= 0) return fd;
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) return fd; 18 0 18
829 0 ▶ if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */
Branch: 1 branch
BranchTrueFalseTotal
if (errno == ENOENT) continue; /* creator unlinked between our two opens; retry */ 0 0 0
830 0 ▶ NDA_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("open %s: %s", path, strerror(errno)); /* ELOOP => symlink rejected */ 0 0 0
831 0 return -1;
832 }
833 0 ▶ NDA_ERR("open %s: create/attach kept racing", path);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("open %s: create/attach kept racing", path); 0 0 0
834 0 return -1;
835 }
836  
837 /* True iff the whole file is zero -- what an abandoned mid-init
838 creator leaves. Lets recovery re-init only a provably-empty file, never
839 one that merely starts with a zero word. Cold path, so a byte scan is
840 fine. Read, not mapped: a read fault on a tmpfs hole allocates the page. */
841 2 static int nda_file_is_zero(int fd, uint64_t size) {
842 char buf[16384];
843 5 ▶ for (uint64_t off = 0; off < size; ) {
Branch: 1 branch
BranchTrueFalseTotal
for (uint64_t off = 0; off < size; ) { 4 1 5
844 4 ssize_t n = pread(fd, buf, size - off < sizeof buf ? (size_t)(size - off) : sizeof buf, (off_t)off);
845 4 ▶ if (n <= 0) return 0;
Branch: 1 branch
BranchTrueFalseTotal
if (n <= 0) return 0; 0 4 4
846 33635 ▶ for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0;
Branch: 2 branches
BranchTrueFalseTotal
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 1 33631 33632
for (ssize_t i = 0; i < n; i++) if (buf[i]) return 0; 33632 3 33635
847 3 off += (uint64_t)n;
848 }
849 1 return 1;
850 }
851  
852 /* A sparse segment would SIGBUS at the first write the filesystem cannot back. */
853 23 static int nda_reserve(int fd, uint64_t size) {
854 23 const char *sp = getenv("DATA_NDARRAY_SHARED_SPARSE");
855 23 ▶ if (sp && *sp && strcmp(sp, "0")) return 0;
Branch: 3 branches
BranchTrueFalseTotal
if (sp && *sp && strcmp(sp, "0")) return 0; 0 23 23
if (sp && *sp && strcmp(sp, "0")) return 0; 0 0 0
if (sp && *sp && strcmp(sp, "0")) return 0; 0 0 0
856 /* Before Linux 6.11 tmpfs gives up at any pending signal and undoes the allocation, so a
857 * periodic one could keep it from ever completing; SIGSTOP cannot be blocked. */
858 sigset_t all, old;
859 23 sigfillset(&all);
860 23 sigprocmask(SIG_BLOCK, &all, &old);
861 int e;
862 23 ▶ do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR);
Branch: 1 branch
BranchTrueFalseTotal
do e = posix_fallocate(fd, 0, (off_t)size); while (e == EINTR); 0 23 23
863 23 sigprocmask(SIG_SETMASK, &old, NULL);
864 23 ▶ if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0;
Branch: 3 branches
BranchTrueFalseTotal
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 23 23
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 0 0
if (e == 0 || e == EOPNOTSUPP || e == EINVAL) return 0; 0 0 0
865 0 errno = e;
866 0 return -1;
867 }
868  
869 131 static NdaHandle *nda_create(const char *path, int dtype,
870 const uint64_t *shape, uint32_t ndim, mode_t mode, char *errbuf) {
871 uint64_t size, strides[NDA_MAX_DIMS], data_bytes;
872 131 ▶ if (!nda_validate_create_args(dtype, shape, ndim, &size, strides, &data_bytes, errbuf))
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_validate_create_args(dtype, shape, ndim, &size, strides, &data_bytes, errbuf)) 1 130 131
873 1 return NULL;
874  
875 130 uint64_t total = nda_total_size(data_bytes);
876 130 int anonymous = (path == NULL);
877 130 int fd = -1;
878 size_t map_size;
879 void *base;
880 struct stat stt;
881  
882 130 ▶ if (anonymous) {
Branch: 1 branch
BranchTrueFalseTotal
if (anonymous) { 95 35 130
883 95 map_size = (size_t)total;
884 95 base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
885 95 ▶ if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; } 0 95 95
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; } 0 0 0
886 } else {
887 35 fd = nda_secure_open(path, mode, errbuf);
888 35 ▶ if (fd < 0) return NULL;
Branch: 1 branch
BranchTrueFalseTotal
if (fd < 0) return NULL; 0 35 35
889 35 ▶ if (flock(fd, LOCK_EX) < 0) { NDA_ERR("flock: %s", strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (flock(fd, LOCK_EX) < 0) { NDA_ERR("flock: %s", strerror(errno)); close(fd); return NULL; } 0 35 35
if (flock(fd, LOCK_EX) < 0) { NDA_ERR("flock: %s", strerror(errno)); close(fd); return NULL; } 0 0 0
890 35 ▶ if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 35 35
if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } 0 0 0
891 35 int is_new = (stt.st_size == 0);
892 35 ▶ if (!is_new && (uint64_t)stt.st_size < sizeof(struct NdaHeader)) {
Branch: 2 branches
BranchTrueFalseTotal
if (!is_new && (uint64_t)stt.st_size < sizeof(struct NdaHeader)) { 18 17 35
if (!is_new && (uint64_t)stt.st_size < sizeof(struct NdaHeader)) { 1 17 18
893 1 ▶ NDA_ERR("%s: file too small (%lld)", path, (long long)stt.st_size);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: file too small (%lld)", path, (long long)stt.st_size); 1 0 1
894 1 flock(fd, LOCK_UN); close(fd); return NULL;
895 }
896 34 ▶ if (is_new && (stt.st_uid != geteuid() || fchmod(fd, mode) < 0)) {
Branch: 3 branches
BranchTrueFalseTotal
if (is_new && (stt.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 17 17 34
if (is_new && (stt.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 17 0 17
if (is_new && (stt.st_uid != geteuid() || fchmod(fd, mode) < 0)) { 0 17 17
897 0 ▶ NDA_ERR("%s: refusing to initialize file not owned by us", path);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: refusing to initialize file not owned by us", path); 0 0 0
898 0 flock(fd, LOCK_UN); close(fd); return NULL;
899 }
900 34 ▶ if (is_new && ftruncate(fd, (off_t)total) < 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (is_new && ftruncate(fd, (off_t)total) < 0) { 17 17 34
if (is_new && ftruncate(fd, (off_t)total) < 0) { 0 17 17
901 0 ▶ NDA_ERR("ftruncate: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("ftruncate: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; 0 0 0
902 }
903 34 ▶ if (is_new && nda_reserve(fd, total) < 0) {
Branch: 2 branches
BranchTrueFalseTotal
if (is_new && nda_reserve(fd, total) < 0) { 17 17 34
if (is_new && nda_reserve(fd, total) < 0) { 0 17 17
904 0 ▶ NDA_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno)); 0 0 0
905 0 if (ftruncate(fd, 0) < 0) { /* best effort */ }
906 0 flock(fd, LOCK_UN); close(fd); return NULL;
907 }
908 34 ▶ map_size = is_new ? (size_t)total : (size_t)stt.st_size;
Branch: 1 branch
BranchTrueFalseTotal
map_size = is_new ? (size_t)total : (size_t)stt.st_size; 17 17 34
909 34 base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
910 34 ▶ if (base == MAP_FAILED) {
Branch: 1 branch
BranchTrueFalseTotal
if (base == MAP_FAILED) { 0 34 34
911 0 ▶ NDA_ERR("mmap: %s", strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("mmap: %s", strerror(errno)); 0 0 0
912 0 ▶ if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ }
Branch: 1 branch
BranchTrueFalseTotal
if (is_new && ftruncate(fd, 0) < 0) { /* best effort */ } 0 0 0
913 0 flock(fd, LOCK_UN); close(fd); return NULL;
914 }
915 34 ▶ if (!is_new) {
Branch: 1 branch
BranchTrueFalseTotal
if (!is_new) { 17 17 34
916 17 ▶ if (!nda_validate_fixed((NdaHeader *)base, (uint64_t)stt.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_validate_fixed((NdaHeader *)base, (uint64_t)stt.st_size)) { 4 13 17
917 /* Recover an abandoned mid-init file: a creator killed
918 * between the ftruncate and the header init leaves a
919 * full-size, all-zero (magic==0) file that would brick every
920 * future open of this path. Re-initialize ONLY when it is
921 * exactly our size, still uninitialized, and owned by us;
922 * anything else still errors. */
923 4 ▶ if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total
Branch: 2 branches
BranchTrueFalseTotal
if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total 3 1 4
if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total 2 1 3
924 2 ▶ && stt.st_uid == geteuid() && nda_file_is_zero(fd, map_size)) {
Branch: 2 branches
BranchTrueFalseTotal
&& stt.st_uid == geteuid() && nda_file_is_zero(fd, map_size)) { 2 0 2
&& stt.st_uid == geteuid() && nda_file_is_zero(fd, map_size)) { 1 1 2
925 1 ▶ if (fchmod(fd, mode) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (fchmod(fd, mode) < 0) { 0 1 1
926 0 ▶ NDA_ERR("%s: fchmod: %s", path, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: fchmod: %s", path, strerror(errno)); 0 0 0
927 0 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
928 }
929 1 ▶ if (nda_reserve(fd, total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (nda_reserve(fd, total) < 0) { 0 1 1
930 0 ▶ NDA_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: cannot reserve %llu bytes: %s", path, (unsigned long long)total, strerror(errno)); 0 0 0
931 0 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
932 }
933 1 nda_init_header(base, dtype, shape, ndim, size, strides, total);
934 1 flock(fd, LOCK_UN); close(fd);
935 1 return nda_setup(base, map_size, path, -1, &stt);
936 }
937 3 ▶ if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total
Branch: 2 branches
BranchTrueFalseTotal
if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total 2 1 3
if (((NdaHeader *)base)->magic == 0 && (uint64_t)stt.st_size == total 1 1 2
938 1 ▶ && stt.st_uid == geteuid()) {
Branch: 1 branch
BranchTrueFalseTotal
&& stt.st_uid == geteuid()) { 1 0 1
939 1 ▶ NDA_ERR("%s: incomplete ndarray file left by an interrupted create; remove it and retry", path);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: incomplete ndarray file left by an interrupted create; remove it and retry", path); 1 0 1
940 1 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
941 }
942 2 ▶ NDA_ERR("invalid ndarray file"); munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("invalid ndarray file"); munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL; 2 0 2
943 }
944 13 ▶ if (((NdaHeader *)base)->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (((NdaHeader *)base)->sealed) { 1 12 13
945 1 ▶ NDA_ERR("%s is frozen (read-only); open it with new_readonly", path);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s is frozen (read-only); open it with new_readonly", path); 1 0 1
946 1 munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
947 }
948 12 flock(fd, LOCK_UN); close(fd);
949 12 return nda_setup_rw(base, map_size, path, -1, &stt);
950 }
951 }
952 112 nda_init_header(base, dtype, shape, ndim, size, strides, total);
953 112 ▶ if (fd >= 0) { flock(fd, LOCK_UN); close(fd); }
Branch: 1 branch
BranchTrueFalseTotal
if (fd >= 0) { flock(fd, LOCK_UN); close(fd); } 17 95 112
954 112 ▶ return nda_setup(base, map_size, path, -1, anonymous ? NULL : &stt);
Branch: 1 branch
BranchTrueFalseTotal
return nda_setup(base, map_size, path, -1, anonymous ? NULL : &stt); 17 95 112
955 }
956  
957 5 static NdaHandle *nda_create_memfd(const char *name, int dtype,
958 const uint64_t *shape, uint32_t ndim, char *errbuf) {
959 uint64_t size, strides[NDA_MAX_DIMS], data_bytes;
960 5 ▶ if (!nda_validate_create_args(dtype, shape, ndim, &size, strides, &data_bytes, errbuf))
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_validate_create_args(dtype, shape, ndim, &size, strides, &data_bytes, errbuf)) 0 5 5
961 0 return NULL;
962  
963 5 uint64_t total = nda_total_size(data_bytes);
964 5 ▶ int fd = memfd_create(name ? name : "ndarray", MFD_CLOEXEC | MFD_ALLOW_SEALING);
Branch: 1 branch
BranchTrueFalseTotal
int fd = memfd_create(name ? name : "ndarray", MFD_CLOEXEC | MFD_ALLOW_SEALING); 1 4 5
965 5 ▶ if (fd < 0) { NDA_ERR("memfd_create: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fd < 0) { NDA_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 5 5
if (fd < 0) { NDA_ERR("memfd_create: %s", strerror(errno)); return NULL; } 0 0 0
966 5 ▶ if (ftruncate(fd, (off_t)total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (ftruncate(fd, (off_t)total) < 0) { 0 5 5
967 0 ▶ NDA_ERR("ftruncate: %s", strerror(errno)); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("ftruncate: %s", strerror(errno)); close(fd); return NULL; 0 0 0
968 }
969 5 ▶ if (nda_reserve(fd, total) < 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (nda_reserve(fd, total) < 0) { 0 5 5
970 0 ▶ NDA_ERR("memfd: cannot reserve %llu bytes: %s", (unsigned long long)total, strerror(errno));
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("memfd: cannot reserve %llu bytes: %s", (unsigned long long)total, strerror(errno)); 0 0 0
971 0 close(fd); return NULL;
972 }
973 5 (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);
974 5 void *base = mmap(NULL, (size_t)total, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
975 5 ▶ if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; } 0 5 5
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; } 0 0 0
976 5 nda_init_header(base, dtype, shape, ndim, size, strides, total);
977 5 return nda_setup(base, (size_t)total, NULL, fd, NULL);
978 }
979  
980 5 static NdaHandle *nda_open_fd(int fd, char *errbuf) {
981 5 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 5 0 5
982 struct stat stt;
983 5 ▶ if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); return NULL; } 0 5 5
if (fstat(fd, &stt) < 0) { NDA_ERR("fstat: %s", strerror(errno)); return NULL; } 0 0 0
984 5 ▶ if ((uint64_t)stt.st_size < sizeof(struct NdaHeader)) { NDA_ERR("too small"); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if ((uint64_t)stt.st_size < sizeof(struct NdaHeader)) { NDA_ERR("too small"); return NULL; } 1 4 5
if ((uint64_t)stt.st_size < sizeof(struct NdaHeader)) { NDA_ERR("too small"); return NULL; } 1 0 1
985 /* before mmap: a creator may truncate the file until it has set magic */
986 uint32_t magic;
987 4 ▶ if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) { 4 0 4
if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) { 0 4 4
988 0 ▶ NDA_ERR("invalid ndarray"); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("invalid ndarray"); return NULL; 0 0 0
989 }
990 4 size_t ms = (size_t)stt.st_size;
991 4 void *base = mmap(NULL, ms, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
992 4 ▶ if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; } 1 3 4
if (base == MAP_FAILED) { NDA_ERR("mmap: %s", strerror(errno)); return NULL; } 1 0 1
993 3 ▶ if (!nda_validate_fixed((NdaHeader *)base, (uint64_t)stt.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_validate_fixed((NdaHeader *)base, (uint64_t)stt.st_size)) { 0 3 3
994 0 ▶ NDA_ERR("invalid ndarray"); munmap(base, ms); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("invalid ndarray"); munmap(base, ms); return NULL; 0 0 0
995 }
996 3 ▶ if (((NdaHeader *)base)->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (((NdaHeader *)base)->sealed) { 0 3 3
997 0 ▶ NDA_ERR("this ndarray is frozen (read-only); open it with new_readonly");
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("this ndarray is frozen (read-only); open it with new_readonly"); 0 0 0
998 0 munmap(base, ms); return NULL;
999 }
1000 3 int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
1001 3 ▶ if (myfd < 0) { NDA_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (myfd < 0) { NDA_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; } 0 3 3
if (myfd < 0) { NDA_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; } 0 0 0
1002 3 return nda_setup_rw(base, ms, NULL, myfd, &stt);
1003 }
1004  
1005 137 static void nda_destroy(NdaHandle *h) {
1006 137 ▶ if (!h) return;
Branch: 1 branch
BranchTrueFalseTotal
if (!h) return; 0 137 137
1007 /* Release our reader slot on clean teardown (else short-lived-reader churn
1008 * exhausts the slot table); skip if a read lock is still held (rdepth>0). */
1009 137 ▶ if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid &&
Branch: 3 branches
BranchTrueFalseTotal
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 137 0 137
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 117 20 137
if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid && 117 0 117
1010 117 ▶ h->cached_fork_gen == __atomic_load_n(&nda_fork_gen, __ATOMIC_RELAXED) &&
Branch: 1 branch
BranchTrueFalseTotal
h->cached_fork_gen == __atomic_load_n(&nda_fork_gen, __ATOMIC_RELAXED) && 117 0 117
1011 117 ▶ __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) {
Branch: 1 branch
BranchTrueFalseTotal
__atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) { 117 0 117
1012 /* Clear our occ bit BEFORE freeing the slot: we still own the pid so no
1013 * claimant can take the slot mid-clear, and rdepth==0 so no writer needs
1014 * to see us. (A crash skips this -> the bit is reclaimed lazily by a
1015 * writer scan / re-claim, same as the pid.) */
1016 117 nda_occ_clear(h, h->my_slot_idx);
1017 117 uint32_t expected = h->cached_pid;
1018 117 __atomic_compare_exchange_n(&h->reader_slots[h->my_slot_idx].pid,
1019 &expected, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
1020 }
1021 137 ▶ if (h->backing_fd >= 0) close(h->backing_fd);
Branch: 1 branch
BranchTrueFalseTotal
if (h->backing_fd >= 0) close(h->backing_fd); 8 129 137
1022 137 ▶ if (h->base) munmap(h->base, h->mmap_size);
Branch: 1 branch
BranchTrueFalseTotal
if (h->base) munmap(h->base, h->mmap_size); 137 0 137
1023 137 free(h->path);
1024 137 free(h);
1025 }
1026  
1027 5 static inline int nda_msync(NdaHandle *h) {
1028 5 ▶ if (!h || !h->base) return 0;
Branch: 2 branches
BranchTrueFalseTotal
if (!h || !h->base) return 0; 5 0 5
if (!h || !h->base) return 0; 0 5 5
1029 5 return msync(h->base, h->mmap_size, MS_SYNC);
1030 }
1031  
1032 /* Open a FROZEN (sealed) file read-only: O_RDONLY + PROT_READ, no lock ever.
1033 * A sealed file is immutable and no read path writes the mapping, so queries
1034 * take no reader-slot / rwlock traffic and any number of processes can share
1035 * one PROT_READ mapping (same architecture; the magic rejects a wrong-endian
1036 * file). */
1037 6 static NdaHandle *nda_open_readonly(const char *path, char *errbuf) {
1038 6 ▶ if (errbuf) errbuf[0] = '\0';
Branch: 1 branch
BranchTrueFalseTotal
if (errbuf) errbuf[0] = '\0'; 6 0 6
1039 6 int fd = open(path, O_RDONLY|O_NOFOLLOW|O_CLOEXEC);
1040 6 ▶ if (fd < 0) { NDA_ERR("open %s: %s", path, strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fd < 0) { NDA_ERR("open %s: %s", path, strerror(errno)); return NULL; } 1 5 6
if (fd < 0) { NDA_ERR("open %s: %s", path, strerror(errno)); return NULL; } 1 0 1
1041 struct stat st;
1042 5 ▶ if (fstat(fd, &st) < 0) { NDA_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (fstat(fd, &st) < 0) { NDA_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; } 0 5 5
if (fstat(fd, &st) < 0) { NDA_ERR("fstat %s: %s", path, strerror(errno)); close(fd); return NULL; } 0 0 0
1043 5 ▶ if ((uint64_t)st.st_size < sizeof(struct NdaHeader)) { NDA_ERR("%s: file too small", path); close(fd); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if ((uint64_t)st.st_size < sizeof(struct NdaHeader)) { NDA_ERR("%s: file too small", path); close(fd); return NULL; } 0 5 5
if ((uint64_t)st.st_size < sizeof(struct NdaHeader)) { NDA_ERR("%s: file too small", path); close(fd); return NULL; } 0 0 0
1044 /* before mmap: a creator may truncate the file until it has set magic */
1045 uint32_t magic;
1046 5 ▶ if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) {
Branch: 2 branches
BranchTrueFalseTotal
if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) { 5 0 5
if (pread(fd, &magic, sizeof magic, offsetof(NdaHeader, magic)) != (ssize_t)sizeof magic || magic != NDA_MAGIC) { 0 5 5
1047 0 ▶ NDA_ERR("%s: invalid ndarray file", path); close(fd); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: invalid ndarray file", path); close(fd); return NULL; 0 0 0
1048 }
1049 5 size_t ms = (size_t)st.st_size;
1050 5 void *base = mmap(NULL, ms, PROT_READ, MAP_SHARED, fd, 0);
1051 5 close(fd); /* the mapping keeps the file; a read-only view needs no fd (no msync/ftruncate) */
1052 5 ▶ if (base == MAP_FAILED) { NDA_ERR("mmap %s: %s", path, strerror(errno)); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (base == MAP_FAILED) { NDA_ERR("mmap %s: %s", path, strerror(errno)); return NULL; } 0 5 5
if (base == MAP_FAILED) { NDA_ERR("mmap %s: %s", path, strerror(errno)); return NULL; } 0 0 0
1053 5 ▶ if (!nda_validate_header((NdaHeader *)base, (uint64_t)st.st_size)) {
Branch: 1 branch
BranchTrueFalseTotal
if (!nda_validate_header((NdaHeader *)base, (uint64_t)st.st_size)) { 0 5 5
1054 0 ▶ NDA_ERR("%s: invalid ndarray file", path); munmap(base, ms); return NULL;
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s: invalid ndarray file", path); munmap(base, ms); return NULL; 0 0 0
1055 }
1056 5 ▶ if (!((NdaHeader *)base)->sealed) {
Branch: 1 branch
BranchTrueFalseTotal
if (!((NdaHeader *)base)->sealed) { 1 4 5
1057 1 ▶ NDA_ERR("%s is not frozen: call ->freeze on the producer before opening read-only", path);
Branch: 1 branch
BranchTrueFalseTotal
NDA_ERR("%s is not frozen: call ->freeze on the producer before opening read-only", path); 1 0 1
1058 1 munmap(base, ms); return NULL;
1059 }
1060 4 NdaHandle *h = nda_setup(base, ms, path, -1, &st); /* munmaps on OOM */
1061 4 ▶ if (!h) { NDA_ERR("out of memory"); return NULL; }
Branch: 2 branches
BranchTrueFalseTotal
if (!h) { NDA_ERR("out of memory"); return NULL; } 0 4 4
if (!h) { NDA_ERR("out of memory"); return NULL; } 0 0 0
1062 4 h->readonly = 1;
1063 4 return h;
1064 }
1065  
1066 /* Seal an array: make it permanently immutable so it can be shipped and opened
1067 * read-only. Takes the write lock so no mutation is in flight, publishes the
1068 * seal, then flushes it (file/memfd-backed). Afterwards every mutator croaks
1069 * and a read-write reopen is refused. */
1070 3 static int nda_freeze(NdaHandle *h) {
1071 3 nda_rwlock_wrlock(h);
1072 3 h->hdr->sealed = 1;
1073 3 nda_rwlock_wrunlock(h);
1074 3 ▶ if (h->path || h->backing_fd >= 0) return nda_msync(h); /* durability for file/memfd-backed */
Branch: 2 branches
BranchTrueFalseTotal
if (h->path || h->backing_fd >= 0) return nda_msync(h); /* durability for file/memfd-backed */ 0 3 3
if (h->path || h->backing_fd >= 0) return nda_msync(h); /* durability for file/memfd-backed */ 0 0 0
1075 0 return 0; /* anonymous: the seal lives in shared memory (visible to forks); nothing to flush */
1076 }
1077  
1078 #endif /* NDARRAY_H */
« blib/lib/Data/NDArray/Shared.pm ↑ Summary Shared.xs »