| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
/* |
|
2
|
|
|
|
|
|
|
* stack.h -- Fixed-size shared-memory LIFO stack for Linux |
|
3
|
|
|
|
|
|
|
* |
|
4
|
|
|
|
|
|
|
* CAS-based position handout (top) with per-slot publication state |
|
5
|
|
|
|
|
|
|
* machine (EMPTY -> WRITING -> FILLED -> READING -> EMPTY, generation |
|
6
|
|
|
|
|
|
|
* bumped on release). The state machine closes the race between a |
|
7
|
|
|
|
|
|
|
* position CAS and the corresponding slot write/read under MPMC. |
|
8
|
|
|
|
|
|
|
* Futex blocking via push/pop_wake_seq when empty (pop) or full (push). |
|
9
|
|
|
|
|
|
|
* |
|
10
|
|
|
|
|
|
|
* Variants: Int (int64_t), Str (length-prefixed) |
|
11
|
|
|
|
|
|
|
*/ |
|
12
|
|
|
|
|
|
|
|
|
13
|
|
|
|
|
|
|
#ifndef STACK_H |
|
14
|
|
|
|
|
|
|
#define STACK_H |
|
15
|
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
#include |
|
17
|
|
|
|
|
|
|
#include |
|
18
|
|
|
|
|
|
|
#include |
|
19
|
|
|
|
|
|
|
#include |
|
20
|
|
|
|
|
|
|
#include |
|
21
|
|
|
|
|
|
|
#include |
|
22
|
|
|
|
|
|
|
#include |
|
23
|
|
|
|
|
|
|
#include |
|
24
|
|
|
|
|
|
|
#include |
|
25
|
|
|
|
|
|
|
#include |
|
26
|
|
|
|
|
|
|
#include |
|
27
|
|
|
|
|
|
|
#include |
|
28
|
|
|
|
|
|
|
#include |
|
29
|
|
|
|
|
|
|
#include |
|
30
|
|
|
|
|
|
|
#include |
|
31
|
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
#define STK_MAGIC 0x53544B32U /* "STK2" — v2 layout (per-slot ctl) */ |
|
33
|
|
|
|
|
|
|
#define STK_VERSION 2 |
|
34
|
|
|
|
|
|
|
#define STK_ERR_BUFLEN 256 |
|
35
|
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
#define STK_VAR_INT 0 |
|
37
|
|
|
|
|
|
|
#define STK_VAR_STR 1 |
|
38
|
|
|
|
|
|
|
|
|
39
|
|
|
|
|
|
|
/* Slot state (low 2 bits of ctl word). Upper 62 bits = generation. */ |
|
40
|
|
|
|
|
|
|
#define STK_SLOT_EMPTY 0u |
|
41
|
|
|
|
|
|
|
#define STK_SLOT_WRITING 1u |
|
42
|
|
|
|
|
|
|
#define STK_SLOT_FILLED 2u |
|
43
|
|
|
|
|
|
|
#define STK_SLOT_READING 3u |
|
44
|
|
|
|
|
|
|
#define STK_SLOT_STATE_MASK 3u |
|
45
|
|
|
|
|
|
|
#define STK_SLOT_STATE(c) ((uint32_t)((c) & STK_SLOT_STATE_MASK)) |
|
46
|
|
|
|
|
|
|
#define STK_SLOT_GEN(c) ((c) >> 2) |
|
47
|
|
|
|
|
|
|
|
|
48
|
|
|
|
|
|
|
/* ================================================================ |
|
49
|
|
|
|
|
|
|
* Header (128 bytes) |
|
50
|
|
|
|
|
|
|
* ================================================================ */ |
|
51
|
|
|
|
|
|
|
|
|
52
|
|
|
|
|
|
|
typedef struct { |
|
53
|
|
|
|
|
|
|
uint32_t magic; |
|
54
|
|
|
|
|
|
|
uint32_t version; |
|
55
|
|
|
|
|
|
|
uint32_t elem_size; |
|
56
|
|
|
|
|
|
|
uint32_t variant_id; |
|
57
|
|
|
|
|
|
|
uint64_t capacity; |
|
58
|
|
|
|
|
|
|
uint64_t total_size; |
|
59
|
|
|
|
|
|
|
uint64_t data_off; |
|
60
|
|
|
|
|
|
|
uint64_t ctl_off; /* offset to per-slot ctl array */ |
|
61
|
|
|
|
|
|
|
uint8_t _pad0[16]; |
|
62
|
|
|
|
|
|
|
|
|
63
|
|
|
|
|
|
|
uint32_t top; /* 64: next free index (0=empty, capacity=full) */ |
|
64
|
|
|
|
|
|
|
uint32_t waiters_push; /* 68 */ |
|
65
|
|
|
|
|
|
|
uint32_t waiters_pop; /* 72 */ |
|
66
|
|
|
|
|
|
|
uint32_t push_wake_seq; /* 76: bumped by every pop, futex word for pushers */ |
|
67
|
|
|
|
|
|
|
uint32_t pop_wake_seq; /* 80: bumped by every push, futex word for poppers */ |
|
68
|
|
|
|
|
|
|
uint32_t _pad1; /* 84 */ |
|
69
|
|
|
|
|
|
|
uint64_t stat_pushes; /* 88 */ |
|
70
|
|
|
|
|
|
|
uint64_t stat_pops; /* 96 */ |
|
71
|
|
|
|
|
|
|
uint64_t stat_waits; /* 104 */ |
|
72
|
|
|
|
|
|
|
uint64_t stat_timeouts; /* 112 */ |
|
73
|
|
|
|
|
|
|
uint8_t _pad2[8]; /* 120-127 */ |
|
74
|
|
|
|
|
|
|
} StkHeader; |
|
75
|
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L |
|
77
|
|
|
|
|
|
|
_Static_assert(sizeof(StkHeader) == 128, "StkHeader must be 128 bytes"); |
|
78
|
|
|
|
|
|
|
#endif |
|
79
|
|
|
|
|
|
|
|
|
80
|
|
|
|
|
|
|
typedef struct { |
|
81
|
|
|
|
|
|
|
StkHeader *hdr; |
|
82
|
|
|
|
|
|
|
uint8_t *data; |
|
83
|
|
|
|
|
|
|
uint64_t *ctl; /* per-slot state+generation word */ |
|
84
|
|
|
|
|
|
|
size_t mmap_size; |
|
85
|
|
|
|
|
|
|
uint32_t elem_size; /* cached from header at open time */ |
|
86
|
|
|
|
|
|
|
char *path; |
|
87
|
|
|
|
|
|
|
int notify_fd; |
|
88
|
|
|
|
|
|
|
int backing_fd; |
|
89
|
|
|
|
|
|
|
} StkHandle; |
|
90
|
|
|
|
|
|
|
|
|
91
|
|
|
|
|
|
|
/* ================================================================ |
|
92
|
|
|
|
|
|
|
* Utility |
|
93
|
|
|
|
|
|
|
* ================================================================ */ |
|
94
|
|
|
|
|
|
|
|
|
95
|
3
|
|
|
|
|
|
static inline void stk_make_deadline(double timeout, struct timespec *dl) { |
|
96
|
3
|
|
|
|
|
|
clock_gettime(CLOCK_MONOTONIC, dl); |
|
97
|
3
|
|
|
|
|
|
dl->tv_sec += (time_t)timeout; |
|
98
|
3
|
|
|
|
|
|
dl->tv_nsec += (long)((timeout - (double)(time_t)timeout) * 1e9); |
|
99
|
3
|
50
|
|
|
|
|
if (dl->tv_nsec >= 1000000000L) { dl->tv_sec++; dl->tv_nsec -= 1000000000L; } |
|
100
|
3
|
|
|
|
|
|
} |
|
101
|
|
|
|
|
|
|
|
|
102
|
5
|
|
|
|
|
|
static inline int stk_remaining(const struct timespec *dl, struct timespec *rem) { |
|
103
|
|
|
|
|
|
|
struct timespec now; |
|
104
|
5
|
|
|
|
|
|
clock_gettime(CLOCK_MONOTONIC, &now); |
|
105
|
5
|
|
|
|
|
|
rem->tv_sec = dl->tv_sec - now.tv_sec; |
|
106
|
5
|
|
|
|
|
|
rem->tv_nsec = dl->tv_nsec - now.tv_nsec; |
|
107
|
5
|
100
|
|
|
|
|
if (rem->tv_nsec < 0) { rem->tv_sec--; rem->tv_nsec += 1000000000L; } |
|
108
|
5
|
|
|
|
|
|
return rem->tv_sec >= 0; |
|
109
|
|
|
|
|
|
|
} |
|
110
|
|
|
|
|
|
|
|
|
111
|
18105
|
|
|
|
|
|
static inline uint8_t *stk_slot(StkHandle *h, uint32_t idx) { |
|
112
|
18105
|
|
|
|
|
|
return h->data + (uint64_t)idx * h->elem_size; |
|
113
|
|
|
|
|
|
|
} |
|
114
|
|
|
|
|
|
|
|
|
115
|
0
|
|
|
|
|
|
static inline void stk_spin_pause(void) { |
|
116
|
|
|
|
|
|
|
#if defined(__x86_64__) || defined(__i386__) |
|
117
|
0
|
|
|
|
|
|
__asm__ volatile("pause" ::: "memory"); |
|
118
|
|
|
|
|
|
|
#elif defined(__aarch64__) |
|
119
|
|
|
|
|
|
|
__asm__ volatile("yield" ::: "memory"); |
|
120
|
|
|
|
|
|
|
#endif |
|
121
|
0
|
|
|
|
|
|
} |
|
122
|
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
/* Slot state machine — same pattern as Data::Deque::Shared. */ |
|
124
|
63
|
|
|
|
|
|
static inline uint64_t stk_slot_claim_write(uint64_t *ctl_word) { |
|
125
|
0
|
|
|
|
|
|
for (;;) { |
|
126
|
63
|
|
|
|
|
|
uint64_t c = __atomic_load_n(ctl_word, __ATOMIC_ACQUIRE); |
|
127
|
63
|
50
|
|
|
|
|
if (STK_SLOT_STATE(c) == STK_SLOT_EMPTY) { |
|
128
|
63
|
|
|
|
|
|
uint64_t nc = (STK_SLOT_GEN(c) << 2) | STK_SLOT_WRITING; |
|
129
|
63
|
50
|
|
|
|
|
if (__atomic_compare_exchange_n(ctl_word, &c, nc, |
|
130
|
|
|
|
|
|
|
0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) |
|
131
|
63
|
|
|
|
|
|
return STK_SLOT_GEN(c); |
|
132
|
|
|
|
|
|
|
} |
|
133
|
0
|
|
|
|
|
|
stk_spin_pause(); |
|
134
|
|
|
|
|
|
|
} |
|
135
|
|
|
|
|
|
|
} |
|
136
|
|
|
|
|
|
|
|
|
137
|
63
|
|
|
|
|
|
static inline void stk_slot_publish(uint64_t *ctl_word, uint64_t gen) { |
|
138
|
63
|
|
|
|
|
|
__atomic_store_n(ctl_word, (gen << 2) | STK_SLOT_FILLED, __ATOMIC_RELEASE); |
|
139
|
63
|
|
|
|
|
|
} |
|
140
|
|
|
|
|
|
|
|
|
141
|
18044
|
|
|
|
|
|
static inline uint64_t stk_slot_claim_read(uint64_t *ctl_word) { |
|
142
|
0
|
|
|
|
|
|
for (;;) { |
|
143
|
18044
|
|
|
|
|
|
uint64_t c = __atomic_load_n(ctl_word, __ATOMIC_ACQUIRE); |
|
144
|
18044
|
50
|
|
|
|
|
if (STK_SLOT_STATE(c) == STK_SLOT_FILLED) { |
|
145
|
18044
|
|
|
|
|
|
uint64_t nc = (STK_SLOT_GEN(c) << 2) | STK_SLOT_READING; |
|
146
|
18044
|
50
|
|
|
|
|
if (__atomic_compare_exchange_n(ctl_word, &c, nc, |
|
147
|
|
|
|
|
|
|
0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) |
|
148
|
18044
|
|
|
|
|
|
return STK_SLOT_GEN(c); |
|
149
|
|
|
|
|
|
|
} |
|
150
|
0
|
|
|
|
|
|
stk_spin_pause(); |
|
151
|
|
|
|
|
|
|
} |
|
152
|
|
|
|
|
|
|
} |
|
153
|
|
|
|
|
|
|
|
|
154
|
18044
|
|
|
|
|
|
static inline void stk_slot_release(uint64_t *ctl_word, uint64_t gen) { |
|
155
|
18044
|
|
|
|
|
|
__atomic_store_n(ctl_word, ((gen + 1) << 2) | STK_SLOT_EMPTY, __ATOMIC_RELEASE); |
|
156
|
18044
|
|
|
|
|
|
} |
|
157
|
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
/* ================================================================ |
|
159
|
|
|
|
|
|
|
* Push (LIFO top++) |
|
160
|
|
|
|
|
|
|
* ================================================================ */ |
|
161
|
|
|
|
|
|
|
|
|
162
|
67
|
|
|
|
|
|
static inline int stk_try_push(StkHandle *h, const void *val, uint32_t vlen) { |
|
163
|
67
|
|
|
|
|
|
StkHeader *hdr = h->hdr; |
|
164
|
67
|
|
|
|
|
|
uint32_t cap = (uint32_t)hdr->capacity; |
|
165
|
0
|
|
|
|
|
|
for (;;) { |
|
166
|
67
|
|
|
|
|
|
uint32_t t = __atomic_load_n(&hdr->top, __ATOMIC_RELAXED); |
|
167
|
130
|
100
|
|
|
|
|
if (t >= cap) return 0; |
|
168
|
63
|
50
|
|
|
|
|
if (__atomic_compare_exchange_n(&hdr->top, &t, t + 1, |
|
169
|
|
|
|
|
|
|
1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { |
|
170
|
63
|
|
|
|
|
|
uint32_t sz = h->elem_size; |
|
171
|
63
|
|
|
|
|
|
uint32_t cp = vlen < sz ? vlen : sz; |
|
172
|
63
|
|
|
|
|
|
uint64_t gen = stk_slot_claim_write(&h->ctl[t]); |
|
173
|
63
|
|
|
|
|
|
memcpy(stk_slot(h, t), val, cp); |
|
174
|
63
|
50
|
|
|
|
|
if (cp < sz) memset(stk_slot(h, t) + cp, 0, sz - cp); |
|
175
|
63
|
|
|
|
|
|
stk_slot_publish(&h->ctl[t], gen); |
|
176
|
63
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_pushes, 1, __ATOMIC_RELAXED); |
|
177
|
63
|
50
|
|
|
|
|
if (__atomic_load_n(&hdr->waiters_pop, __ATOMIC_RELAXED) > 0) { |
|
178
|
0
|
|
|
|
|
|
__atomic_add_fetch(&hdr->pop_wake_seq, 1, __ATOMIC_RELEASE); |
|
179
|
0
|
|
|
|
|
|
syscall(SYS_futex, &hdr->pop_wake_seq, FUTEX_WAKE, 1, NULL, NULL, 0); |
|
180
|
|
|
|
|
|
|
} |
|
181
|
63
|
|
|
|
|
|
return 1; |
|
182
|
|
|
|
|
|
|
} |
|
183
|
|
|
|
|
|
|
} |
|
184
|
|
|
|
|
|
|
} |
|
185
|
|
|
|
|
|
|
|
|
186
|
1
|
|
|
|
|
|
static inline int stk_push(StkHandle *h, const void *val, uint32_t vlen, double timeout) { |
|
187
|
1
|
50
|
|
|
|
|
if (stk_try_push(h, val, vlen)) return 1; |
|
188
|
1
|
50
|
|
|
|
|
if (timeout == 0) return 0; |
|
189
|
|
|
|
|
|
|
|
|
190
|
1
|
|
|
|
|
|
StkHeader *hdr = h->hdr; |
|
191
|
1
|
|
|
|
|
|
uint32_t cap = (uint32_t)hdr->capacity; |
|
192
|
|
|
|
|
|
|
struct timespec dl, rem; |
|
193
|
1
|
|
|
|
|
|
int has_dl = (timeout > 0); |
|
194
|
1
|
50
|
|
|
|
|
if (has_dl) stk_make_deadline(timeout, &dl); |
|
195
|
1
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED); |
|
196
|
|
|
|
|
|
|
|
|
197
|
0
|
|
|
|
|
|
for (;;) { |
|
198
|
1
|
|
|
|
|
|
uint32_t wseq = __atomic_load_n(&hdr->push_wake_seq, __ATOMIC_ACQUIRE); |
|
199
|
1
|
|
|
|
|
|
__atomic_add_fetch(&hdr->waiters_push, 1, __ATOMIC_RELEASE); |
|
200
|
1
|
|
|
|
|
|
uint32_t t = __atomic_load_n(&hdr->top, __ATOMIC_ACQUIRE); |
|
201
|
1
|
50
|
|
|
|
|
if (t >= cap) { |
|
202
|
1
|
|
|
|
|
|
struct timespec *pts = NULL; |
|
203
|
1
|
50
|
|
|
|
|
if (has_dl) { |
|
204
|
1
|
50
|
|
|
|
|
if (!stk_remaining(&dl, &rem)) { |
|
205
|
0
|
|
|
|
|
|
__atomic_sub_fetch(&hdr->waiters_push, 1, __ATOMIC_RELAXED); |
|
206
|
0
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED); |
|
207
|
0
|
|
|
|
|
|
return 0; |
|
208
|
|
|
|
|
|
|
} |
|
209
|
1
|
|
|
|
|
|
pts = &rem; |
|
210
|
|
|
|
|
|
|
} |
|
211
|
1
|
|
|
|
|
|
syscall(SYS_futex, &hdr->push_wake_seq, FUTEX_WAIT, wseq, pts, NULL, 0); |
|
212
|
|
|
|
|
|
|
} |
|
213
|
1
|
|
|
|
|
|
__atomic_sub_fetch(&hdr->waiters_push, 1, __ATOMIC_RELAXED); |
|
214
|
1
|
50
|
|
|
|
|
if (stk_try_push(h, val, vlen)) return 1; |
|
215
|
1
|
50
|
|
|
|
|
if (has_dl && !stk_remaining(&dl, &rem)) { |
|
|
|
50
|
|
|
|
|
|
|
216
|
1
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED); |
|
217
|
1
|
|
|
|
|
|
return 0; |
|
218
|
|
|
|
|
|
|
} |
|
219
|
|
|
|
|
|
|
} |
|
220
|
|
|
|
|
|
|
} |
|
221
|
|
|
|
|
|
|
|
|
222
|
|
|
|
|
|
|
/* ================================================================ |
|
223
|
|
|
|
|
|
|
* Pop (LIFO top--) |
|
224
|
|
|
|
|
|
|
* ================================================================ */ |
|
225
|
|
|
|
|
|
|
|
|
226
|
18047
|
|
|
|
|
|
static inline int stk_try_pop(StkHandle *h, void *out) { |
|
227
|
18047
|
|
|
|
|
|
StkHeader *hdr = h->hdr; |
|
228
|
0
|
|
|
|
|
|
for (;;) { |
|
229
|
18047
|
|
|
|
|
|
uint32_t t = __atomic_load_n(&hdr->top, __ATOMIC_ACQUIRE); |
|
230
|
36086
|
100
|
|
|
|
|
if (t == 0) return 0; |
|
231
|
18039
|
50
|
|
|
|
|
if (__atomic_compare_exchange_n(&hdr->top, &t, t - 1, |
|
232
|
|
|
|
|
|
|
1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { |
|
233
|
18039
|
|
|
|
|
|
uint64_t gen = stk_slot_claim_read(&h->ctl[t - 1]); |
|
234
|
18039
|
|
|
|
|
|
memcpy(out, stk_slot(h, t - 1), h->elem_size); |
|
235
|
18039
|
|
|
|
|
|
stk_slot_release(&h->ctl[t - 1], gen); |
|
236
|
18039
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_pops, 1, __ATOMIC_RELAXED); |
|
237
|
18039
|
50
|
|
|
|
|
if (__atomic_load_n(&hdr->waiters_push, __ATOMIC_RELAXED) > 0) { |
|
238
|
0
|
|
|
|
|
|
__atomic_add_fetch(&hdr->push_wake_seq, 1, __ATOMIC_RELEASE); |
|
239
|
0
|
|
|
|
|
|
syscall(SYS_futex, &hdr->push_wake_seq, FUTEX_WAKE, 1, NULL, NULL, 0); |
|
240
|
|
|
|
|
|
|
} |
|
241
|
18039
|
|
|
|
|
|
return 1; |
|
242
|
|
|
|
|
|
|
} |
|
243
|
|
|
|
|
|
|
} |
|
244
|
|
|
|
|
|
|
} |
|
245
|
|
|
|
|
|
|
|
|
246
|
2
|
|
|
|
|
|
static inline int stk_pop(StkHandle *h, void *out, double timeout) { |
|
247
|
2
|
50
|
|
|
|
|
if (stk_try_pop(h, out)) return 1; |
|
248
|
2
|
50
|
|
|
|
|
if (timeout == 0) return 0; |
|
249
|
|
|
|
|
|
|
|
|
250
|
2
|
|
|
|
|
|
StkHeader *hdr = h->hdr; |
|
251
|
|
|
|
|
|
|
struct timespec dl, rem; |
|
252
|
2
|
|
|
|
|
|
int has_dl = (timeout > 0); |
|
253
|
2
|
50
|
|
|
|
|
if (has_dl) stk_make_deadline(timeout, &dl); |
|
254
|
2
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED); |
|
255
|
|
|
|
|
|
|
|
|
256
|
0
|
|
|
|
|
|
for (;;) { |
|
257
|
2
|
|
|
|
|
|
uint32_t wseq = __atomic_load_n(&hdr->pop_wake_seq, __ATOMIC_ACQUIRE); |
|
258
|
2
|
|
|
|
|
|
__atomic_add_fetch(&hdr->waiters_pop, 1, __ATOMIC_RELEASE); |
|
259
|
2
|
|
|
|
|
|
uint32_t t = __atomic_load_n(&hdr->top, __ATOMIC_ACQUIRE); |
|
260
|
2
|
50
|
|
|
|
|
if (t == 0) { |
|
261
|
2
|
|
|
|
|
|
struct timespec *pts = NULL; |
|
262
|
2
|
50
|
|
|
|
|
if (has_dl) { |
|
263
|
2
|
50
|
|
|
|
|
if (!stk_remaining(&dl, &rem)) { |
|
264
|
0
|
|
|
|
|
|
__atomic_sub_fetch(&hdr->waiters_pop, 1, __ATOMIC_RELAXED); |
|
265
|
0
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED); |
|
266
|
0
|
|
|
|
|
|
return 0; |
|
267
|
|
|
|
|
|
|
} |
|
268
|
2
|
|
|
|
|
|
pts = &rem; |
|
269
|
|
|
|
|
|
|
} |
|
270
|
2
|
|
|
|
|
|
syscall(SYS_futex, &hdr->pop_wake_seq, FUTEX_WAIT, wseq, pts, NULL, 0); |
|
271
|
|
|
|
|
|
|
} |
|
272
|
2
|
|
|
|
|
|
__atomic_sub_fetch(&hdr->waiters_pop, 1, __ATOMIC_RELAXED); |
|
273
|
2
|
100
|
|
|
|
|
if (stk_try_pop(h, out)) return 1; |
|
274
|
1
|
50
|
|
|
|
|
if (has_dl && !stk_remaining(&dl, &rem)) { |
|
|
|
50
|
|
|
|
|
|
|
275
|
1
|
|
|
|
|
|
__atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED); |
|
276
|
1
|
|
|
|
|
|
return 0; |
|
277
|
|
|
|
|
|
|
} |
|
278
|
|
|
|
|
|
|
} |
|
279
|
|
|
|
|
|
|
} |
|
280
|
|
|
|
|
|
|
|
|
281
|
|
|
|
|
|
|
/* ================================================================ |
|
282
|
|
|
|
|
|
|
* Peek / Status |
|
283
|
|
|
|
|
|
|
* ================================================================ */ |
|
284
|
|
|
|
|
|
|
|
|
285
|
|
|
|
|
|
|
/* Best-effort peek: seqlock-style retry against slot ctl. Returns 0 if |
|
286
|
|
|
|
|
|
|
* empty OR if the top is mutating concurrently after the retry budget. |
|
287
|
|
|
|
|
|
|
* Under no contention this is a single load. */ |
|
288
|
3
|
|
|
|
|
|
static inline int stk_peek(StkHandle *h, void *out) { |
|
289
|
3
|
50
|
|
|
|
|
for (int tries = 0; tries < 64; tries++) { |
|
290
|
3
|
|
|
|
|
|
uint32_t t = __atomic_load_n(&h->hdr->top, __ATOMIC_ACQUIRE); |
|
291
|
3
|
50
|
|
|
|
|
if (t == 0) return 0; |
|
292
|
3
|
|
|
|
|
|
uint64_t c1 = __atomic_load_n(&h->ctl[t - 1], __ATOMIC_ACQUIRE); |
|
293
|
3
|
50
|
|
|
|
|
if (STK_SLOT_STATE(c1) != STK_SLOT_FILLED) { |
|
294
|
0
|
|
|
|
|
|
stk_spin_pause(); |
|
295
|
0
|
|
|
|
|
|
continue; |
|
296
|
|
|
|
|
|
|
} |
|
297
|
3
|
|
|
|
|
|
memcpy(out, stk_slot(h, t - 1), h->elem_size); |
|
298
|
3
|
|
|
|
|
|
uint64_t c2 = __atomic_load_n(&h->ctl[t - 1], __ATOMIC_ACQUIRE); |
|
299
|
3
|
|
|
|
|
|
uint32_t t2 = __atomic_load_n(&h->hdr->top, __ATOMIC_ACQUIRE); |
|
300
|
3
|
50
|
|
|
|
|
if (c1 == c2 && t == t2) return 1; |
|
|
|
50
|
|
|
|
|
|
|
301
|
|
|
|
|
|
|
} |
|
302
|
0
|
|
|
|
|
|
return 0; |
|
303
|
|
|
|
|
|
|
} |
|
304
|
|
|
|
|
|
|
|
|
305
|
17
|
|
|
|
|
|
static inline uint32_t stk_size(StkHandle *h) { |
|
306
|
17
|
|
|
|
|
|
return __atomic_load_n(&h->hdr->top, __ATOMIC_RELAXED); |
|
307
|
|
|
|
|
|
|
} |
|
308
|
|
|
|
|
|
|
|
|
309
|
|
|
|
|
|
|
/* ================================================================ |
|
310
|
|
|
|
|
|
|
* Create / Open / Close |
|
311
|
|
|
|
|
|
|
* ================================================================ */ |
|
312
|
|
|
|
|
|
|
|
|
313
|
|
|
|
|
|
|
#define STK_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, STK_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while(0) |
|
314
|
|
|
|
|
|
|
|
|
315
|
36
|
|
|
|
|
|
static inline uint64_t stk_ctl_offset(uint32_t elem_size, uint64_t capacity) { |
|
316
|
36
|
|
|
|
|
|
uint64_t data_end = sizeof(StkHeader) + capacity * elem_size; |
|
317
|
36
|
|
|
|
|
|
return (data_end + 7u) & ~(uint64_t)7u; |
|
318
|
|
|
|
|
|
|
} |
|
319
|
|
|
|
|
|
|
|
|
320
|
19
|
|
|
|
|
|
static inline uint64_t stk_total_size(uint32_t elem_size, uint64_t capacity) { |
|
321
|
19
|
|
|
|
|
|
return stk_ctl_offset(elem_size, capacity) + capacity * sizeof(uint64_t); |
|
322
|
|
|
|
|
|
|
} |
|
323
|
|
|
|
|
|
|
|
|
324
|
15
|
|
|
|
|
|
static inline void stk_init_header(void *base, uint64_t total, |
|
325
|
|
|
|
|
|
|
uint32_t elem_size, uint32_t variant_id, |
|
326
|
|
|
|
|
|
|
uint64_t capacity) { |
|
327
|
15
|
|
|
|
|
|
StkHeader *hdr = (StkHeader *)base; |
|
328
|
15
|
|
|
|
|
|
memset(base, 0, (size_t)total); /* zeroes ctl array → all slots EMPTY, gen=0 */ |
|
329
|
15
|
|
|
|
|
|
hdr->magic = STK_MAGIC; |
|
330
|
15
|
|
|
|
|
|
hdr->version = STK_VERSION; |
|
331
|
15
|
|
|
|
|
|
hdr->elem_size = elem_size; |
|
332
|
15
|
|
|
|
|
|
hdr->variant_id = variant_id; |
|
333
|
15
|
|
|
|
|
|
hdr->capacity = capacity; |
|
334
|
15
|
|
|
|
|
|
hdr->total_size = total; |
|
335
|
15
|
|
|
|
|
|
hdr->data_off = sizeof(StkHeader); |
|
336
|
15
|
|
|
|
|
|
hdr->ctl_off = stk_ctl_offset(elem_size, capacity); |
|
337
|
15
|
|
|
|
|
|
__atomic_thread_fence(__ATOMIC_SEQ_CST); |
|
338
|
15
|
|
|
|
|
|
} |
|
339
|
|
|
|
|
|
|
|
|
340
|
17
|
|
|
|
|
|
static inline StkHandle *stk_setup(void *base, size_t msize, |
|
341
|
|
|
|
|
|
|
const char *path, int bfd) { |
|
342
|
17
|
|
|
|
|
|
StkHeader *hdr = (StkHeader *)base; |
|
343
|
17
|
|
|
|
|
|
StkHandle *h = (StkHandle *)calloc(1, sizeof(StkHandle)); |
|
344
|
17
|
50
|
|
|
|
|
if (!h) { munmap(base, msize); return NULL; } |
|
345
|
17
|
|
|
|
|
|
h->hdr = hdr; |
|
346
|
17
|
|
|
|
|
|
h->data = (uint8_t *)base + hdr->data_off; |
|
347
|
17
|
|
|
|
|
|
h->ctl = (uint64_t *)((uint8_t *)base + hdr->ctl_off); |
|
348
|
17
|
|
|
|
|
|
h->mmap_size = msize; |
|
349
|
17
|
|
|
|
|
|
h->elem_size = hdr->elem_size; /* cached — safe from shared-mem tampering */ |
|
350
|
17
|
100
|
|
|
|
|
h->path = path ? strdup(path) : NULL; |
|
351
|
17
|
|
|
|
|
|
h->notify_fd = -1; |
|
352
|
17
|
|
|
|
|
|
h->backing_fd = bfd; |
|
353
|
17
|
|
|
|
|
|
return h; |
|
354
|
|
|
|
|
|
|
} |
|
355
|
|
|
|
|
|
|
|
|
356
|
|
|
|
|
|
|
/* Validate a mapped header (shared by stk_create reopen and stk_open_fd). */ |
|
357
|
3
|
|
|
|
|
|
static inline int stk_validate_header(const StkHeader *hdr, uint64_t file_size, |
|
358
|
|
|
|
|
|
|
uint32_t expected_variant) { |
|
359
|
3
|
100
|
|
|
|
|
if (hdr->magic != STK_MAGIC) return 0; |
|
360
|
2
|
50
|
|
|
|
|
if (hdr->version != STK_VERSION) return 0; |
|
361
|
2
|
50
|
|
|
|
|
if (hdr->variant_id != expected_variant) return 0; |
|
362
|
2
|
50
|
|
|
|
|
if (hdr->elem_size == 0 || hdr->capacity == 0) return 0; |
|
|
|
50
|
|
|
|
|
|
|
363
|
2
|
50
|
|
|
|
|
if (hdr->capacity > 0x7FFFFFFFu) return 0; |
|
364
|
2
|
50
|
|
|
|
|
if (hdr->total_size != file_size) return 0; |
|
365
|
2
|
50
|
|
|
|
|
if (hdr->data_off != sizeof(StkHeader)) return 0; |
|
366
|
2
|
50
|
|
|
|
|
if (hdr->ctl_off != stk_ctl_offset(hdr->elem_size, hdr->capacity)) return 0; |
|
367
|
2
|
50
|
|
|
|
|
if (hdr->total_size != stk_total_size(hdr->elem_size, hdr->capacity)) return 0; |
|
368
|
2
|
|
|
|
|
|
return 1; |
|
369
|
|
|
|
|
|
|
} |
|
370
|
|
|
|
|
|
|
|
|
371
|
16
|
|
|
|
|
|
static StkHandle *stk_create(const char *path, uint64_t capacity, |
|
372
|
|
|
|
|
|
|
uint32_t elem_size, uint32_t variant_id, |
|
373
|
|
|
|
|
|
|
char *errbuf) { |
|
374
|
16
|
50
|
|
|
|
|
if (errbuf) errbuf[0] = '\0'; |
|
375
|
16
|
50
|
|
|
|
|
if (capacity == 0) { STK_ERR("capacity must be > 0"); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
376
|
16
|
50
|
|
|
|
|
if (elem_size == 0) { STK_ERR("elem_size must be > 0"); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
377
|
16
|
50
|
|
|
|
|
if (capacity > (UINT64_MAX - sizeof(StkHeader) - 16) / (elem_size + sizeof(uint64_t))) { |
|
378
|
0
|
0
|
|
|
|
|
STK_ERR("capacity * elem_size overflow"); return NULL; |
|
379
|
|
|
|
|
|
|
} |
|
380
|
|
|
|
|
|
|
|
|
381
|
16
|
|
|
|
|
|
uint64_t total = stk_total_size(elem_size, capacity); |
|
382
|
16
|
|
|
|
|
|
int anonymous = (path == NULL); |
|
383
|
16
|
|
|
|
|
|
int fd = -1; |
|
384
|
|
|
|
|
|
|
size_t map_size; |
|
385
|
|
|
|
|
|
|
void *base; |
|
386
|
|
|
|
|
|
|
|
|
387
|
16
|
100
|
|
|
|
|
if (anonymous) { |
|
388
|
12
|
|
|
|
|
|
map_size = (size_t)total; |
|
389
|
12
|
|
|
|
|
|
base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, |
|
390
|
|
|
|
|
|
|
MAP_SHARED|MAP_ANONYMOUS, -1, 0); |
|
391
|
12
|
50
|
|
|
|
|
if (base == MAP_FAILED) { STK_ERR("mmap: %s", strerror(errno)); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
392
|
|
|
|
|
|
|
} else { |
|
393
|
4
|
|
|
|
|
|
fd = open(path, O_RDWR|O_CREAT, 0666); |
|
394
|
6
|
50
|
|
|
|
|
if (fd < 0) { STK_ERR("open(%s): %s", path, strerror(errno)); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
395
|
4
|
50
|
|
|
|
|
if (flock(fd, LOCK_EX) < 0) { STK_ERR("flock: %s", strerror(errno)); close(fd); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
396
|
|
|
|
|
|
|
|
|
397
|
|
|
|
|
|
|
struct stat st; |
|
398
|
4
|
50
|
|
|
|
|
if (fstat(fd, &st) < 0) { |
|
399
|
0
|
0
|
|
|
|
|
STK_ERR("fstat: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; |
|
400
|
|
|
|
|
|
|
} |
|
401
|
4
|
|
|
|
|
|
int is_new = (st.st_size == 0); |
|
402
|
4
|
100
|
|
|
|
|
if (!is_new && (uint64_t)st.st_size < sizeof(StkHeader)) { |
|
|
|
50
|
|
|
|
|
|
|
403
|
0
|
0
|
|
|
|
|
STK_ERR("%s: file too small (%lld)", path, (long long)st.st_size); |
|
404
|
0
|
|
|
|
|
|
flock(fd, LOCK_UN); close(fd); return NULL; |
|
405
|
|
|
|
|
|
|
} |
|
406
|
|
|
|
|
|
|
|
|
407
|
4
|
100
|
|
|
|
|
if (is_new) { |
|
408
|
2
|
50
|
|
|
|
|
if (ftruncate(fd, (off_t)total) < 0) { |
|
409
|
0
|
0
|
|
|
|
|
STK_ERR("ftruncate: %s", strerror(errno)); |
|
410
|
0
|
|
|
|
|
|
flock(fd, LOCK_UN); close(fd); return NULL; |
|
411
|
|
|
|
|
|
|
} |
|
412
|
|
|
|
|
|
|
} |
|
413
|
4
|
100
|
|
|
|
|
map_size = is_new ? (size_t)total : (size_t)st.st_size; |
|
414
|
4
|
|
|
|
|
|
base = mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); |
|
415
|
4
|
50
|
|
|
|
|
if (base == MAP_FAILED) { STK_ERR("mmap: %s", strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
416
|
|
|
|
|
|
|
|
|
417
|
4
|
100
|
|
|
|
|
if (!is_new) { |
|
418
|
2
|
100
|
|
|
|
|
if (!stk_validate_header((StkHeader *)base, (uint64_t)st.st_size, variant_id)) { |
|
419
|
1
|
50
|
|
|
|
|
STK_ERR("invalid or incompatible stack file"); |
|
420
|
1
|
|
|
|
|
|
munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL; |
|
421
|
|
|
|
|
|
|
} |
|
422
|
1
|
|
|
|
|
|
flock(fd, LOCK_UN); close(fd); |
|
423
|
1
|
|
|
|
|
|
return stk_setup(base, map_size, path, -1); |
|
424
|
|
|
|
|
|
|
} |
|
425
|
|
|
|
|
|
|
} |
|
426
|
|
|
|
|
|
|
|
|
427
|
14
|
|
|
|
|
|
stk_init_header(base, total, elem_size, variant_id, capacity); |
|
428
|
14
|
100
|
|
|
|
|
if (fd >= 0) { flock(fd, LOCK_UN); close(fd); } |
|
429
|
14
|
|
|
|
|
|
return stk_setup(base, map_size, path, -1); |
|
430
|
|
|
|
|
|
|
} |
|
431
|
|
|
|
|
|
|
|
|
432
|
1
|
|
|
|
|
|
static StkHandle *stk_create_memfd(const char *name, uint64_t capacity, |
|
433
|
|
|
|
|
|
|
uint32_t elem_size, uint32_t variant_id, |
|
434
|
|
|
|
|
|
|
char *errbuf) { |
|
435
|
1
|
50
|
|
|
|
|
if (errbuf) errbuf[0] = '\0'; |
|
436
|
1
|
50
|
|
|
|
|
if (capacity == 0) { STK_ERR("capacity must be > 0"); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
437
|
1
|
50
|
|
|
|
|
if (elem_size == 0) { STK_ERR("elem_size must be > 0"); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
438
|
1
|
50
|
|
|
|
|
if (capacity > (UINT64_MAX - sizeof(StkHeader) - 16) / (elem_size + sizeof(uint64_t))) { |
|
439
|
0
|
0
|
|
|
|
|
STK_ERR("capacity * elem_size overflow"); return NULL; |
|
440
|
|
|
|
|
|
|
} |
|
441
|
|
|
|
|
|
|
|
|
442
|
1
|
|
|
|
|
|
uint64_t total = stk_total_size(elem_size, capacity); |
|
443
|
1
|
50
|
|
|
|
|
int fd = memfd_create(name ? name : "stack", MFD_CLOEXEC | MFD_ALLOW_SEALING); |
|
444
|
1
|
50
|
|
|
|
|
if (fd < 0) { STK_ERR("memfd_create: %s", strerror(errno)); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
445
|
1
|
50
|
|
|
|
|
if (ftruncate(fd, (off_t)total) < 0) { STK_ERR("ftruncate: %s", strerror(errno)); close(fd); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
446
|
1
|
|
|
|
|
|
(void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW); |
|
447
|
1
|
|
|
|
|
|
void *base = mmap(NULL, (size_t)total, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); |
|
448
|
1
|
50
|
|
|
|
|
if (base == MAP_FAILED) { STK_ERR("mmap: %s", strerror(errno)); close(fd); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
449
|
1
|
|
|
|
|
|
stk_init_header(base, total, elem_size, variant_id, capacity); |
|
450
|
1
|
|
|
|
|
|
return stk_setup(base, (size_t)total, NULL, fd); |
|
451
|
|
|
|
|
|
|
} |
|
452
|
|
|
|
|
|
|
|
|
453
|
1
|
|
|
|
|
|
static StkHandle *stk_open_fd(int fd, uint32_t variant_id, char *errbuf) { |
|
454
|
1
|
50
|
|
|
|
|
if (errbuf) errbuf[0] = '\0'; |
|
455
|
|
|
|
|
|
|
struct stat st; |
|
456
|
1
|
50
|
|
|
|
|
if (fstat(fd, &st) < 0) { STK_ERR("fstat: %s", strerror(errno)); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
457
|
1
|
50
|
|
|
|
|
if ((uint64_t)st.st_size < sizeof(StkHeader)) { STK_ERR("too small"); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
458
|
1
|
|
|
|
|
|
size_t ms = (size_t)st.st_size; |
|
459
|
1
|
|
|
|
|
|
void *base = mmap(NULL, ms, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); |
|
460
|
1
|
50
|
|
|
|
|
if (base == MAP_FAILED) { STK_ERR("mmap: %s", strerror(errno)); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
461
|
1
|
50
|
|
|
|
|
if (!stk_validate_header((StkHeader *)base, (uint64_t)st.st_size, variant_id)) { |
|
462
|
0
|
0
|
|
|
|
|
STK_ERR("invalid stack"); munmap(base, ms); return NULL; |
|
463
|
|
|
|
|
|
|
} |
|
464
|
1
|
|
|
|
|
|
int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0); |
|
465
|
1
|
50
|
|
|
|
|
if (myfd < 0) { STK_ERR("fcntl: %s", strerror(errno)); munmap(base, ms); return NULL; } |
|
|
|
0
|
|
|
|
|
|
|
466
|
1
|
|
|
|
|
|
return stk_setup(base, ms, NULL, myfd); |
|
467
|
|
|
|
|
|
|
} |
|
468
|
|
|
|
|
|
|
|
|
469
|
17
|
|
|
|
|
|
static void stk_destroy(StkHandle *h) { |
|
470
|
17
|
50
|
|
|
|
|
if (!h) return; |
|
471
|
17
|
100
|
|
|
|
|
if (h->notify_fd >= 0) close(h->notify_fd); |
|
472
|
17
|
100
|
|
|
|
|
if (h->backing_fd >= 0) close(h->backing_fd); |
|
473
|
17
|
50
|
|
|
|
|
if (h->hdr) munmap(h->hdr, h->mmap_size); |
|
474
|
17
|
|
|
|
|
|
free(h->path); |
|
475
|
17
|
|
|
|
|
|
free(h); |
|
476
|
|
|
|
|
|
|
} |
|
477
|
|
|
|
|
|
|
|
|
478
|
|
|
|
|
|
|
/* NOT concurrency-safe — use drain() for concurrent scenarios */ |
|
479
|
3
|
|
|
|
|
|
static void stk_clear(StkHandle *h) { |
|
480
|
3
|
|
|
|
|
|
__atomic_store_n(&h->hdr->top, 0, __ATOMIC_RELEASE); |
|
481
|
3
|
|
|
|
|
|
memset(h->ctl, 0, (size_t)h->hdr->capacity * sizeof(uint64_t)); |
|
482
|
|
|
|
|
|
|
/* clear() frees the entire stack at once — wake all waiters. */ |
|
483
|
3
|
50
|
|
|
|
|
if (__atomic_load_n(&h->hdr->waiters_push, __ATOMIC_RELAXED) > 0) { |
|
484
|
0
|
|
|
|
|
|
__atomic_add_fetch(&h->hdr->push_wake_seq, 1, __ATOMIC_RELEASE); |
|
485
|
0
|
|
|
|
|
|
syscall(SYS_futex, &h->hdr->push_wake_seq, FUTEX_WAKE, INT_MAX, NULL, NULL, 0); |
|
486
|
|
|
|
|
|
|
} |
|
487
|
3
|
50
|
|
|
|
|
if (__atomic_load_n(&h->hdr->waiters_pop, __ATOMIC_RELAXED) > 0) { |
|
488
|
0
|
|
|
|
|
|
__atomic_add_fetch(&h->hdr->pop_wake_seq, 1, __ATOMIC_RELEASE); |
|
489
|
0
|
|
|
|
|
|
syscall(SYS_futex, &h->hdr->pop_wake_seq, FUTEX_WAKE, INT_MAX, NULL, NULL, 0); |
|
490
|
|
|
|
|
|
|
} |
|
491
|
3
|
|
|
|
|
|
} |
|
492
|
|
|
|
|
|
|
|
|
493
|
|
|
|
|
|
|
/* Concurrency-safe drain: atomically swap top to 0, then release each |
|
494
|
|
|
|
|
|
|
* drained slot through the state machine. Returns count drained. */ |
|
495
|
2
|
|
|
|
|
|
static inline uint32_t stk_drain(StkHandle *h) { |
|
496
|
2
|
|
|
|
|
|
StkHeader *hdr = h->hdr; |
|
497
|
2
|
|
|
|
|
|
uint32_t t = __atomic_exchange_n(&hdr->top, 0, __ATOMIC_ACQ_REL); |
|
498
|
2
|
100
|
|
|
|
|
if (t == 0) return 0; |
|
499
|
6
|
100
|
|
|
|
|
for (uint32_t i = 0; i < t; i++) { |
|
500
|
5
|
|
|
|
|
|
uint64_t gen = stk_slot_claim_read(&h->ctl[i]); |
|
501
|
5
|
|
|
|
|
|
stk_slot_release(&h->ctl[i], gen); |
|
502
|
|
|
|
|
|
|
} |
|
503
|
|
|
|
|
|
|
/* drain freed `t` slots at once — wake up to that many. */ |
|
504
|
1
|
50
|
|
|
|
|
if (__atomic_load_n(&hdr->waiters_push, __ATOMIC_RELAXED) > 0) { |
|
505
|
0
|
|
|
|
|
|
__atomic_add_fetch(&hdr->push_wake_seq, 1, __ATOMIC_RELEASE); |
|
506
|
0
|
0
|
|
|
|
|
syscall(SYS_futex, &hdr->push_wake_seq, FUTEX_WAKE, |
|
507
|
|
|
|
|
|
|
t < INT_MAX ? (int)t : INT_MAX, NULL, NULL, 0); |
|
508
|
|
|
|
|
|
|
} |
|
509
|
1
|
|
|
|
|
|
return t; |
|
510
|
|
|
|
|
|
|
} |
|
511
|
|
|
|
|
|
|
|
|
512
|
|
|
|
|
|
|
/* eventfd */ |
|
513
|
2
|
|
|
|
|
|
static int stk_create_eventfd(StkHandle *h) { |
|
514
|
2
|
50
|
|
|
|
|
if (h->notify_fd >= 0) return h->notify_fd; |
|
515
|
2
|
|
|
|
|
|
int efd = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC); |
|
516
|
2
|
50
|
|
|
|
|
if (efd < 0) return -1; |
|
517
|
2
|
|
|
|
|
|
h->notify_fd = efd; |
|
518
|
2
|
|
|
|
|
|
return efd; |
|
519
|
|
|
|
|
|
|
} |
|
520
|
3
|
|
|
|
|
|
static int stk_notify(StkHandle *h) { |
|
521
|
3
|
50
|
|
|
|
|
if (h->notify_fd < 0) return 0; |
|
522
|
3
|
|
|
|
|
|
uint64_t v = 1; |
|
523
|
3
|
|
|
|
|
|
return write(h->notify_fd, &v, sizeof(v)) == sizeof(v); |
|
524
|
|
|
|
|
|
|
} |
|
525
|
2
|
|
|
|
|
|
static int64_t stk_eventfd_consume(StkHandle *h) { |
|
526
|
2
|
50
|
|
|
|
|
if (h->notify_fd < 0) return -1; |
|
527
|
2
|
|
|
|
|
|
uint64_t v = 0; |
|
528
|
2
|
50
|
|
|
|
|
if (read(h->notify_fd, &v, sizeof(v)) != sizeof(v)) return -1; |
|
529
|
2
|
|
|
|
|
|
return (int64_t)v; |
|
530
|
|
|
|
|
|
|
} |
|
531
|
1
|
|
|
|
|
|
static int stk_msync(StkHandle *h) { return msync(h->hdr, h->mmap_size, MS_SYNC); } |
|
532
|
|
|
|
|
|
|
|
|
533
|
|
|
|
|
|
|
#endif /* STACK_H */ |