include/fetch/backend_iouring.c

filter: Statementstmt 50.0% 1 / 2 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 50.0% 1 / 2 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« xs/future.xs ↑ Summary include/fetch/backend_kqueue.c »
1 /* Hyperman::Event::IOUring - io_uring readiness backend (Linux), built on
2 * liburing. Enabled by Makefile.PL when liburing is found (pkg-config or a
3 * -luring compile probe), which defines HM_HAVE_LIBURING and links -luring;
4 * otherwise this compiles to graceful "unavailable" stubs.
5 *
6 * This drives the ring in *poll mode*: IORING_OP_POLL_ADD readiness
7 * completions mapped onto the hm_backend vtable (persistent watchers are
8 * re-armed on delivery), timers as IORING_OP_TIMEOUT, and shutdown signals
9 * via a signalfd polled on the ring. Completion-based read/write submission
10 * (the io_uring headline) needs the connection state machine reshaped and
11 * comes later.
12 *
13 * Opt-in for now: HYPERMAN_BACKEND=io_uring or Hyperman::Loop->new('io_uring');
14 * auto-selection stays kqueue > epoll > poll until this is benchmarked.
15 */
16  
17 #include "ft_backend.h"
18  
19 #if defined(__linux__) && defined(HM_HAVE_LIBURING)
20 #define HM_IOURING_ACTIVE 1
21 #endif
22  
23 #ifdef HM_IOURING_ACTIVE
24  
25 #include <liburing.h>
26 #include <sys/signalfd.h>
27 #include <signal.h>
28 #include <string.h>
29 #include <stdlib.h>
30 #include <unistd.h>
31 #include <errno.h>
32 #include <poll.h>
33  
34 /* user_data tagging (low 3 bits). Timer completions carry a pointer to
35 * their ur_timer, which is 8-aligned so its tag is 0. liburing's own
36 * internal wait-timeout uses LIBURING_UDATA_TIMEOUT and is skipped. */
37 #define UR_TAG_MASK 7ULL
38 #define UR_TAG_TIMER 0ULL /* aligned ur_timer* */
39 #define UR_TAG_READ 1ULL
40 #define UR_TAG_WRITE 2ULL
41 #define UR_TAG_IGNORE 5ULL /* poll-remove completions: drop */
42  
43 /* io_uring_sqe_set_data64() was only added in liburing 2.2; assign the
44 * user_data field directly (exactly what that helper does) so we build and
45 * load against every liburing version. */
46 #define UR_SET_DATA(sqe, val) ((sqe)->user_data = (__u64)(val))
47  
48 typedef struct ur_timer {
49 struct __kernel_timespec ts;
50 void *udata;
51 int interval; /* re-arm on fire? */
52 int cancelled; /* freed when its completion lands, not before */
53 struct ur_timer *prev, *next;
54 } ur_timer;
55  
56 typedef struct {
57 struct io_uring ring;
58 unsigned char mask[HM_MAXFD]; /* wanted HM_EV bits */
59 unsigned char once[HM_MAXFD]; /* oneshot HM_EV bits */
60 unsigned char armed[HM_MAXFD]; /* HM_EV bits with in-flight sqe */
61 ur_timer *timers; /* doubly-linked, for cleanup */
62 int sfd; /* signalfd, -1 until first sig */
63 sigset_t sigs;
64 } hm_ur_state;
65  
66 /* Get an sqe, flushing the ring if the submission queue is momentarily full. */
67 static struct io_uring_sqe *ur_sqe(hm_ur_state *st) {
68 struct io_uring_sqe *sqe = io_uring_get_sqe(&st->ring);
69 if (!sqe) {
70 io_uring_submit(&st->ring);
71 sqe = io_uring_get_sqe(&st->ring);
72 }
73 return sqe;
74 }
75  
76 static void ur_arm_poll(hm_ur_state *st, int fd, int evbit) {
77 struct io_uring_sqe *sqe = ur_sqe(st);
78 unsigned mask = (evbit == HM_EV_READ) ? (POLLIN | POLLHUP | POLLERR)
79 : (POLLOUT | POLLHUP | POLLERR);
80 if (!sqe) return;
81 io_uring_prep_poll_add(sqe, fd, mask);
82 UR_SET_DATA(sqe,
83 ((__u64)(unsigned)fd << 3)
84 | (evbit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE));
85 st->armed[fd] |= (unsigned char)evbit;
86 }
87  
88 static int ur_add_io(hm_backend *be, int fd, int mask, int oneshot) {
89 hm_ur_state *st = (hm_ur_state *)be->state;
90 int bit;
91 if (fd < 0 || fd >= HM_MAXFD) return -1;
92 st->mask[fd] |= (unsigned char)mask;
93 if (oneshot) st->once[fd] |= (unsigned char)mask;
94 else st->once[fd] &= (unsigned char)~mask;
95 for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1)
96 if ((mask & bit) && !(st->armed[fd] & bit))
97 ur_arm_poll(st, fd, bit);
98 return 0;
99 }
100  
101 static int ur_modify_io(hm_backend *be, int fd, int mask, int oneshot) {
102 return ur_add_io(be, fd, mask, oneshot);
103 }
104  
105 static int ur_remove_io(hm_backend *be, int fd, int mask) {
106 hm_ur_state *st = (hm_ur_state *)be->state;
107 int bit;
108 if (fd < 0 || fd >= HM_MAXFD) return -1;
109 st->mask[fd] &= (unsigned char)~mask;
110 st->once[fd] &= (unsigned char)~mask;
111 /* cancel any in-flight poll for the dropped interest, so a closed+reused
112 * fd can't be woken by a stale completion */
113 for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) {
114 if ((mask & bit) && (st->armed[fd] & bit)) {
115 struct io_uring_sqe *sqe = ur_sqe(st);
116 if (sqe) {
117 io_uring_prep_poll_remove(sqe,
118 ((__u64)(unsigned)fd << 3)
119 | (bit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE));
120 UR_SET_DATA(sqe, UR_TAG_IGNORE);
121 }
122 st->armed[fd] &= (unsigned char)~bit;
123 }
124 }
125 return 0;
126 }
127  
128 static void ur_arm_timer(hm_ur_state *st, ur_timer *t) {
129 struct io_uring_sqe *sqe = ur_sqe(st);
130 if (!sqe) return;
131 io_uring_prep_timeout(sqe, &t->ts, 0, 0);
132 UR_SET_DATA(sqe, (__u64)(uintptr_t)t); /* tag 0 (aligned) */
133 }
134  
135 static int ur_add_timer(hm_backend *be, double secs, int oneshot, void *udata) {
136 hm_ur_state *st = (hm_ur_state *)be->state;
137 ur_timer *t = (ur_timer *)calloc(1, sizeof(ur_timer));
138 if (!t) return -1;
139 t->ts.tv_sec = (int64_t)secs;
140 t->ts.tv_nsec = (int64_t)((secs - (double)t->ts.tv_sec) * 1e9);
141 if (t->ts.tv_sec == 0 && t->ts.tv_nsec == 0) t->ts.tv_nsec = 1;
142 t->udata = udata;
143 t->interval = !oneshot;
144 t->next = st->timers;
145 if (st->timers) st->timers->prev = t;
146 st->timers = t;
147 ur_arm_timer(st, t);
148 return 0;
149 }
150  
151 static void ur_timer_unlink(hm_ur_state *st, ur_timer *t) {
152 if (t->prev) t->prev->next = t->next;
153 else st->timers = t->next;
154 if (t->next) t->next->prev = t->prev;
155 }
156  
157 /* Cancel a timer. The timeout is already in flight in the kernel and its
158 * user_data IS this pointer, so freeing here would leave the kernel holding a
159 * dangling reference: the completion still arrives, ur_wait dereferences it
160 * and frees it a second time. Worse, the block is the right size to be handed
161 * straight back to the next ur_add_timer, so the stale completion lands on a
162 * live timer and takes that down instead - which is why this showed up as an
163 * intermittent "double free or corruption" rather than a reliable one.
164 *
165 * So: ask the kernel to remove the timeout and mark the watcher dead, but
166 * leave it linked and allocated. Its completion (-ECANCELED, or a genuine
167 * fire that raced the removal) frees it in ur_wait; anything still linked at
168 * shutdown is freed by ur_destroy. */
169 static int ur_del_timer(hm_backend *be, void *udata) {
170 hm_ur_state *st = (hm_ur_state *)be->state;
171 ur_timer *t;
172 for (t = st->timers; t; t = t->next) {
173 if (t->udata == udata && !t->cancelled) {
174 struct io_uring_sqe *sqe;
175 t->cancelled = 1;
176 t->interval = 0; /* never re-arm a cancelled timer */
177 if ((sqe = ur_sqe(st)) != NULL) {
178 io_uring_prep_timeout_remove(sqe, (__u64)(uintptr_t)t, 0);
179 UR_SET_DATA(sqe, UR_TAG_IGNORE);
180 }
181 return 0;
182 }
183 }
184 return -1;
185 }
186  
187 static int ur_add_signal(hm_backend *be, int signo) {
188 hm_ur_state *st = (hm_ur_state *)be->state;
189 sigaddset(&st->sigs, signo);
190 if (sigprocmask(SIG_BLOCK, &st->sigs, NULL) < 0) return -1;
191 {
192 int sfd = signalfd(st->sfd, &st->sigs, SFD_NONBLOCK | SFD_CLOEXEC);
193 if (sfd < 0) return -1;
194 if (st->sfd < 0) {
195 st->sfd = sfd;
196 st->mask[sfd] |= HM_EV_READ; /* persistent poll on the ring */
197 ur_arm_poll(st, sfd, HM_EV_READ);
198 }
199 }
200 return 0;
201 }
202  
203 static int ur_wait(hm_backend *be, hm_event *out, int max, double timeout) {
204 hm_ur_state *st = (hm_ur_state *)be->state;
205 struct io_uring_cqe *cqes[HM_MAXEV];
206 struct io_uring_cqe *cqe;
207 unsigned want, n, i;
208 int nev = 0, ret;
209  
210 io_uring_submit(&st->ring); /* flush re-arms, cancels, timers */
211  
212 if (timeout < 0) {
213 ret = io_uring_wait_cqe(&st->ring, &cqe);
214 } else if (timeout == 0) {
215 ret = io_uring_peek_cqe(&st->ring, &cqe);
216 if (ret == -EAGAIN) return 0;
217 } else {
218 struct __kernel_timespec ts;
219 ts.tv_sec = (int64_t)timeout;
220 ts.tv_nsec = (int64_t)((timeout - (double)ts.tv_sec) * 1e9);
221 ret = io_uring_wait_cqe_timeout(&st->ring, &cqe, &ts);
222 if (ret == -ETIME) return 0;
223 }
224 if (ret < 0) return (ret == -EINTR) ? 0 : -1;
225  
226 /* headroom so a single signalfd completion (multiple queued signals)
227 * can't overflow the caller's event buffer */
228 want = (max > 8) ? (unsigned)(max - 8) : 1u;
229 if (want > HM_MAXEV) want = HM_MAXEV;
230 n = io_uring_peek_batch_cqe(&st->ring, cqes, want);
231  
232 for (i = 0; i < n; i++) {
233 __u64 data = cqes[i]->user_data;
234 int res = cqes[i]->res;
235 __u64 tag = data & UR_TAG_MASK;
236  
237 if (data == LIBURING_UDATA_TIMEOUT) continue; /* liburing internal */
238 if (tag == UR_TAG_IGNORE) continue; /* poll-remove ack */
239  
240 if (tag == UR_TAG_READ || tag == UR_TAG_WRITE) {
241 int fd = (int)(data >> 3);
242 int bit = (tag == UR_TAG_READ) ? HM_EV_READ : HM_EV_WRITE;
243 st->armed[fd] &= (unsigned char)~bit;
244 if (res == -ECANCELED || !(st->mask[fd] & bit)) continue;
245  
246 if (st->sfd >= 0 && fd == st->sfd) {
247 struct signalfd_siginfo si;
248 while (nev < max
249 && read(st->sfd, &si, sizeof(si)) == (ssize_t)sizeof(si)) {
250 out[nev].kind = HM_EV_SIGNAL;
251 out[nev].fd = (int)si.ssi_signo;
252 out[nev].udata = NULL;
253 nev++;
254 }
255 ur_arm_poll(st, st->sfd, HM_EV_READ); /* persistent */
256 continue;
257 }
258  
259 out[nev].kind = bit;
260 out[nev].fd = fd;
261 out[nev].udata = NULL;
262 nev++;
263 if (st->once[fd] & bit) {
264 st->mask[fd] &= (unsigned char)~bit;
265 st->once[fd] &= (unsigned char)~bit;
266 } else {
267 ur_arm_poll(st, fd, bit); /* persistent: re-arm */
268 }
269 continue;
270 }
271  
272 { /* timer: user_data is the ur_timer pointer (tag 0) */
273 ur_timer *t = (ur_timer *)(uintptr_t)data;
274 if (!t) continue;
275 /* A cancelled timer's completion still arrives; it owns the
276 * struct, and it must not be reported to the loop. -ECANCELED
277 * covers the removal succeeding, t->cancelled the case where the
278 * timeout fired before the removal reached the kernel. */
279 if (t->cancelled || res == -ECANCELED) {
280 ur_timer_unlink(st, t);
281 free(t);
282 continue;
283 }
284 out[nev].kind = HM_EV_TIMER;
285 out[nev].fd = -1;
286 out[nev].udata = t->udata;
287 nev++;
288 if (t->interval) {
289 ur_arm_timer(st, t);
290 } else {
291 ur_timer_unlink(st, t);
292 free(t);
293 }
294 }
295 }
296 io_uring_cq_advance(&st->ring, n);
297 return nev;
298 }
299  
300 static void ur_destroy(hm_backend *be) {
301 hm_ur_state *st = (hm_ur_state *)be->state;
302 if (st) {
303 while (st->timers) {
304 ur_timer *t = st->timers;
305 st->timers = t->next;
306 free(t);
307 }
308 /* be->foreign: inherited across a fork. The signalfd and the ring's
309 * own descriptor are the parent's; io_uring_queue_exit would close
310 * the ring fd and unmap rings this process never set up. */
311 if (!be->foreign) {
312 if (st->sfd >= 0) close(st->sfd);
313 io_uring_queue_exit(&st->ring);
314 }
315 free(st);
316 }
317 free(be);
318 }
319  
320 hm_backend *hm_backend_iouring_new(void) {
321 hm_backend *be = (hm_backend *)calloc(1, sizeof(hm_backend));
322 hm_ur_state *st = (hm_ur_state *)calloc(1, sizeof(hm_ur_state));
323 if (!be || !st) { free(be); free(st); return NULL; }
324 st->sfd = -1;
325 sigemptyset(&st->sigs);
326 if (io_uring_queue_init(1024, &st->ring, 0) < 0) {
327 free(be); free(st); return NULL;
328 }
329 be->name = "io_uring";
330 be->state = st;
331 be->add_io = ur_add_io;
332 be->modify_io = ur_modify_io;
333 be->remove_io = ur_remove_io;
334 be->add_timer = ur_add_timer;
335 be->del_timer = ur_del_timer;
336 be->add_signal = ur_add_signal;
337 be->wait_ev = ur_wait;
338 be->destroy = ur_destroy;
339 return be;
340 }
341  
342 int hm_backend_iouring_available(void) {
343 /* liburing is linked; confirm the kernel actually supports io_uring
344 * (not the case under, e.g., default Docker seccomp) */
345 static int avail = -1;
346 if (avail < 0) {
347 struct io_uring ring;
348 if (io_uring_queue_init(2, &ring, 0) == 0) {
349 io_uring_queue_exit(&ring);
350 avail = 1;
351 } else {
352 avail = 0;
353 }
354 }
355 return avail;
356 }
357  
358 #else /* !HM_IOURING_ACTIVE */
359  
360 44 hm_backend *hm_backend_iouring_new(void) { return 0; }
361 0 int hm_backend_iouring_available(void) { return 0; }
362  
363 #endif
« xs/future.xs ↑ Summary include/fetch/backend_kqueue.c »