include/hyperman/backend_iouring.c

filter: Statementstmt 100% 2 / 2 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub n/a Podpod n/a totaltotal 100% 2 / 2 CC 0 SCAR 0
CC
0
Cov
0%
CRAP 0
« blib/lib/Hyperman/Loop.pm ↑ Summary include/hyperman/backend_kqueue.c »
1 /* Hyperman::Event::IOUring - io_uring readiness backend (Linux), built on
2 * liburing. Enabled by Makefile.PL when liburing is found (pkg-config or a
3 * -luring compile probe), which defines HM_HAVE_LIBURING and links -luring;
4 * otherwise this compiles to graceful "unavailable" stubs.
5 *
6 * Readiness comes from the ring in *poll mode*: IORING_OP_POLL_ADD
7 * completions mapped onto the hm_backend vtable (persistent watchers are
8 * re-armed on delivery), timers as IORING_OP_TIMEOUT, and shutdown signals
9 * via a signalfd polled on the ring.
10 *
11 * COMPLETION-MODE READS (HYPERMAN_COMPLETION=1) go further: the core asks for
12 * the read up front with ur_submit_recv and is handed bytes that are already
13 * in its buffer, so the read stops being a syscall per request and becomes
14 * part of the one batched submission per turn of the loop. Measured, 2000
15 * keep-alive requests over 50 connections:
16 *
17 * poll mode read 10050 writev 10000 enter 222 2.02 syscalls/req
18 * completion mode read 0 writev 10000 enter 203 1.02 syscalls/req
19 *
20 * The write side is still a writev per response and is the next thing to move.
21 *
22 * Opt-in for now: HYPERMAN_BACKEND=io_uring or Hyperman::Loop->new('io_uring');
23 * auto-selection stays kqueue > epoll > poll until this is benchmarked on
24 * reference hardware.
25 */
26  
27 #include "hyperman.h"
28  
29 #if defined(__linux__) && defined(HM_HAVE_LIBURING)
30 #define HM_IOURING_ACTIVE 1
31 #endif
32  
33 #ifdef HM_IOURING_ACTIVE
34  
35 #include <liburing.h>
36 #include <sys/signalfd.h>
37 #include <signal.h>
38 #include <string.h>
39 #include <stdlib.h>
40 #include <unistd.h>
41 #include <errno.h>
42 #include <poll.h>
43  
44 /* user_data tagging (low 3 bits). Timer completions carry a pointer to
45 * their ur_timer, which is 8-aligned so its tag is 0. liburing's own
46 * internal wait-timeout uses LIBURING_UDATA_TIMEOUT and is skipped. */
47 #define UR_TAG_MASK 7ULL
48 #define UR_TAG_TIMER 0ULL /* aligned ur_timer* */
49 #define UR_TAG_READ 1ULL
50 #define UR_TAG_WRITE 2ULL
51 #define UR_TAG_RECV 3ULL /* completion-mode recv (see ur_submit_recv) */
52 #define UR_TAG_IGNORE 5ULL /* poll-remove completions: drop */
53  
54 /* io_uring_sqe_set_data64() was only added in liburing 2.2; assign the
55 * user_data field directly (exactly what that helper does) so we build and
56 * load against every liburing version. */
57 #define UR_SET_DATA(sqe, val) ((sqe)->user_data = (__u64)(val))
58  
59 typedef struct ur_timer {
60 struct __kernel_timespec ts;
61 void *udata;
62 int interval; /* re-arm on fire? */
63 int cancelled; /* freed when its completion lands, not before */
64 struct ur_timer *prev, *next;
65 } ur_timer;
66  
67 typedef struct {
68 struct io_uring ring;
69 unsigned char mask[HM_MAXFD]; /* wanted HM_EV bits */
70 unsigned char once[HM_MAXFD]; /* oneshot HM_EV bits */
71 unsigned char armed[HM_MAXFD]; /* HM_EV bits with in-flight sqe */
72 /* completion-mode recv: at most ONE in flight per fd, and the token that
73 * identifies whose it is. Kept per-fd rather than in user_data because
74 * user_data is 64 bits and already carries the fd and the tag. */
75 uint64_t recv_token[HM_MAXFD];
76 unsigned char recv_armed[HM_MAXFD];
77 ur_timer *timers; /* doubly-linked, for cleanup */
78 int sfd; /* signalfd, -1 until first sig */
79 sigset_t sigs;
80 /* A poll re-arm found no submission slot (the ring was full and the
81 * submit that should have drained it failed, -EBUSY under completion
82 * overflow). The interest is still in mask[] with armed[] clear, and
83 * ur_wait re-arms every such fd before it sleeps, so a connection is
84 * never left unwatched by a momentary full ring. */
85 int sq_dropped;
86 } hm_ur_state;
87  
88 /* Get an sqe, flushing the ring if the submission queue is momentarily full. */
89 /*
90 # uncoverable statement
91 # uncoverable subroutine */
92 static struct io_uring_sqe *ur_sqe(hm_ur_state *st) {
93 struct io_uring_sqe *sqe = io_uring_get_sqe(&st->ring); /* # uncoverable statement */
94 /*
95 # uncoverable statement
96 # uncoverable branch true
97 # uncoverable branch false
98 # uncoverable condition left count:1
99 # uncoverable condition right count:1 */
100 if (!sqe) {
101 io_uring_submit(&st->ring); /* # uncoverable statement */
102 sqe = io_uring_get_sqe(&st->ring); /* # uncoverable statement */
103 }
104 return sqe; /* # uncoverable statement */
105 }
106  
107 /*
108 # uncoverable statement
109 # uncoverable subroutine */
110 static void ur_arm_poll(hm_ur_state *st, int fd, int evbit) {
111 struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */
112 unsigned mask = (evbit == HM_EV_READ) ? (POLLIN | POLLHUP | POLLERR) /* # uncoverable statement */
113 : (POLLOUT | POLLHUP | POLLERR); /* # uncoverable statement */
114 /*
115 # uncoverable statement
116 # uncoverable branch true
117 # uncoverable branch false
118 # uncoverable condition left count:1
119 # uncoverable condition right count:1 */
120 if (!sqe) { st->sq_dropped = 1; return; }
121 io_uring_prep_poll_add(sqe, fd, mask); /* # uncoverable statement */
122 UR_SET_DATA(sqe, /* # uncoverable statement */
123 ((__u64)(unsigned)fd << 3)
124 | (evbit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE));
125 st->armed[fd] |= (unsigned char)evbit; /* # uncoverable statement */
126 }
127  
128 /*
129 # uncoverable statement
130 # uncoverable subroutine */
131 static int ur_add_io(hm_backend *be, int fd, int mask, int oneshot) {
132 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
133 int bit;
134 /*
135 # uncoverable statement
136 # uncoverable branch true
137 # uncoverable branch false
138 # uncoverable condition left count:1
139 # uncoverable condition right count:1
140 # uncoverable condition left count:2
141 # uncoverable condition right count:2 */
142 if (fd < 0 || fd >= HM_MAXFD) return -1;
143 st->mask[fd] |= (unsigned char)mask; /* # uncoverable statement */
144 /*
145 # uncoverable statement
146 # uncoverable branch true
147 # uncoverable branch false
148 # uncoverable condition left count:1
149 # uncoverable condition right count:1 */
150 if (oneshot) st->once[fd] |= (unsigned char)mask;
151 else st->once[fd] &= (unsigned char)~mask; /* # uncoverable statement */
152 /*
153 # uncoverable statement
154 # uncoverable branch true
155 # uncoverable branch false
156 # uncoverable condition left count:1
157 # uncoverable condition right count:1 */
158 for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1)
159 /*
160 # uncoverable statement
161 # uncoverable branch true
162 # uncoverable branch false
163 # uncoverable condition left count:1
164 # uncoverable condition right count:1
165 # uncoverable condition left count:2
166 # uncoverable condition right count:2 */
167 if ((mask & bit) && !(st->armed[fd] & bit))
168 ur_arm_poll(st, fd, bit); /* # uncoverable statement */
169 return 0; /* # uncoverable statement */
170 }
171  
172 /*
173 # uncoverable statement
174 # uncoverable subroutine */
175 static int ur_modify_io(hm_backend *be, int fd, int mask, int oneshot) {
176 return ur_add_io(be, fd, mask, oneshot); /* # uncoverable statement */
177 }
178  
179 /*
180 # uncoverable statement
181 # uncoverable subroutine */
182 static int ur_remove_io(hm_backend *be, int fd, int mask) {
183 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
184 int bit;
185 /*
186 # uncoverable statement
187 # uncoverable branch true
188 # uncoverable branch false
189 # uncoverable condition left count:1
190 # uncoverable condition right count:1
191 # uncoverable condition left count:2
192 # uncoverable condition right count:2 */
193 if (fd < 0 || fd >= HM_MAXFD) return -1;
194 st->mask[fd] &= (unsigned char)~mask; /* # uncoverable statement */
195 st->once[fd] &= (unsigned char)~mask; /* # uncoverable statement */
196 /* cancel any in-flight poll for the dropped interest, so a closed+reused
197 * fd can't be woken by a stale completion */
198 /*
199 # uncoverable statement
200 # uncoverable branch true
201 # uncoverable branch false
202 # uncoverable condition left count:1
203 # uncoverable condition right count:1 */
204 for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) {
205 /*
206 # uncoverable statement
207 # uncoverable branch true
208 # uncoverable branch false
209 # uncoverable condition left count:1
210 # uncoverable condition right count:1
211 # uncoverable condition left count:2
212 # uncoverable condition right count:2 */
213 if ((mask & bit) && (st->armed[fd] & bit)) {
214 struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */
215 /*
216 # uncoverable statement
217 # uncoverable branch true
218 # uncoverable branch false
219 # uncoverable condition left count:1
220 # uncoverable condition right count:1 */
221 if (sqe) {
222 io_uring_prep_poll_remove(sqe, /* # uncoverable statement */
223 ((__u64)(unsigned)fd << 3) /* # uncoverable statement */
224 /*
225 # uncoverable statement
226 # uncoverable condition left count:1
227 # uncoverable condition right count:1 */
228 | (bit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE));
229 UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */
230 st->armed[fd] &= (unsigned char)~bit; /* # uncoverable statement */
231 }
232 /* No slot for the cancel: the poll stays in flight and armed
233 * stays SET, so a later add does not arm a second one; its
234 * completion finds the interest gone (mask) and is dropped
235 * there, which clears armed. */
236 }
237 }
238 return 0; /* # uncoverable statement */
239 }
240  
241 #ifdef HM_HAVE_URING_RECV
242 /* Ask for one read into `buf`, delivered later as HM_EV_RECV.
243 *
244 * REFUSES WHILE ONE IS ALREADY IN FLIGHT ON THIS FD, and that refusal is
245 * load-bearing rather than defensive. The token lives in a per-fd slot, so a
246 * second submission would overwrite the first one's token - and then the
247 * FIRST completion, belonging to a connection that has already gone, would
248 * arrive wearing the SECOND connection's identity, pass the core's generation
249 * check, and have its byte count added to a buffer it never wrote into. The
250 * core reads -1 as "do it yourself" and falls back to a readiness read, which
251 * is always correct, just not as cheap. */
252 /*
253 # uncoverable statement
254 # uncoverable subroutine */
255 static int ur_submit_recv(hm_backend *be, int fd, void *buf, size_t len,
256 uint64_t token) {
257 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
258 struct io_uring_sqe *sqe;
259 /*
260 # uncoverable statement
261 # uncoverable branch true
262 # uncoverable branch false
263 # uncoverable condition left count:1
264 # uncoverable condition right count:1
265 # uncoverable condition left count:2
266 # uncoverable condition right count:2
267 # uncoverable condition left count:3
268 # uncoverable condition right count:3 */
269 if (fd < 0 || fd >= HM_MAXFD || len == 0) return -1;
270 /*
271 # uncoverable statement
272 # uncoverable branch true
273 # uncoverable branch false
274 # uncoverable condition left count:1
275 # uncoverable condition right count:1 */
276 if (st->recv_armed[fd]) return -1;
277 sqe = ur_sqe(st); /* # uncoverable statement */
278 /*
279 # uncoverable statement
280 # uncoverable branch true
281 # uncoverable branch false
282 # uncoverable condition left count:1
283 # uncoverable condition right count:1 */
284 if (!sqe) return -1;
285 io_uring_prep_recv(sqe, fd, buf, len, 0); /* # uncoverable statement */
286 UR_SET_DATA(sqe, ((__u64)(unsigned)fd << 3) | UR_TAG_RECV); /* # uncoverable statement */
287 st->recv_token[fd] = token; /* # uncoverable statement */
288 st->recv_armed[fd] = 1; /* # uncoverable statement */
289 return 0; /* # uncoverable statement */
290 }
291  
292 /* Best-effort cancel at close.
293 *
294 * recv_armed is deliberately NOT cleared here. The completion still lands -
295 * cancelled or not - and it is the only moment at which the buffer the kernel
296 * was given can be known to be free again. Clearing the flag early would let
297 * the next connection on this fd number arm a second recv while the first is
298 * still outstanding, which is exactly the overwrite ur_submit_recv refuses. */
299 /*
300 # uncoverable statement
301 # uncoverable subroutine */
302 static void ur_cancel_recv(hm_backend *be, int fd) {
303 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
304 struct io_uring_sqe *sqe;
305 /*
306 # uncoverable statement
307 # uncoverable branch true
308 # uncoverable branch false
309 # uncoverable condition left count:1
310 # uncoverable condition right count:1
311 # uncoverable condition left count:2
312 # uncoverable condition right count:2
313 # uncoverable condition left count:3
314 # uncoverable condition right count:3 */
315 if (fd < 0 || fd >= HM_MAXFD || !st->recv_armed[fd]) return;
316 sqe = ur_sqe(st); /* # uncoverable statement */
317 /*
318 # uncoverable statement
319 # uncoverable branch true
320 # uncoverable branch false
321 # uncoverable condition left count:1
322 # uncoverable condition right count:1 */
323 if (!sqe) return;
324 io_uring_prep_cancel(sqe, /* # uncoverable statement */
325 (void *)(uintptr_t)(((__u64)(unsigned)fd << 3) | UR_TAG_RECV), 0); /* # uncoverable statement */
326 UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */
327 }
328 #endif /* HM_HAVE_URING_RECV */
329  
330 /*
331 # uncoverable statement
332 # uncoverable subroutine */
333 static int ur_arm_timer(hm_ur_state *st, ur_timer *t) {
334 struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */
335 /*
336 # uncoverable statement
337 # uncoverable branch true
338 # uncoverable branch false
339 # uncoverable condition left count:1
340 # uncoverable condition right count:1 */
341 if (!sqe) return -1;
342 io_uring_prep_timeout(sqe, &t->ts, 0, 0); /* # uncoverable statement */
343 UR_SET_DATA(sqe, (__u64)(uintptr_t)t); /* tag 0 (aligned) */ /* # uncoverable statement */
344 return 0; /* # uncoverable statement */
345 }
346  
347 /*
348 # uncoverable statement
349 # uncoverable subroutine */
350 static int ur_add_timer(hm_backend *be, double secs, int oneshot, void *udata) {
351 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
352 ur_timer *t = (ur_timer *)hm_xcalloc(1, sizeof(ur_timer)); /* # uncoverable statement */
353 /*
354 # uncoverable statement
355 # uncoverable branch true
356 # uncoverable branch false
357 # uncoverable condition left count:1
358 # uncoverable condition right count:1 */
359 if (!t) return -1;
360 t->ts.tv_sec = (int64_t)secs; /* # uncoverable statement */
361 t->ts.tv_nsec = (int64_t)((secs - (double)t->ts.tv_sec) * 1e9); /* # uncoverable statement */
362 /*
363 # uncoverable statement
364 # uncoverable branch true
365 # uncoverable branch false
366 # uncoverable condition left count:1
367 # uncoverable condition right count:1
368 # uncoverable condition left count:2
369 # uncoverable condition right count:2 */
370 if (t->ts.tv_sec == 0 && t->ts.tv_nsec == 0) t->ts.tv_nsec = 1;
371 t->udata = udata; /* # uncoverable statement */
372 t->interval = !oneshot; /* # uncoverable statement */
373 t->next = st->timers; /* # uncoverable statement */
374 /*
375 # uncoverable statement
376 # uncoverable branch true
377 # uncoverable branch false
378 # uncoverable condition left count:1
379 # uncoverable condition right count:1 */
380 if (st->timers) st->timers->prev = t;
381 st->timers = t; /* # uncoverable statement */
382 /*
383 # uncoverable statement
384 # uncoverable branch true
385 # uncoverable branch false
386 # uncoverable condition left count:1
387 # uncoverable condition right count:1 */
388 if (ur_arm_timer(st, t) < 0) {
389 /* No submission slot: the caller is told, as the other backends
390 * tell it when a timerfd cannot be had, rather than owning a timer
391 * that will never fire. t is the list head, just linked. */
392 st->timers = t->next; /* # uncoverable statement */
393 /*
394 # uncoverable statement
395 # uncoverable branch true
396 # uncoverable branch false
397 # uncoverable condition left count:1
398 # uncoverable condition right count:1 */
399 if (t->next) t->next->prev = NULL;
400 free(t); /* # uncoverable statement */
401 errno = EAGAIN; /* # uncoverable statement */
402 return -1; /* # uncoverable statement */
403 }
404 return 0; /* # uncoverable statement */
405 }
406  
407 /*
408 # uncoverable statement
409 # uncoverable subroutine */
410 static void ur_timer_unlink(hm_ur_state *st, ur_timer *t) {
411 /*
412 # uncoverable statement
413 # uncoverable branch true
414 # uncoverable branch false
415 # uncoverable condition left count:1
416 # uncoverable condition right count:1 */
417 if (t->prev) t->prev->next = t->next;
418 else st->timers = t->next; /* # uncoverable statement */
419 /*
420 # uncoverable statement
421 # uncoverable branch true
422 # uncoverable branch false
423 # uncoverable condition left count:1
424 # uncoverable condition right count:1 */
425 if (t->next) t->next->prev = t->prev;
426 }
427  
428 /* Cancel a timer. The timeout is already in flight in the kernel and its
429 * user_data IS this pointer, so freeing here would leave the kernel holding a
430 * dangling reference: the completion still arrives, ur_wait dereferences it
431 * and frees it a second time. Worse, the block is the right size to be handed
432 * straight back to the next ur_add_timer, so the stale completion lands on a
433 * live timer and takes that down instead - which is why this showed up as an
434 * intermittent "double free or corruption" rather than a reliable one.
435 *
436 * So: ask the kernel to remove the timeout and mark the watcher dead, but
437 * leave it linked and allocated. Its completion (-ECANCELED, or a genuine
438 * fire that raced the removal) frees it in ur_wait; anything still linked at
439 * shutdown is freed by ur_destroy. */
440 /*
441 # uncoverable statement
442 # uncoverable subroutine */
443 static int ur_del_timer(hm_backend *be, void *udata) {
444 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
445 ur_timer *t;
446 /*
447 # uncoverable statement
448 # uncoverable branch true
449 # uncoverable branch false
450 # uncoverable condition left count:1
451 # uncoverable condition right count:1 */
452 for (t = st->timers; t; t = t->next) {
453 /*
454 # uncoverable statement
455 # uncoverable branch true
456 # uncoverable branch false
457 # uncoverable condition left count:1
458 # uncoverable condition right count:1
459 # uncoverable condition left count:2
460 # uncoverable condition right count:2 */
461 if (t->udata == udata && !t->cancelled) {
462 struct io_uring_sqe *sqe;
463 t->cancelled = 1; /* # uncoverable statement */
464 t->interval = 0; /* never re-arm a cancelled timer */ /* # uncoverable statement */
465 /*
466 # uncoverable statement
467 # uncoverable branch true
468 # uncoverable branch false
469 # uncoverable condition left count:1
470 # uncoverable condition right count:1 */
471 if ((sqe = ur_sqe(st)) != NULL) {
472 io_uring_prep_timeout_remove(sqe, (__u64)(uintptr_t)t, 0); /* # uncoverable statement */
473 UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */
474 }
475 return 0; /* # uncoverable statement */
476 }
477 }
478 return -1; /* # uncoverable statement */
479 }
480  
481 /*
482 # uncoverable statement
483 # uncoverable subroutine */
484 static int ur_add_signal(hm_backend *be, int signo) {
485 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
486 sigaddset(&st->sigs, signo); /* # uncoverable statement */
487 /*
488 # uncoverable statement
489 # uncoverable branch true
490 # uncoverable branch false
491 # uncoverable condition left count:1
492 # uncoverable condition right count:1 */
493 if (sigprocmask(SIG_BLOCK, &st->sigs, NULL) < 0) return -1;
494 {
495 int sfd = signalfd(st->sfd, &st->sigs, SFD_NONBLOCK | SFD_CLOEXEC); /* # uncoverable statement */
496 /*
497 # uncoverable statement
498 # uncoverable branch true
499 # uncoverable branch false
500 # uncoverable condition left count:1
501 # uncoverable condition right count:1 */
502 if (sfd < 0) return -1;
503 /*
504 # uncoverable statement
505 # uncoverable branch true
506 # uncoverable branch false
507 # uncoverable condition left count:1
508 # uncoverable condition right count:1 */
509 if (st->sfd < 0) {
510 st->sfd = sfd; /* # uncoverable statement */
511 st->mask[sfd] |= HM_EV_READ; /* persistent poll on the ring */ /* # uncoverable statement */
512 ur_arm_poll(st, sfd, HM_EV_READ); /* # uncoverable statement */
513 }
514 }
515 return 0; /* # uncoverable statement */
516 }
517  
518 /*
519 # uncoverable statement
520 # uncoverable subroutine */
521 static int ur_wait(hm_backend *be, hm_event *out, int max, double timeout) {
522 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
523 struct io_uring_cqe *cqes[HM_MAXEV];
524 struct io_uring_cqe *cqe;
525 unsigned want, n, i;
526 int nev = 0, ret; /* # uncoverable statement */
527  
528 /*
529 # uncoverable statement
530 # uncoverable branch true
531 # uncoverable branch false
532 # uncoverable condition left count:1
533 # uncoverable condition right count:1 */
534 if (st->sq_dropped) {
535 /* A re-arm found the ring full: every interest with no poll in
536 * flight is armed again now, with the ring drained by the submit
537 * below between attempts. Rare, so the full-table walk is fine. */
538 int fd;
539 st->sq_dropped = 0; /* # uncoverable statement */
540 /*
541 # uncoverable statement
542 # uncoverable branch true
543 # uncoverable branch false
544 # uncoverable condition left count:1
545 # uncoverable condition right count:1 */
546 for (fd = 0; fd < HM_MAXFD; fd++) {
547 unsigned char need = st->mask[fd] & (unsigned char)~st->armed[fd]; /* # uncoverable statement */
548 /*
549 # uncoverable statement
550 # uncoverable branch true
551 # uncoverable branch false
552 # uncoverable condition left count:1
553 # uncoverable condition right count:1 */
554 if (!need) continue;
555 /*
556 # uncoverable statement
557 # uncoverable branch true
558 # uncoverable branch false
559 # uncoverable condition left count:1
560 # uncoverable condition right count:1 */
561 if (need & HM_EV_READ) ur_arm_poll(st, fd, HM_EV_READ);
562 /*
563 # uncoverable statement
564 # uncoverable branch true
565 # uncoverable branch false
566 # uncoverable condition left count:1
567 # uncoverable condition right count:1 */
568 if (need & HM_EV_WRITE) ur_arm_poll(st, fd, HM_EV_WRITE);
569 }
570 }
571 io_uring_submit(&st->ring); /* flush re-arms, cancels, timers */ /* # uncoverable statement */
572  
573 /*
574 # uncoverable statement
575 # uncoverable branch true
576 # uncoverable branch false
577 # uncoverable condition left count:1
578 # uncoverable condition right count:1 */
579 if (timeout < 0) {
580 ret = io_uring_wait_cqe(&st->ring, &cqe); /* # uncoverable statement */
581 /*
582 # uncoverable statement
583 # uncoverable branch true
584 # uncoverable branch false
585 # uncoverable condition left count:1
586 # uncoverable condition right count:1 */
587 } else if (timeout == 0) {
588 ret = io_uring_peek_cqe(&st->ring, &cqe); /* # uncoverable statement */
589 /*
590 # uncoverable statement
591 # uncoverable branch true
592 # uncoverable branch false
593 # uncoverable condition left count:1
594 # uncoverable condition right count:1 */
595 if (ret == -EAGAIN) return 0;
596 } else {
597 struct __kernel_timespec ts;
598 ts.tv_sec = (int64_t)timeout; /* # uncoverable statement */
599 ts.tv_nsec = (int64_t)((timeout - (double)ts.tv_sec) * 1e9); /* # uncoverable statement */
600 ret = io_uring_wait_cqe_timeout(&st->ring, &cqe, &ts); /* # uncoverable statement */
601 /*
602 # uncoverable statement
603 # uncoverable branch true
604 # uncoverable branch false
605 # uncoverable condition left count:1
606 # uncoverable condition right count:1 */
607 if (ret == -ETIME) return 0;
608 }
609 /*
610 # uncoverable statement
611 # uncoverable branch true
612 # uncoverable branch false
613 # uncoverable condition left count:1
614 # uncoverable condition right count:1
615 # uncoverable condition left count:2
616 # uncoverable condition right count:2 */
617 if (ret < 0) return (ret == -EINTR) ? 0 : -1;
618  
619 /* headroom so a single signalfd completion (multiple queued signals)
620 * can't overflow the caller's event buffer */
621 /*
622 # uncoverable statement
623 # uncoverable condition left count:1
624 # uncoverable condition right count:1 */
625 want = (max > 8) ? (unsigned)(max - 8) : 1u;
626 /*
627 # uncoverable statement
628 # uncoverable branch true
629 # uncoverable branch false
630 # uncoverable condition left count:1
631 # uncoverable condition right count:1 */
632 if (want > HM_MAXEV) want = HM_MAXEV;
633 n = io_uring_peek_batch_cqe(&st->ring, cqes, want); /* # uncoverable statement */
634  
635 /*
636 # uncoverable statement
637 # uncoverable branch true
638 # uncoverable branch false
639 # uncoverable condition left count:1
640 # uncoverable condition right count:1 */
641 for (i = 0; i < n; i++) {
642 __u64 data = cqes[i]->user_data; /* # uncoverable statement */
643 int res = cqes[i]->res; /* # uncoverable statement */
644 __u64 tag = data & UR_TAG_MASK; /* # uncoverable statement */
645  
646 /*
647 # uncoverable statement
648 # uncoverable branch true
649 # uncoverable branch false
650 # uncoverable condition left count:1
651 # uncoverable condition right count:1 */
652 if (data == LIBURING_UDATA_TIMEOUT) continue; /* liburing internal */
653 /*
654 # uncoverable statement
655 # uncoverable branch true
656 # uncoverable branch false
657 # uncoverable condition left count:1
658 # uncoverable condition right count:1 */
659 if (tag == UR_TAG_IGNORE) continue; /* poll-remove ack */
660  
661 #ifdef HM_HAVE_URING_RECV
662 /*
663 # uncoverable statement
664 # uncoverable branch true
665 # uncoverable branch false
666 # uncoverable condition left count:1
667 # uncoverable condition right count:1 */
668 if (tag == UR_TAG_RECV) {
669 int fd = (int)(data >> 3); /* # uncoverable statement */
670 st->recv_armed[fd] = 0; /* the buffer is the kernel's no longer */ /* # uncoverable statement */
671 /*
672 # uncoverable statement
673 # uncoverable branch true
674 # uncoverable branch false
675 # uncoverable condition left count:1
676 # uncoverable condition right count:1 */
677 if (nev >= max) break;
678 /* Delivered even when cancelled or failed: a completion is the
679 * only signal that the buffer handed to the kernel is free, and
680 * the core has one to release whether the read worked or not. */
681 out[nev].kind = HM_EV_RECV; /* # uncoverable statement */
682 out[nev].fd = fd; /* # uncoverable statement */
683 out[nev].res = res; /* # uncoverable statement */
684 out[nev].token = st->recv_token[fd]; /* # uncoverable statement */
685 out[nev].udata = NULL; /* # uncoverable statement */
686 nev++; /* # uncoverable statement */
687 continue; /* # uncoverable statement */
688 }
689 #endif
690  
691 /*
692 # uncoverable statement
693 # uncoverable branch true
694 # uncoverable branch false
695 # uncoverable condition left count:1
696 # uncoverable condition right count:1
697 # uncoverable condition left count:2
698 # uncoverable condition right count:2 */
699 if (tag == UR_TAG_READ || tag == UR_TAG_WRITE) {
700 int fd = (int)(data >> 3); /* # uncoverable statement */
701 /*
702 # uncoverable statement
703 # uncoverable condition left count:1
704 # uncoverable condition right count:1 */
705 int bit = (tag == UR_TAG_READ) ? HM_EV_READ : HM_EV_WRITE;
706 st->armed[fd] &= (unsigned char)~bit; /* # uncoverable statement */
707 /*
708 # uncoverable statement
709 # uncoverable branch true
710 # uncoverable branch false
711 # uncoverable condition left count:1
712 # uncoverable condition right count:1
713 # uncoverable condition left count:2
714 # uncoverable condition right count:2 */
715 if (res == -ECANCELED || !(st->mask[fd] & bit)) continue;
716  
717 /*
718 # uncoverable statement
719 # uncoverable branch true
720 # uncoverable branch false
721 # uncoverable condition left count:1
722 # uncoverable condition right count:1
723 # uncoverable condition left count:2
724 # uncoverable condition right count:2 */
725 if (st->sfd >= 0 && fd == st->sfd) {
726 struct signalfd_siginfo si;
727 /*
728 # uncoverable statement
729 # uncoverable branch true
730 # uncoverable branch false */
731 while (nev < max
732 /*
733 # uncoverable statement
734 # uncoverable condition left count:1
735 # uncoverable condition right count:1
736 # uncoverable condition left count:2
737 # uncoverable condition right count:2 */
738 && read(st->sfd, &si, sizeof(si)) == (ssize_t)sizeof(si)) {
739 out[nev].kind = HM_EV_SIGNAL; /* # uncoverable statement */
740 out[nev].fd = (int)si.ssi_signo; /* # uncoverable statement */
741 out[nev].udata = NULL; /* # uncoverable statement */
742 nev++; /* # uncoverable statement */
743 }
744 ur_arm_poll(st, st->sfd, HM_EV_READ); /* persistent */ /* # uncoverable statement */
745 continue; /* # uncoverable statement */
746 }
747  
748 out[nev].kind = bit; /* # uncoverable statement */
749 out[nev].fd = fd; /* # uncoverable statement */
750 out[nev].udata = NULL; /* # uncoverable statement */
751 nev++; /* # uncoverable statement */
752 /*
753 # uncoverable statement
754 # uncoverable branch true
755 # uncoverable branch false
756 # uncoverable condition left count:1
757 # uncoverable condition right count:1 */
758 if (st->once[fd] & bit) {
759 st->mask[fd] &= (unsigned char)~bit; /* # uncoverable statement */
760 st->once[fd] &= (unsigned char)~bit; /* # uncoverable statement */
761 } else {
762 ur_arm_poll(st, fd, bit); /* persistent: re-arm */ /* # uncoverable statement */
763 }
764 continue; /* # uncoverable statement */
765 }
766  
767 { /* timer: user_data is the ur_timer pointer (tag 0) */
768 ur_timer *t = (ur_timer *)(uintptr_t)data; /* # uncoverable statement */
769 /*
770 # uncoverable statement
771 # uncoverable branch true
772 # uncoverable branch false
773 # uncoverable condition left count:1
774 # uncoverable condition right count:1 */
775 if (!t) continue;
776 /* A cancelled timer's completion still arrives; it owns the
777 * struct, and it must not be reported to the loop. -ECANCELED
778 * covers the removal succeeding, t->cancelled the case where the
779 * timeout fired before the removal reached the kernel. */
780 /*
781 # uncoverable statement
782 # uncoverable branch true
783 # uncoverable branch false
784 # uncoverable condition left count:1
785 # uncoverable condition right count:1
786 # uncoverable condition left count:2
787 # uncoverable condition right count:2 */
788 if (t->cancelled || res == -ECANCELED) {
789 ur_timer_unlink(st, t); /* # uncoverable statement */
790 free(t); /* # uncoverable statement */
791 continue; /* # uncoverable statement */
792 }
793 out[nev].kind = HM_EV_TIMER; /* # uncoverable statement */
794 out[nev].fd = -1; /* # uncoverable statement */
795 out[nev].udata = t->udata; /* # uncoverable statement */
796 nev++; /* # uncoverable statement */
797 /*
798 # uncoverable statement
799 # uncoverable branch true
800 # uncoverable branch false
801 # uncoverable condition left count:1
802 # uncoverable condition right count:1 */
803 if (t->interval) {
804 ur_arm_timer(st, t); /* # uncoverable statement */
805 } else {
806 ur_timer_unlink(st, t); /* # uncoverable statement */
807 free(t); /* # uncoverable statement */
808 }
809 }
810 }
811 io_uring_cq_advance(&st->ring, n); /* # uncoverable statement */
812 return nev; /* # uncoverable statement */
813 }
814  
815 /*
816 # uncoverable statement
817 # uncoverable subroutine */
818 static void ur_destroy(hm_backend *be) {
819 hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */
820 /*
821 # uncoverable statement
822 # uncoverable branch true
823 # uncoverable branch false
824 # uncoverable condition left count:1
825 # uncoverable condition right count:1 */
826 if (st) {
827 /*
828 # uncoverable statement
829 # uncoverable branch true
830 # uncoverable branch false
831 # uncoverable condition left count:1
832 # uncoverable condition right count:1 */
833 while (st->timers) {
834 ur_timer *t = st->timers; /* # uncoverable statement */
835 st->timers = t->next; /* # uncoverable statement */
836 free(t); /* # uncoverable statement */
837 }
838 /* be->foreign: inherited across a fork. The signalfd and the ring's
839 * own descriptor are the parent's; io_uring_queue_exit would close
840 * the ring fd and unmap rings this process never set up. */
841 /*
842 # uncoverable statement
843 # uncoverable branch true
844 # uncoverable branch false
845 # uncoverable condition left count:1
846 # uncoverable condition right count:1 */
847 if (!be->foreign) {
848 /*
849 # uncoverable statement
850 # uncoverable branch true
851 # uncoverable branch false
852 # uncoverable condition left count:1
853 # uncoverable condition right count:1 */
854 if (st->sfd >= 0) close(st->sfd);
855 io_uring_queue_exit(&st->ring); /* # uncoverable statement */
856 }
857 free(st); /* # uncoverable statement */
858 }
859 free(be); /* # uncoverable statement */
860 }
861  
862 hm_backend *hm_backend_iouring_new(void) {
863 hm_backend *be = (hm_backend *)hm_xcalloc(1, sizeof(hm_backend));
864 hm_ur_state *st = (hm_ur_state *)hm_xcalloc(1, sizeof(hm_ur_state));
865 /*
866 # uncoverable branch true
867 # uncoverable condition left count:1
868 # uncoverable condition left count:2 */
869 if (!be || !st) { free(be); free(st); return NULL; }
870 st->sfd = -1;
871 sigemptyset(&st->sigs);
872 /*
873 # uncoverable branch false
874 # uncoverable condition right count:1 */
875 if (io_uring_queue_init(1024, &st->ring, 0) < 0) {
876 free(be); free(st); return NULL;
877 }
878 be->name = "io_uring"; /* # uncoverable statement */
879 be->state = st; /* # uncoverable statement */
880 be->add_io = ur_add_io; /* # uncoverable statement */
881 be->modify_io = ur_modify_io; /* # uncoverable statement */
882 be->remove_io = ur_remove_io; /* # uncoverable statement */
883 be->add_timer = ur_add_timer; /* # uncoverable statement */
884 be->del_timer = ur_del_timer; /* # uncoverable statement */
885 be->add_signal = ur_add_signal; /* # uncoverable statement */
886 be->wait = ur_wait; /* # uncoverable statement */
887 be->destroy = ur_destroy; /* # uncoverable statement */
888 #ifdef HM_HAVE_URING_RECV
889 be->submit_recv = ur_submit_recv; /* # uncoverable statement */
890 be->cancel_recv = ur_cancel_recv; /* # uncoverable statement */
891 #endif
892 return be; /* # uncoverable statement */
893 }
894  
895 int hm_backend_iouring_available(void) {
896 /* liburing is linked; confirm the kernel actually supports io_uring
897 * (not the case under, e.g., default Docker seccomp) */
898 static int avail = -1;
899 /*
900 # uncoverable branch false
901 # uncoverable condition right count:1 */
902 if (avail < 0) {
903 struct io_uring ring;
904 /*
905 # uncoverable branch true
906 # uncoverable condition left count:1 */
907 if (io_uring_queue_init(2, &ring, 0) == 0) {
908 io_uring_queue_exit(&ring); /* # uncoverable statement */
909 avail = 1; /* # uncoverable statement */
910 } else {
911 avail = 0;
912 }
913 }
914 return avail;
915 }
916  
917 #else /* !HM_IOURING_ACTIVE */
918  
919 151 hm_backend *hm_backend_iouring_new(void) { return 0; }
920 5 int hm_backend_iouring_available(void) { return 0; }
921  
922 #endif
« blib/lib/Hyperman/Loop.pm ↑ Summary include/hyperman/backend_kqueue.c »