| 1 | /* Hyperman::Event::IOUring - io_uring readiness backend (Linux), built on | ||
| 2 | * liburing. Enabled by Makefile.PL when liburing is found (pkg-config or a | ||
| 3 | * -luring compile probe), which defines HM_HAVE_LIBURING and links -luring; | ||
| 4 | * otherwise this compiles to graceful "unavailable" stubs. | ||
| 5 | * | ||
| 6 | * Readiness comes from the ring in *poll mode*: IORING_OP_POLL_ADD | ||
| 7 | * completions mapped onto the hm_backend vtable (persistent watchers are | ||
| 8 | * re-armed on delivery), timers as IORING_OP_TIMEOUT, and shutdown signals | ||
| 9 | * via a signalfd polled on the ring. | ||
| 10 | * | ||
| 11 | * COMPLETION-MODE READS (HYPERMAN_COMPLETION=1) go further: the core asks for | ||
| 12 | * the read up front with ur_submit_recv and is handed bytes that are already | ||
| 13 | * in its buffer, so the read stops being a syscall per request and becomes | ||
| 14 | * part of the one batched submission per turn of the loop. Measured, 2000 | ||
| 15 | * keep-alive requests over 50 connections: | ||
| 16 | * | ||
| 17 | * poll mode read 10050 writev 10000 enter 222 2.02 syscalls/req | ||
| 18 | * completion mode read 0 writev 10000 enter 203 1.02 syscalls/req | ||
| 19 | * | ||
| 20 | * The write side is still a writev per response and is the next thing to move. | ||
| 21 | * | ||
| 22 | * Opt-in for now: HYPERMAN_BACKEND=io_uring or Hyperman::Loop->new('io_uring'); | ||
| 23 | * auto-selection stays kqueue > epoll > poll until this is benchmarked on | ||
| 24 | * reference hardware. | ||
| 25 | */ | ||
| 26 | |||
| 27 | #include "hyperman.h" | ||
| 28 | |||
| 29 | #if defined(__linux__) && defined(HM_HAVE_LIBURING) | ||
| 30 | #define HM_IOURING_ACTIVE 1 | ||
| 31 | #endif | ||
| 32 | |||
| 33 | #ifdef HM_IOURING_ACTIVE | ||
| 34 | |||
| 35 | #include <liburing.h> | ||
| 36 | #include <sys/signalfd.h> | ||
| 37 | #include <signal.h> | ||
| 38 | #include <string.h> | ||
| 39 | #include <stdlib.h> | ||
| 40 | #include <unistd.h> | ||
| 41 | #include <errno.h> | ||
| 42 | #include <poll.h> | ||
| 43 | |||
| 44 | /* user_data tagging (low 3 bits). Timer completions carry a pointer to | ||
| 45 | * their ur_timer, which is 8-aligned so its tag is 0. liburing's own | ||
| 46 | * internal wait-timeout uses LIBURING_UDATA_TIMEOUT and is skipped. */ | ||
| 47 | #define UR_TAG_MASK 7ULL | ||
| 48 | #define UR_TAG_TIMER 0ULL /* aligned ur_timer* */ | ||
| 49 | #define UR_TAG_READ 1ULL | ||
| 50 | #define UR_TAG_WRITE 2ULL | ||
| 51 | #define UR_TAG_RECV 3ULL /* completion-mode recv (see ur_submit_recv) */ | ||
| 52 | #define UR_TAG_IGNORE 5ULL /* poll-remove completions: drop */ | ||
| 53 | |||
| 54 | /* io_uring_sqe_set_data64() was only added in liburing 2.2; assign the | ||
| 55 | * user_data field directly (exactly what that helper does) so we build and | ||
| 56 | * load against every liburing version. */ | ||
| 57 | #define UR_SET_DATA(sqe, val) ((sqe)->user_data = (__u64)(val)) | ||
| 58 | |||
| 59 | typedef struct ur_timer { | ||
| 60 | struct __kernel_timespec ts; | ||
| 61 | void *udata; | ||
| 62 | int interval; /* re-arm on fire? */ | ||
| 63 | int cancelled; /* freed when its completion lands, not before */ | ||
| 64 | struct ur_timer *prev, *next; | ||
| 65 | } ur_timer; | ||
| 66 | |||
| 67 | typedef struct { | ||
| 68 | struct io_uring ring; | ||
| 69 | unsigned char mask[HM_MAXFD]; /* wanted HM_EV bits */ | ||
| 70 | unsigned char once[HM_MAXFD]; /* oneshot HM_EV bits */ | ||
| 71 | unsigned char armed[HM_MAXFD]; /* HM_EV bits with in-flight sqe */ | ||
| 72 | /* completion-mode recv: at most ONE in flight per fd, and the token that | ||
| 73 | * identifies whose it is. Kept per-fd rather than in user_data because | ||
| 74 | * user_data is 64 bits and already carries the fd and the tag. */ | ||
| 75 | uint64_t recv_token[HM_MAXFD]; | ||
| 76 | unsigned char recv_armed[HM_MAXFD]; | ||
| 77 | ur_timer *timers; /* doubly-linked, for cleanup */ | ||
| 78 | int sfd; /* signalfd, -1 until first sig */ | ||
| 79 | sigset_t sigs; | ||
| 80 | /* A poll re-arm found no submission slot (the ring was full and the | ||
| 81 | * submit that should have drained it failed, -EBUSY under completion | ||
| 82 | * overflow). The interest is still in mask[] with armed[] clear, and | ||
| 83 | * ur_wait re-arms every such fd before it sleeps, so a connection is | ||
| 84 | * never left unwatched by a momentary full ring. */ | ||
| 85 | int sq_dropped; | ||
| 86 | } hm_ur_state; | ||
| 87 | |||
| 88 | /* Get an sqe, flushing the ring if the submission queue is momentarily full. */ | ||
| 89 | /* | ||
| 90 | # uncoverable statement | ||
| 91 | # uncoverable subroutine */ | ||
| 92 | static struct io_uring_sqe *ur_sqe(hm_ur_state *st) { | ||
| 93 | struct io_uring_sqe *sqe = io_uring_get_sqe(&st->ring); /* # uncoverable statement */ | ||
| 94 | /* | ||
| 95 | # uncoverable statement | ||
| 96 | # uncoverable branch true | ||
| 97 | # uncoverable branch false | ||
| 98 | # uncoverable condition left count:1 | ||
| 99 | # uncoverable condition right count:1 */ | ||
| 100 | if (!sqe) { | ||
| 101 | io_uring_submit(&st->ring); /* # uncoverable statement */ | ||
| 102 | sqe = io_uring_get_sqe(&st->ring); /* # uncoverable statement */ | ||
| 103 | } | ||
| 104 | return sqe; /* # uncoverable statement */ | ||
| 105 | } | ||
| 106 | |||
| 107 | /* | ||
| 108 | # uncoverable statement | ||
| 109 | # uncoverable subroutine */ | ||
| 110 | static void ur_arm_poll(hm_ur_state *st, int fd, int evbit) { | ||
| 111 | struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */ | ||
| 112 | unsigned mask = (evbit == HM_EV_READ) ? (POLLIN | POLLHUP | POLLERR) /* # uncoverable statement */ | ||
| 113 | : (POLLOUT | POLLHUP | POLLERR); /* # uncoverable statement */ | ||
| 114 | /* | ||
| 115 | # uncoverable statement | ||
| 116 | # uncoverable branch true | ||
| 117 | # uncoverable branch false | ||
| 118 | # uncoverable condition left count:1 | ||
| 119 | # uncoverable condition right count:1 */ | ||
| 120 | if (!sqe) { st->sq_dropped = 1; return; } | ||
| 121 | io_uring_prep_poll_add(sqe, fd, mask); /* # uncoverable statement */ | ||
| 122 | UR_SET_DATA(sqe, /* # uncoverable statement */ | ||
| 123 | ((__u64)(unsigned)fd << 3) | ||
| 124 | | (evbit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE)); | ||
| 125 | st->armed[fd] |= (unsigned char)evbit; /* # uncoverable statement */ | ||
| 126 | } | ||
| 127 | |||
| 128 | /* | ||
| 129 | # uncoverable statement | ||
| 130 | # uncoverable subroutine */ | ||
| 131 | static int ur_add_io(hm_backend *be, int fd, int mask, int oneshot) { | ||
| 132 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 133 | int bit; | ||
| 134 | /* | ||
| 135 | # uncoverable statement | ||
| 136 | # uncoverable branch true | ||
| 137 | # uncoverable branch false | ||
| 138 | # uncoverable condition left count:1 | ||
| 139 | # uncoverable condition right count:1 | ||
| 140 | # uncoverable condition left count:2 | ||
| 141 | # uncoverable condition right count:2 */ | ||
| 142 | if (fd < 0 || fd >= HM_MAXFD) return -1; | ||
| 143 | st->mask[fd] |= (unsigned char)mask; /* # uncoverable statement */ | ||
| 144 | /* | ||
| 145 | # uncoverable statement | ||
| 146 | # uncoverable branch true | ||
| 147 | # uncoverable branch false | ||
| 148 | # uncoverable condition left count:1 | ||
| 149 | # uncoverable condition right count:1 */ | ||
| 150 | if (oneshot) st->once[fd] |= (unsigned char)mask; | ||
| 151 | else st->once[fd] &= (unsigned char)~mask; /* # uncoverable statement */ | ||
| 152 | /* | ||
| 153 | # uncoverable statement | ||
| 154 | # uncoverable branch true | ||
| 155 | # uncoverable branch false | ||
| 156 | # uncoverable condition left count:1 | ||
| 157 | # uncoverable condition right count:1 */ | ||
| 158 | for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) | ||
| 159 | /* | ||
| 160 | # uncoverable statement | ||
| 161 | # uncoverable branch true | ||
| 162 | # uncoverable branch false | ||
| 163 | # uncoverable condition left count:1 | ||
| 164 | # uncoverable condition right count:1 | ||
| 165 | # uncoverable condition left count:2 | ||
| 166 | # uncoverable condition right count:2 */ | ||
| 167 | if ((mask & bit) && !(st->armed[fd] & bit)) | ||
| 168 | ur_arm_poll(st, fd, bit); /* # uncoverable statement */ | ||
| 169 | return 0; /* # uncoverable statement */ | ||
| 170 | } | ||
| 171 | |||
| 172 | /* | ||
| 173 | # uncoverable statement | ||
| 174 | # uncoverable subroutine */ | ||
| 175 | static int ur_modify_io(hm_backend *be, int fd, int mask, int oneshot) { | ||
| 176 | return ur_add_io(be, fd, mask, oneshot); /* # uncoverable statement */ | ||
| 177 | } | ||
| 178 | |||
| 179 | /* | ||
| 180 | # uncoverable statement | ||
| 181 | # uncoverable subroutine */ | ||
| 182 | static int ur_remove_io(hm_backend *be, int fd, int mask) { | ||
| 183 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 184 | int bit; | ||
| 185 | /* | ||
| 186 | # uncoverable statement | ||
| 187 | # uncoverable branch true | ||
| 188 | # uncoverable branch false | ||
| 189 | # uncoverable condition left count:1 | ||
| 190 | # uncoverable condition right count:1 | ||
| 191 | # uncoverable condition left count:2 | ||
| 192 | # uncoverable condition right count:2 */ | ||
| 193 | if (fd < 0 || fd >= HM_MAXFD) return -1; | ||
| 194 | st->mask[fd] &= (unsigned char)~mask; /* # uncoverable statement */ | ||
| 195 | st->once[fd] &= (unsigned char)~mask; /* # uncoverable statement */ | ||
| 196 | /* cancel any in-flight poll for the dropped interest, so a closed+reused | ||
| 197 | * fd can't be woken by a stale completion */ | ||
| 198 | /* | ||
| 199 | # uncoverable statement | ||
| 200 | # uncoverable branch true | ||
| 201 | # uncoverable branch false | ||
| 202 | # uncoverable condition left count:1 | ||
| 203 | # uncoverable condition right count:1 */ | ||
| 204 | for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) { | ||
| 205 | /* | ||
| 206 | # uncoverable statement | ||
| 207 | # uncoverable branch true | ||
| 208 | # uncoverable branch false | ||
| 209 | # uncoverable condition left count:1 | ||
| 210 | # uncoverable condition right count:1 | ||
| 211 | # uncoverable condition left count:2 | ||
| 212 | # uncoverable condition right count:2 */ | ||
| 213 | if ((mask & bit) && (st->armed[fd] & bit)) { | ||
| 214 | struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */ | ||
| 215 | /* | ||
| 216 | # uncoverable statement | ||
| 217 | # uncoverable branch true | ||
| 218 | # uncoverable branch false | ||
| 219 | # uncoverable condition left count:1 | ||
| 220 | # uncoverable condition right count:1 */ | ||
| 221 | if (sqe) { | ||
| 222 | io_uring_prep_poll_remove(sqe, /* # uncoverable statement */ | ||
| 223 | ((__u64)(unsigned)fd << 3) /* # uncoverable statement */ | ||
| 224 | /* | ||
| 225 | # uncoverable statement | ||
| 226 | # uncoverable condition left count:1 | ||
| 227 | # uncoverable condition right count:1 */ | ||
| 228 | | (bit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE)); | ||
| 229 | UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */ | ||
| 230 | st->armed[fd] &= (unsigned char)~bit; /* # uncoverable statement */ | ||
| 231 | } | ||
| 232 | /* No slot for the cancel: the poll stays in flight and armed | ||
| 233 | * stays SET, so a later add does not arm a second one; its | ||
| 234 | * completion finds the interest gone (mask) and is dropped | ||
| 235 | * there, which clears armed. */ | ||
| 236 | } | ||
| 237 | } | ||
| 238 | return 0; /* # uncoverable statement */ | ||
| 239 | } | ||
| 240 | |||
| 241 | #ifdef HM_HAVE_URING_RECV | ||
| 242 | /* Ask for one read into `buf`, delivered later as HM_EV_RECV. | ||
| 243 | * | ||
| 244 | * REFUSES WHILE ONE IS ALREADY IN FLIGHT ON THIS FD, and that refusal is | ||
| 245 | * load-bearing rather than defensive. The token lives in a per-fd slot, so a | ||
| 246 | * second submission would overwrite the first one's token - and then the | ||
| 247 | * FIRST completion, belonging to a connection that has already gone, would | ||
| 248 | * arrive wearing the SECOND connection's identity, pass the core's generation | ||
| 249 | * check, and have its byte count added to a buffer it never wrote into. The | ||
| 250 | * core reads -1 as "do it yourself" and falls back to a readiness read, which | ||
| 251 | * is always correct, just not as cheap. */ | ||
| 252 | /* | ||
| 253 | # uncoverable statement | ||
| 254 | # uncoverable subroutine */ | ||
| 255 | static int ur_submit_recv(hm_backend *be, int fd, void *buf, size_t len, | ||
| 256 | uint64_t token) { | ||
| 257 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 258 | struct io_uring_sqe *sqe; | ||
| 259 | /* | ||
| 260 | # uncoverable statement | ||
| 261 | # uncoverable branch true | ||
| 262 | # uncoverable branch false | ||
| 263 | # uncoverable condition left count:1 | ||
| 264 | # uncoverable condition right count:1 | ||
| 265 | # uncoverable condition left count:2 | ||
| 266 | # uncoverable condition right count:2 | ||
| 267 | # uncoverable condition left count:3 | ||
| 268 | # uncoverable condition right count:3 */ | ||
| 269 | if (fd < 0 || fd >= HM_MAXFD || len == 0) return -1; | ||
| 270 | /* | ||
| 271 | # uncoverable statement | ||
| 272 | # uncoverable branch true | ||
| 273 | # uncoverable branch false | ||
| 274 | # uncoverable condition left count:1 | ||
| 275 | # uncoverable condition right count:1 */ | ||
| 276 | if (st->recv_armed[fd]) return -1; | ||
| 277 | sqe = ur_sqe(st); /* # uncoverable statement */ | ||
| 278 | /* | ||
| 279 | # uncoverable statement | ||
| 280 | # uncoverable branch true | ||
| 281 | # uncoverable branch false | ||
| 282 | # uncoverable condition left count:1 | ||
| 283 | # uncoverable condition right count:1 */ | ||
| 284 | if (!sqe) return -1; | ||
| 285 | io_uring_prep_recv(sqe, fd, buf, len, 0); /* # uncoverable statement */ | ||
| 286 | UR_SET_DATA(sqe, ((__u64)(unsigned)fd << 3) | UR_TAG_RECV); /* # uncoverable statement */ | ||
| 287 | st->recv_token[fd] = token; /* # uncoverable statement */ | ||
| 288 | st->recv_armed[fd] = 1; /* # uncoverable statement */ | ||
| 289 | return 0; /* # uncoverable statement */ | ||
| 290 | } | ||
| 291 | |||
| 292 | /* Best-effort cancel at close. | ||
| 293 | * | ||
| 294 | * recv_armed is deliberately NOT cleared here. The completion still lands - | ||
| 295 | * cancelled or not - and it is the only moment at which the buffer the kernel | ||
| 296 | * was given can be known to be free again. Clearing the flag early would let | ||
| 297 | * the next connection on this fd number arm a second recv while the first is | ||
| 298 | * still outstanding, which is exactly the overwrite ur_submit_recv refuses. */ | ||
| 299 | /* | ||
| 300 | # uncoverable statement | ||
| 301 | # uncoverable subroutine */ | ||
| 302 | static void ur_cancel_recv(hm_backend *be, int fd) { | ||
| 303 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 304 | struct io_uring_sqe *sqe; | ||
| 305 | /* | ||
| 306 | # uncoverable statement | ||
| 307 | # uncoverable branch true | ||
| 308 | # uncoverable branch false | ||
| 309 | # uncoverable condition left count:1 | ||
| 310 | # uncoverable condition right count:1 | ||
| 311 | # uncoverable condition left count:2 | ||
| 312 | # uncoverable condition right count:2 | ||
| 313 | # uncoverable condition left count:3 | ||
| 314 | # uncoverable condition right count:3 */ | ||
| 315 | if (fd < 0 || fd >= HM_MAXFD || !st->recv_armed[fd]) return; | ||
| 316 | sqe = ur_sqe(st); /* # uncoverable statement */ | ||
| 317 | /* | ||
| 318 | # uncoverable statement | ||
| 319 | # uncoverable branch true | ||
| 320 | # uncoverable branch false | ||
| 321 | # uncoverable condition left count:1 | ||
| 322 | # uncoverable condition right count:1 */ | ||
| 323 | if (!sqe) return; | ||
| 324 | io_uring_prep_cancel(sqe, /* # uncoverable statement */ | ||
| 325 | (void *)(uintptr_t)(((__u64)(unsigned)fd << 3) | UR_TAG_RECV), 0); /* # uncoverable statement */ | ||
| 326 | UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */ | ||
| 327 | } | ||
| 328 | #endif /* HM_HAVE_URING_RECV */ | ||
| 329 | |||
| 330 | /* | ||
| 331 | # uncoverable statement | ||
| 332 | # uncoverable subroutine */ | ||
| 333 | static int ur_arm_timer(hm_ur_state *st, ur_timer *t) { | ||
| 334 | struct io_uring_sqe *sqe = ur_sqe(st); /* # uncoverable statement */ | ||
| 335 | /* | ||
| 336 | # uncoverable statement | ||
| 337 | # uncoverable branch true | ||
| 338 | # uncoverable branch false | ||
| 339 | # uncoverable condition left count:1 | ||
| 340 | # uncoverable condition right count:1 */ | ||
| 341 | if (!sqe) return -1; | ||
| 342 | io_uring_prep_timeout(sqe, &t->ts, 0, 0); /* # uncoverable statement */ | ||
| 343 | UR_SET_DATA(sqe, (__u64)(uintptr_t)t); /* tag 0 (aligned) */ /* # uncoverable statement */ | ||
| 344 | return 0; /* # uncoverable statement */ | ||
| 345 | } | ||
| 346 | |||
| 347 | /* | ||
| 348 | # uncoverable statement | ||
| 349 | # uncoverable subroutine */ | ||
| 350 | static int ur_add_timer(hm_backend *be, double secs, int oneshot, void *udata) { | ||
| 351 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 352 | ur_timer *t = (ur_timer *)hm_xcalloc(1, sizeof(ur_timer)); /* # uncoverable statement */ | ||
| 353 | /* | ||
| 354 | # uncoverable statement | ||
| 355 | # uncoverable branch true | ||
| 356 | # uncoverable branch false | ||
| 357 | # uncoverable condition left count:1 | ||
| 358 | # uncoverable condition right count:1 */ | ||
| 359 | if (!t) return -1; | ||
| 360 | t->ts.tv_sec = (int64_t)secs; /* # uncoverable statement */ | ||
| 361 | t->ts.tv_nsec = (int64_t)((secs - (double)t->ts.tv_sec) * 1e9); /* # uncoverable statement */ | ||
| 362 | /* | ||
| 363 | # uncoverable statement | ||
| 364 | # uncoverable branch true | ||
| 365 | # uncoverable branch false | ||
| 366 | # uncoverable condition left count:1 | ||
| 367 | # uncoverable condition right count:1 | ||
| 368 | # uncoverable condition left count:2 | ||
| 369 | # uncoverable condition right count:2 */ | ||
| 370 | if (t->ts.tv_sec == 0 && t->ts.tv_nsec == 0) t->ts.tv_nsec = 1; | ||
| 371 | t->udata = udata; /* # uncoverable statement */ | ||
| 372 | t->interval = !oneshot; /* # uncoverable statement */ | ||
| 373 | t->next = st->timers; /* # uncoverable statement */ | ||
| 374 | /* | ||
| 375 | # uncoverable statement | ||
| 376 | # uncoverable branch true | ||
| 377 | # uncoverable branch false | ||
| 378 | # uncoverable condition left count:1 | ||
| 379 | # uncoverable condition right count:1 */ | ||
| 380 | if (st->timers) st->timers->prev = t; | ||
| 381 | st->timers = t; /* # uncoverable statement */ | ||
| 382 | /* | ||
| 383 | # uncoverable statement | ||
| 384 | # uncoverable branch true | ||
| 385 | # uncoverable branch false | ||
| 386 | # uncoverable condition left count:1 | ||
| 387 | # uncoverable condition right count:1 */ | ||
| 388 | if (ur_arm_timer(st, t) < 0) { | ||
| 389 | /* No submission slot: the caller is told, as the other backends | ||
| 390 | * tell it when a timerfd cannot be had, rather than owning a timer | ||
| 391 | * that will never fire. t is the list head, just linked. */ | ||
| 392 | st->timers = t->next; /* # uncoverable statement */ | ||
| 393 | /* | ||
| 394 | # uncoverable statement | ||
| 395 | # uncoverable branch true | ||
| 396 | # uncoverable branch false | ||
| 397 | # uncoverable condition left count:1 | ||
| 398 | # uncoverable condition right count:1 */ | ||
| 399 | if (t->next) t->next->prev = NULL; | ||
| 400 | free(t); /* # uncoverable statement */ | ||
| 401 | errno = EAGAIN; /* # uncoverable statement */ | ||
| 402 | return -1; /* # uncoverable statement */ | ||
| 403 | } | ||
| 404 | return 0; /* # uncoverable statement */ | ||
| 405 | } | ||
| 406 | |||
| 407 | /* | ||
| 408 | # uncoverable statement | ||
| 409 | # uncoverable subroutine */ | ||
| 410 | static void ur_timer_unlink(hm_ur_state *st, ur_timer *t) { | ||
| 411 | /* | ||
| 412 | # uncoverable statement | ||
| 413 | # uncoverable branch true | ||
| 414 | # uncoverable branch false | ||
| 415 | # uncoverable condition left count:1 | ||
| 416 | # uncoverable condition right count:1 */ | ||
| 417 | if (t->prev) t->prev->next = t->next; | ||
| 418 | else st->timers = t->next; /* # uncoverable statement */ | ||
| 419 | /* | ||
| 420 | # uncoverable statement | ||
| 421 | # uncoverable branch true | ||
| 422 | # uncoverable branch false | ||
| 423 | # uncoverable condition left count:1 | ||
| 424 | # uncoverable condition right count:1 */ | ||
| 425 | if (t->next) t->next->prev = t->prev; | ||
| 426 | } | ||
| 427 | |||
| 428 | /* Cancel a timer. The timeout is already in flight in the kernel and its | ||
| 429 | * user_data IS this pointer, so freeing here would leave the kernel holding a | ||
| 430 | * dangling reference: the completion still arrives, ur_wait dereferences it | ||
| 431 | * and frees it a second time. Worse, the block is the right size to be handed | ||
| 432 | * straight back to the next ur_add_timer, so the stale completion lands on a | ||
| 433 | * live timer and takes that down instead - which is why this showed up as an | ||
| 434 | * intermittent "double free or corruption" rather than a reliable one. | ||
| 435 | * | ||
| 436 | * So: ask the kernel to remove the timeout and mark the watcher dead, but | ||
| 437 | * leave it linked and allocated. Its completion (-ECANCELED, or a genuine | ||
| 438 | * fire that raced the removal) frees it in ur_wait; anything still linked at | ||
| 439 | * shutdown is freed by ur_destroy. */ | ||
| 440 | /* | ||
| 441 | # uncoverable statement | ||
| 442 | # uncoverable subroutine */ | ||
| 443 | static int ur_del_timer(hm_backend *be, void *udata) { | ||
| 444 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 445 | ur_timer *t; | ||
| 446 | /* | ||
| 447 | # uncoverable statement | ||
| 448 | # uncoverable branch true | ||
| 449 | # uncoverable branch false | ||
| 450 | # uncoverable condition left count:1 | ||
| 451 | # uncoverable condition right count:1 */ | ||
| 452 | for (t = st->timers; t; t = t->next) { | ||
| 453 | /* | ||
| 454 | # uncoverable statement | ||
| 455 | # uncoverable branch true | ||
| 456 | # uncoverable branch false | ||
| 457 | # uncoverable condition left count:1 | ||
| 458 | # uncoverable condition right count:1 | ||
| 459 | # uncoverable condition left count:2 | ||
| 460 | # uncoverable condition right count:2 */ | ||
| 461 | if (t->udata == udata && !t->cancelled) { | ||
| 462 | struct io_uring_sqe *sqe; | ||
| 463 | t->cancelled = 1; /* # uncoverable statement */ | ||
| 464 | t->interval = 0; /* never re-arm a cancelled timer */ /* # uncoverable statement */ | ||
| 465 | /* | ||
| 466 | # uncoverable statement | ||
| 467 | # uncoverable branch true | ||
| 468 | # uncoverable branch false | ||
| 469 | # uncoverable condition left count:1 | ||
| 470 | # uncoverable condition right count:1 */ | ||
| 471 | if ((sqe = ur_sqe(st)) != NULL) { | ||
| 472 | io_uring_prep_timeout_remove(sqe, (__u64)(uintptr_t)t, 0); /* # uncoverable statement */ | ||
| 473 | UR_SET_DATA(sqe, UR_TAG_IGNORE); /* # uncoverable statement */ | ||
| 474 | } | ||
| 475 | return 0; /* # uncoverable statement */ | ||
| 476 | } | ||
| 477 | } | ||
| 478 | return -1; /* # uncoverable statement */ | ||
| 479 | } | ||
| 480 | |||
| 481 | /* | ||
| 482 | # uncoverable statement | ||
| 483 | # uncoverable subroutine */ | ||
| 484 | static int ur_add_signal(hm_backend *be, int signo) { | ||
| 485 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 486 | sigaddset(&st->sigs, signo); /* # uncoverable statement */ | ||
| 487 | /* | ||
| 488 | # uncoverable statement | ||
| 489 | # uncoverable branch true | ||
| 490 | # uncoverable branch false | ||
| 491 | # uncoverable condition left count:1 | ||
| 492 | # uncoverable condition right count:1 */ | ||
| 493 | if (sigprocmask(SIG_BLOCK, &st->sigs, NULL) < 0) return -1; | ||
| 494 | { | ||
| 495 | int sfd = signalfd(st->sfd, &st->sigs, SFD_NONBLOCK | SFD_CLOEXEC); /* # uncoverable statement */ | ||
| 496 | /* | ||
| 497 | # uncoverable statement | ||
| 498 | # uncoverable branch true | ||
| 499 | # uncoverable branch false | ||
| 500 | # uncoverable condition left count:1 | ||
| 501 | # uncoverable condition right count:1 */ | ||
| 502 | if (sfd < 0) return -1; | ||
| 503 | /* | ||
| 504 | # uncoverable statement | ||
| 505 | # uncoverable branch true | ||
| 506 | # uncoverable branch false | ||
| 507 | # uncoverable condition left count:1 | ||
| 508 | # uncoverable condition right count:1 */ | ||
| 509 | if (st->sfd < 0) { | ||
| 510 | st->sfd = sfd; /* # uncoverable statement */ | ||
| 511 | st->mask[sfd] |= HM_EV_READ; /* persistent poll on the ring */ /* # uncoverable statement */ | ||
| 512 | ur_arm_poll(st, sfd, HM_EV_READ); /* # uncoverable statement */ | ||
| 513 | } | ||
| 514 | } | ||
| 515 | return 0; /* # uncoverable statement */ | ||
| 516 | } | ||
| 517 | |||
| 518 | /* | ||
| 519 | # uncoverable statement | ||
| 520 | # uncoverable subroutine */ | ||
| 521 | static int ur_wait(hm_backend *be, hm_event *out, int max, double timeout) { | ||
| 522 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 523 | struct io_uring_cqe *cqes[HM_MAXEV]; | ||
| 524 | struct io_uring_cqe *cqe; | ||
| 525 | unsigned want, n, i; | ||
| 526 | int nev = 0, ret; /* # uncoverable statement */ | ||
| 527 | |||
| 528 | /* | ||
| 529 | # uncoverable statement | ||
| 530 | # uncoverable branch true | ||
| 531 | # uncoverable branch false | ||
| 532 | # uncoverable condition left count:1 | ||
| 533 | # uncoverable condition right count:1 */ | ||
| 534 | if (st->sq_dropped) { | ||
| 535 | /* A re-arm found the ring full: every interest with no poll in | ||
| 536 | * flight is armed again now, with the ring drained by the submit | ||
| 537 | * below between attempts. Rare, so the full-table walk is fine. */ | ||
| 538 | int fd; | ||
| 539 | st->sq_dropped = 0; /* # uncoverable statement */ | ||
| 540 | /* | ||
| 541 | # uncoverable statement | ||
| 542 | # uncoverable branch true | ||
| 543 | # uncoverable branch false | ||
| 544 | # uncoverable condition left count:1 | ||
| 545 | # uncoverable condition right count:1 */ | ||
| 546 | for (fd = 0; fd < HM_MAXFD; fd++) { | ||
| 547 | unsigned char need = st->mask[fd] & (unsigned char)~st->armed[fd]; /* # uncoverable statement */ | ||
| 548 | /* | ||
| 549 | # uncoverable statement | ||
| 550 | # uncoverable branch true | ||
| 551 | # uncoverable branch false | ||
| 552 | # uncoverable condition left count:1 | ||
| 553 | # uncoverable condition right count:1 */ | ||
| 554 | if (!need) continue; | ||
| 555 | /* | ||
| 556 | # uncoverable statement | ||
| 557 | # uncoverable branch true | ||
| 558 | # uncoverable branch false | ||
| 559 | # uncoverable condition left count:1 | ||
| 560 | # uncoverable condition right count:1 */ | ||
| 561 | if (need & HM_EV_READ) ur_arm_poll(st, fd, HM_EV_READ); | ||
| 562 | /* | ||
| 563 | # uncoverable statement | ||
| 564 | # uncoverable branch true | ||
| 565 | # uncoverable branch false | ||
| 566 | # uncoverable condition left count:1 | ||
| 567 | # uncoverable condition right count:1 */ | ||
| 568 | if (need & HM_EV_WRITE) ur_arm_poll(st, fd, HM_EV_WRITE); | ||
| 569 | } | ||
| 570 | } | ||
| 571 | io_uring_submit(&st->ring); /* flush re-arms, cancels, timers */ /* # uncoverable statement */ | ||
| 572 | |||
| 573 | /* | ||
| 574 | # uncoverable statement | ||
| 575 | # uncoverable branch true | ||
| 576 | # uncoverable branch false | ||
| 577 | # uncoverable condition left count:1 | ||
| 578 | # uncoverable condition right count:1 */ | ||
| 579 | if (timeout < 0) { | ||
| 580 | ret = io_uring_wait_cqe(&st->ring, &cqe); /* # uncoverable statement */ | ||
| 581 | /* | ||
| 582 | # uncoverable statement | ||
| 583 | # uncoverable branch true | ||
| 584 | # uncoverable branch false | ||
| 585 | # uncoverable condition left count:1 | ||
| 586 | # uncoverable condition right count:1 */ | ||
| 587 | } else if (timeout == 0) { | ||
| 588 | ret = io_uring_peek_cqe(&st->ring, &cqe); /* # uncoverable statement */ | ||
| 589 | /* | ||
| 590 | # uncoverable statement | ||
| 591 | # uncoverable branch true | ||
| 592 | # uncoverable branch false | ||
| 593 | # uncoverable condition left count:1 | ||
| 594 | # uncoverable condition right count:1 */ | ||
| 595 | if (ret == -EAGAIN) return 0; | ||
| 596 | } else { | ||
| 597 | struct __kernel_timespec ts; | ||
| 598 | ts.tv_sec = (int64_t)timeout; /* # uncoverable statement */ | ||
| 599 | ts.tv_nsec = (int64_t)((timeout - (double)ts.tv_sec) * 1e9); /* # uncoverable statement */ | ||
| 600 | ret = io_uring_wait_cqe_timeout(&st->ring, &cqe, &ts); /* # uncoverable statement */ | ||
| 601 | /* | ||
| 602 | # uncoverable statement | ||
| 603 | # uncoverable branch true | ||
| 604 | # uncoverable branch false | ||
| 605 | # uncoverable condition left count:1 | ||
| 606 | # uncoverable condition right count:1 */ | ||
| 607 | if (ret == -ETIME) return 0; | ||
| 608 | } | ||
| 609 | /* | ||
| 610 | # uncoverable statement | ||
| 611 | # uncoverable branch true | ||
| 612 | # uncoverable branch false | ||
| 613 | # uncoverable condition left count:1 | ||
| 614 | # uncoverable condition right count:1 | ||
| 615 | # uncoverable condition left count:2 | ||
| 616 | # uncoverable condition right count:2 */ | ||
| 617 | if (ret < 0) return (ret == -EINTR) ? 0 : -1; | ||
| 618 | |||
| 619 | /* headroom so a single signalfd completion (multiple queued signals) | ||
| 620 | * can't overflow the caller's event buffer */ | ||
| 621 | /* | ||
| 622 | # uncoverable statement | ||
| 623 | # uncoverable condition left count:1 | ||
| 624 | # uncoverable condition right count:1 */ | ||
| 625 | want = (max > 8) ? (unsigned)(max - 8) : 1u; | ||
| 626 | /* | ||
| 627 | # uncoverable statement | ||
| 628 | # uncoverable branch true | ||
| 629 | # uncoverable branch false | ||
| 630 | # uncoverable condition left count:1 | ||
| 631 | # uncoverable condition right count:1 */ | ||
| 632 | if (want > HM_MAXEV) want = HM_MAXEV; | ||
| 633 | n = io_uring_peek_batch_cqe(&st->ring, cqes, want); /* # uncoverable statement */ | ||
| 634 | |||
| 635 | /* | ||
| 636 | # uncoverable statement | ||
| 637 | # uncoverable branch true | ||
| 638 | # uncoverable branch false | ||
| 639 | # uncoverable condition left count:1 | ||
| 640 | # uncoverable condition right count:1 */ | ||
| 641 | for (i = 0; i < n; i++) { | ||
| 642 | __u64 data = cqes[i]->user_data; /* # uncoverable statement */ | ||
| 643 | int res = cqes[i]->res; /* # uncoverable statement */ | ||
| 644 | __u64 tag = data & UR_TAG_MASK; /* # uncoverable statement */ | ||
| 645 | |||
| 646 | /* | ||
| 647 | # uncoverable statement | ||
| 648 | # uncoverable branch true | ||
| 649 | # uncoverable branch false | ||
| 650 | # uncoverable condition left count:1 | ||
| 651 | # uncoverable condition right count:1 */ | ||
| 652 | if (data == LIBURING_UDATA_TIMEOUT) continue; /* liburing internal */ | ||
| 653 | /* | ||
| 654 | # uncoverable statement | ||
| 655 | # uncoverable branch true | ||
| 656 | # uncoverable branch false | ||
| 657 | # uncoverable condition left count:1 | ||
| 658 | # uncoverable condition right count:1 */ | ||
| 659 | if (tag == UR_TAG_IGNORE) continue; /* poll-remove ack */ | ||
| 660 | |||
| 661 | #ifdef HM_HAVE_URING_RECV | ||
| 662 | /* | ||
| 663 | # uncoverable statement | ||
| 664 | # uncoverable branch true | ||
| 665 | # uncoverable branch false | ||
| 666 | # uncoverable condition left count:1 | ||
| 667 | # uncoverable condition right count:1 */ | ||
| 668 | if (tag == UR_TAG_RECV) { | ||
| 669 | int fd = (int)(data >> 3); /* # uncoverable statement */ | ||
| 670 | st->recv_armed[fd] = 0; /* the buffer is the kernel's no longer */ /* # uncoverable statement */ | ||
| 671 | /* | ||
| 672 | # uncoverable statement | ||
| 673 | # uncoverable branch true | ||
| 674 | # uncoverable branch false | ||
| 675 | # uncoverable condition left count:1 | ||
| 676 | # uncoverable condition right count:1 */ | ||
| 677 | if (nev >= max) break; | ||
| 678 | /* Delivered even when cancelled or failed: a completion is the | ||
| 679 | * only signal that the buffer handed to the kernel is free, and | ||
| 680 | * the core has one to release whether the read worked or not. */ | ||
| 681 | out[nev].kind = HM_EV_RECV; /* # uncoverable statement */ | ||
| 682 | out[nev].fd = fd; /* # uncoverable statement */ | ||
| 683 | out[nev].res = res; /* # uncoverable statement */ | ||
| 684 | out[nev].token = st->recv_token[fd]; /* # uncoverable statement */ | ||
| 685 | out[nev].udata = NULL; /* # uncoverable statement */ | ||
| 686 | nev++; /* # uncoverable statement */ | ||
| 687 | continue; /* # uncoverable statement */ | ||
| 688 | } | ||
| 689 | #endif | ||
| 690 | |||
| 691 | /* | ||
| 692 | # uncoverable statement | ||
| 693 | # uncoverable branch true | ||
| 694 | # uncoverable branch false | ||
| 695 | # uncoverable condition left count:1 | ||
| 696 | # uncoverable condition right count:1 | ||
| 697 | # uncoverable condition left count:2 | ||
| 698 | # uncoverable condition right count:2 */ | ||
| 699 | if (tag == UR_TAG_READ || tag == UR_TAG_WRITE) { | ||
| 700 | int fd = (int)(data >> 3); /* # uncoverable statement */ | ||
| 701 | /* | ||
| 702 | # uncoverable statement | ||
| 703 | # uncoverable condition left count:1 | ||
| 704 | # uncoverable condition right count:1 */ | ||
| 705 | int bit = (tag == UR_TAG_READ) ? HM_EV_READ : HM_EV_WRITE; | ||
| 706 | st->armed[fd] &= (unsigned char)~bit; /* # uncoverable statement */ | ||
| 707 | /* | ||
| 708 | # uncoverable statement | ||
| 709 | # uncoverable branch true | ||
| 710 | # uncoverable branch false | ||
| 711 | # uncoverable condition left count:1 | ||
| 712 | # uncoverable condition right count:1 | ||
| 713 | # uncoverable condition left count:2 | ||
| 714 | # uncoverable condition right count:2 */ | ||
| 715 | if (res == -ECANCELED || !(st->mask[fd] & bit)) continue; | ||
| 716 | |||
| 717 | /* | ||
| 718 | # uncoverable statement | ||
| 719 | # uncoverable branch true | ||
| 720 | # uncoverable branch false | ||
| 721 | # uncoverable condition left count:1 | ||
| 722 | # uncoverable condition right count:1 | ||
| 723 | # uncoverable condition left count:2 | ||
| 724 | # uncoverable condition right count:2 */ | ||
| 725 | if (st->sfd >= 0 && fd == st->sfd) { | ||
| 726 | struct signalfd_siginfo si; | ||
| 727 | /* | ||
| 728 | # uncoverable statement | ||
| 729 | # uncoverable branch true | ||
| 730 | # uncoverable branch false */ | ||
| 731 | while (nev < max | ||
| 732 | /* | ||
| 733 | # uncoverable statement | ||
| 734 | # uncoverable condition left count:1 | ||
| 735 | # uncoverable condition right count:1 | ||
| 736 | # uncoverable condition left count:2 | ||
| 737 | # uncoverable condition right count:2 */ | ||
| 738 | && read(st->sfd, &si, sizeof(si)) == (ssize_t)sizeof(si)) { | ||
| 739 | out[nev].kind = HM_EV_SIGNAL; /* # uncoverable statement */ | ||
| 740 | out[nev].fd = (int)si.ssi_signo; /* # uncoverable statement */ | ||
| 741 | out[nev].udata = NULL; /* # uncoverable statement */ | ||
| 742 | nev++; /* # uncoverable statement */ | ||
| 743 | } | ||
| 744 | ur_arm_poll(st, st->sfd, HM_EV_READ); /* persistent */ /* # uncoverable statement */ | ||
| 745 | continue; /* # uncoverable statement */ | ||
| 746 | } | ||
| 747 | |||
| 748 | out[nev].kind = bit; /* # uncoverable statement */ | ||
| 749 | out[nev].fd = fd; /* # uncoverable statement */ | ||
| 750 | out[nev].udata = NULL; /* # uncoverable statement */ | ||
| 751 | nev++; /* # uncoverable statement */ | ||
| 752 | /* | ||
| 753 | # uncoverable statement | ||
| 754 | # uncoverable branch true | ||
| 755 | # uncoverable branch false | ||
| 756 | # uncoverable condition left count:1 | ||
| 757 | # uncoverable condition right count:1 */ | ||
| 758 | if (st->once[fd] & bit) { | ||
| 759 | st->mask[fd] &= (unsigned char)~bit; /* # uncoverable statement */ | ||
| 760 | st->once[fd] &= (unsigned char)~bit; /* # uncoverable statement */ | ||
| 761 | } else { | ||
| 762 | ur_arm_poll(st, fd, bit); /* persistent: re-arm */ /* # uncoverable statement */ | ||
| 763 | } | ||
| 764 | continue; /* # uncoverable statement */ | ||
| 765 | } | ||
| 766 | |||
| 767 | { /* timer: user_data is the ur_timer pointer (tag 0) */ | ||
| 768 | ur_timer *t = (ur_timer *)(uintptr_t)data; /* # uncoverable statement */ | ||
| 769 | /* | ||
| 770 | # uncoverable statement | ||
| 771 | # uncoverable branch true | ||
| 772 | # uncoverable branch false | ||
| 773 | # uncoverable condition left count:1 | ||
| 774 | # uncoverable condition right count:1 */ | ||
| 775 | if (!t) continue; | ||
| 776 | /* A cancelled timer's completion still arrives; it owns the | ||
| 777 | * struct, and it must not be reported to the loop. -ECANCELED | ||
| 778 | * covers the removal succeeding, t->cancelled the case where the | ||
| 779 | * timeout fired before the removal reached the kernel. */ | ||
| 780 | /* | ||
| 781 | # uncoverable statement | ||
| 782 | # uncoverable branch true | ||
| 783 | # uncoverable branch false | ||
| 784 | # uncoverable condition left count:1 | ||
| 785 | # uncoverable condition right count:1 | ||
| 786 | # uncoverable condition left count:2 | ||
| 787 | # uncoverable condition right count:2 */ | ||
| 788 | if (t->cancelled || res == -ECANCELED) { | ||
| 789 | ur_timer_unlink(st, t); /* # uncoverable statement */ | ||
| 790 | free(t); /* # uncoverable statement */ | ||
| 791 | continue; /* # uncoverable statement */ | ||
| 792 | } | ||
| 793 | out[nev].kind = HM_EV_TIMER; /* # uncoverable statement */ | ||
| 794 | out[nev].fd = -1; /* # uncoverable statement */ | ||
| 795 | out[nev].udata = t->udata; /* # uncoverable statement */ | ||
| 796 | nev++; /* # uncoverable statement */ | ||
| 797 | /* | ||
| 798 | # uncoverable statement | ||
| 799 | # uncoverable branch true | ||
| 800 | # uncoverable branch false | ||
| 801 | # uncoverable condition left count:1 | ||
| 802 | # uncoverable condition right count:1 */ | ||
| 803 | if (t->interval) { | ||
| 804 | ur_arm_timer(st, t); /* # uncoverable statement */ | ||
| 805 | } else { | ||
| 806 | ur_timer_unlink(st, t); /* # uncoverable statement */ | ||
| 807 | free(t); /* # uncoverable statement */ | ||
| 808 | } | ||
| 809 | } | ||
| 810 | } | ||
| 811 | io_uring_cq_advance(&st->ring, n); /* # uncoverable statement */ | ||
| 812 | return nev; /* # uncoverable statement */ | ||
| 813 | } | ||
| 814 | |||
| 815 | /* | ||
| 816 | # uncoverable statement | ||
| 817 | # uncoverable subroutine */ | ||
| 818 | static void ur_destroy(hm_backend *be) { | ||
| 819 | hm_ur_state *st = (hm_ur_state *)be->state; /* # uncoverable statement */ | ||
| 820 | /* | ||
| 821 | # uncoverable statement | ||
| 822 | # uncoverable branch true | ||
| 823 | # uncoverable branch false | ||
| 824 | # uncoverable condition left count:1 | ||
| 825 | # uncoverable condition right count:1 */ | ||
| 826 | if (st) { | ||
| 827 | /* | ||
| 828 | # uncoverable statement | ||
| 829 | # uncoverable branch true | ||
| 830 | # uncoverable branch false | ||
| 831 | # uncoverable condition left count:1 | ||
| 832 | # uncoverable condition right count:1 */ | ||
| 833 | while (st->timers) { | ||
| 834 | ur_timer *t = st->timers; /* # uncoverable statement */ | ||
| 835 | st->timers = t->next; /* # uncoverable statement */ | ||
| 836 | free(t); /* # uncoverable statement */ | ||
| 837 | } | ||
| 838 | /* be->foreign: inherited across a fork. The signalfd and the ring's | ||
| 839 | * own descriptor are the parent's; io_uring_queue_exit would close | ||
| 840 | * the ring fd and unmap rings this process never set up. */ | ||
| 841 | /* | ||
| 842 | # uncoverable statement | ||
| 843 | # uncoverable branch true | ||
| 844 | # uncoverable branch false | ||
| 845 | # uncoverable condition left count:1 | ||
| 846 | # uncoverable condition right count:1 */ | ||
| 847 | if (!be->foreign) { | ||
| 848 | /* | ||
| 849 | # uncoverable statement | ||
| 850 | # uncoverable branch true | ||
| 851 | # uncoverable branch false | ||
| 852 | # uncoverable condition left count:1 | ||
| 853 | # uncoverable condition right count:1 */ | ||
| 854 | if (st->sfd >= 0) close(st->sfd); | ||
| 855 | io_uring_queue_exit(&st->ring); /* # uncoverable statement */ | ||
| 856 | } | ||
| 857 | free(st); /* # uncoverable statement */ | ||
| 858 | } | ||
| 859 | free(be); /* # uncoverable statement */ | ||
| 860 | } | ||
| 861 | |||
| 862 | hm_backend *hm_backend_iouring_new(void) { | ||
| 863 | hm_backend *be = (hm_backend *)hm_xcalloc(1, sizeof(hm_backend)); | ||
| 864 | hm_ur_state *st = (hm_ur_state *)hm_xcalloc(1, sizeof(hm_ur_state)); | ||
| 865 | /* | ||
| 866 | # uncoverable branch true | ||
| 867 | # uncoverable condition left count:1 | ||
| 868 | # uncoverable condition left count:2 */ | ||
| 869 | if (!be || !st) { free(be); free(st); return NULL; } | ||
| 870 | st->sfd = -1; | ||
| 871 | sigemptyset(&st->sigs); | ||
| 872 | /* | ||
| 873 | # uncoverable branch false | ||
| 874 | # uncoverable condition right count:1 */ | ||
| 875 | if (io_uring_queue_init(1024, &st->ring, 0) < 0) { | ||
| 876 | free(be); free(st); return NULL; | ||
| 877 | } | ||
| 878 | be->name = "io_uring"; /* # uncoverable statement */ | ||
| 879 | be->state = st; /* # uncoverable statement */ | ||
| 880 | be->add_io = ur_add_io; /* # uncoverable statement */ | ||
| 881 | be->modify_io = ur_modify_io; /* # uncoverable statement */ | ||
| 882 | be->remove_io = ur_remove_io; /* # uncoverable statement */ | ||
| 883 | be->add_timer = ur_add_timer; /* # uncoverable statement */ | ||
| 884 | be->del_timer = ur_del_timer; /* # uncoverable statement */ | ||
| 885 | be->add_signal = ur_add_signal; /* # uncoverable statement */ | ||
| 886 | be->wait = ur_wait; /* # uncoverable statement */ | ||
| 887 | be->destroy = ur_destroy; /* # uncoverable statement */ | ||
| 888 | #ifdef HM_HAVE_URING_RECV | ||
| 889 | be->submit_recv = ur_submit_recv; /* # uncoverable statement */ | ||
| 890 | be->cancel_recv = ur_cancel_recv; /* # uncoverable statement */ | ||
| 891 | #endif | ||
| 892 | return be; /* # uncoverable statement */ | ||
| 893 | } | ||
| 894 | |||
| 895 | int hm_backend_iouring_available(void) { | ||
| 896 | /* liburing is linked; confirm the kernel actually supports io_uring | ||
| 897 | * (not the case under, e.g., default Docker seccomp) */ | ||
| 898 | static int avail = -1; | ||
| 899 | /* | ||
| 900 | # uncoverable branch false | ||
| 901 | # uncoverable condition right count:1 */ | ||
| 902 | if (avail < 0) { | ||
| 903 | struct io_uring ring; | ||
| 904 | /* | ||
| 905 | # uncoverable branch true | ||
| 906 | # uncoverable condition left count:1 */ | ||
| 907 | if (io_uring_queue_init(2, &ring, 0) == 0) { | ||
| 908 | io_uring_queue_exit(&ring); /* # uncoverable statement */ | ||
| 909 | avail = 1; /* # uncoverable statement */ | ||
| 910 | } else { | ||
| 911 | avail = 0; | ||
| 912 | } | ||
| 913 | } | ||
| 914 | return avail; | ||
| 915 | } | ||
| 916 | |||
| 917 | #else /* !HM_IOURING_ACTIVE */ | ||
| 918 | |||
| 919 | 151 | hm_backend *hm_backend_iouring_new(void) { return 0; } | |
| 920 | 5 | int hm_backend_iouring_available(void) { return 0; } | |
| 921 | |||
| 922 | #endif |