| 1 | /* Hyperman::Event::IOUring - io_uring readiness backend (Linux), built on | ||
| 2 | * liburing. Enabled by Makefile.PL when liburing is found (pkg-config or a | ||
| 3 | * -luring compile probe), which defines HM_HAVE_LIBURING and links -luring; | ||
| 4 | * otherwise this compiles to graceful "unavailable" stubs. | ||
| 5 | * | ||
| 6 | * This drives the ring in *poll mode*: IORING_OP_POLL_ADD readiness | ||
| 7 | * completions mapped onto the hm_backend vtable (persistent watchers are | ||
| 8 | * re-armed on delivery), timers as IORING_OP_TIMEOUT, and shutdown signals | ||
| 9 | * via a signalfd polled on the ring. Completion-based read/write submission | ||
| 10 | * (the io_uring headline) needs the connection state machine reshaped and | ||
| 11 | * comes later. | ||
| 12 | * | ||
| 13 | * Opt-in for now: HYPERMAN_BACKEND=io_uring or Hyperman::Loop->new('io_uring'); | ||
| 14 | * auto-selection stays kqueue > epoll > poll until this is benchmarked. | ||
| 15 | */ | ||
| 16 | |||
| 17 | #include "ft_backend.h" | ||
| 18 | |||
| 19 | #if defined(__linux__) && defined(HM_HAVE_LIBURING) | ||
| 20 | #define HM_IOURING_ACTIVE 1 | ||
| 21 | #endif | ||
| 22 | |||
| 23 | #ifdef HM_IOURING_ACTIVE | ||
| 24 | |||
| 25 | #include <liburing.h> | ||
| 26 | #include <sys/signalfd.h> | ||
| 27 | #include <signal.h> | ||
| 28 | #include <string.h> | ||
| 29 | #include <stdlib.h> | ||
| 30 | #include <unistd.h> | ||
| 31 | #include <errno.h> | ||
| 32 | #include <poll.h> | ||
| 33 | |||
| 34 | /* user_data tagging (low 3 bits). Timer completions carry a pointer to | ||
| 35 | * their ur_timer, which is 8-aligned so its tag is 0. liburing's own | ||
| 36 | * internal wait-timeout uses LIBURING_UDATA_TIMEOUT and is skipped. */ | ||
| 37 | #define UR_TAG_MASK 7ULL | ||
| 38 | #define UR_TAG_TIMER 0ULL /* aligned ur_timer* */ | ||
| 39 | #define UR_TAG_READ 1ULL | ||
| 40 | #define UR_TAG_WRITE 2ULL | ||
| 41 | #define UR_TAG_IGNORE 5ULL /* poll-remove completions: drop */ | ||
| 42 | |||
| 43 | /* io_uring_sqe_set_data64() was only added in liburing 2.2; assign the | ||
| 44 | * user_data field directly (exactly what that helper does) so we build and | ||
| 45 | * load against every liburing version. */ | ||
| 46 | #define UR_SET_DATA(sqe, val) ((sqe)->user_data = (__u64)(val)) | ||
| 47 | |||
| 48 | typedef struct ur_timer { | ||
| 49 | struct __kernel_timespec ts; | ||
| 50 | void *udata; | ||
| 51 | int interval; /* re-arm on fire? */ | ||
| 52 | int cancelled; /* freed when its completion lands, not before */ | ||
| 53 | struct ur_timer *prev, *next; | ||
| 54 | } ur_timer; | ||
| 55 | |||
| 56 | typedef struct { | ||
| 57 | struct io_uring ring; | ||
| 58 | unsigned char mask[HM_MAXFD]; /* wanted HM_EV bits */ | ||
| 59 | unsigned char once[HM_MAXFD]; /* oneshot HM_EV bits */ | ||
| 60 | unsigned char armed[HM_MAXFD]; /* HM_EV bits with in-flight sqe */ | ||
| 61 | ur_timer *timers; /* doubly-linked, for cleanup */ | ||
| 62 | int sfd; /* signalfd, -1 until first sig */ | ||
| 63 | sigset_t sigs; | ||
| 64 | } hm_ur_state; | ||
| 65 | |||
| 66 | /* Get an sqe, flushing the ring if the submission queue is momentarily full. */ | ||
| 67 | static struct io_uring_sqe *ur_sqe(hm_ur_state *st) { | ||
| 68 | struct io_uring_sqe *sqe = io_uring_get_sqe(&st->ring); | ||
| 69 | if (!sqe) { | ||
| 70 | io_uring_submit(&st->ring); | ||
| 71 | sqe = io_uring_get_sqe(&st->ring); | ||
| 72 | } | ||
| 73 | return sqe; | ||
| 74 | } | ||
| 75 | |||
| 76 | static void ur_arm_poll(hm_ur_state *st, int fd, int evbit) { | ||
| 77 | struct io_uring_sqe *sqe = ur_sqe(st); | ||
| 78 | unsigned mask = (evbit == HM_EV_READ) ? (POLLIN | POLLHUP | POLLERR) | ||
| 79 | : (POLLOUT | POLLHUP | POLLERR); | ||
| 80 | if (!sqe) return; | ||
| 81 | io_uring_prep_poll_add(sqe, fd, mask); | ||
| 82 | UR_SET_DATA(sqe, | ||
| 83 | ((__u64)(unsigned)fd << 3) | ||
| 84 | | (evbit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE)); | ||
| 85 | st->armed[fd] |= (unsigned char)evbit; | ||
| 86 | } | ||
| 87 | |||
| 88 | static int ur_add_io(hm_backend *be, int fd, int mask, int oneshot) { | ||
| 89 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 90 | int bit; | ||
| 91 | if (fd < 0 || fd >= HM_MAXFD) return -1; | ||
| 92 | st->mask[fd] |= (unsigned char)mask; | ||
| 93 | if (oneshot) st->once[fd] |= (unsigned char)mask; | ||
| 94 | else st->once[fd] &= (unsigned char)~mask; | ||
| 95 | for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) | ||
| 96 | if ((mask & bit) && !(st->armed[fd] & bit)) | ||
| 97 | ur_arm_poll(st, fd, bit); | ||
| 98 | return 0; | ||
| 99 | } | ||
| 100 | |||
| 101 | static int ur_modify_io(hm_backend *be, int fd, int mask, int oneshot) { | ||
| 102 | return ur_add_io(be, fd, mask, oneshot); | ||
| 103 | } | ||
| 104 | |||
| 105 | static int ur_remove_io(hm_backend *be, int fd, int mask) { | ||
| 106 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 107 | int bit; | ||
| 108 | if (fd < 0 || fd >= HM_MAXFD) return -1; | ||
| 109 | st->mask[fd] &= (unsigned char)~mask; | ||
| 110 | st->once[fd] &= (unsigned char)~mask; | ||
| 111 | /* cancel any in-flight poll for the dropped interest, so a closed+reused | ||
| 112 | * fd can't be woken by a stale completion */ | ||
| 113 | for (bit = HM_EV_READ; bit <= HM_EV_WRITE; bit <<= 1) { | ||
| 114 | if ((mask & bit) && (st->armed[fd] & bit)) { | ||
| 115 | struct io_uring_sqe *sqe = ur_sqe(st); | ||
| 116 | if (sqe) { | ||
| 117 | io_uring_prep_poll_remove(sqe, | ||
| 118 | ((__u64)(unsigned)fd << 3) | ||
| 119 | | (bit == HM_EV_READ ? UR_TAG_READ : UR_TAG_WRITE)); | ||
| 120 | UR_SET_DATA(sqe, UR_TAG_IGNORE); | ||
| 121 | } | ||
| 122 | st->armed[fd] &= (unsigned char)~bit; | ||
| 123 | } | ||
| 124 | } | ||
| 125 | return 0; | ||
| 126 | } | ||
| 127 | |||
| 128 | static void ur_arm_timer(hm_ur_state *st, ur_timer *t) { | ||
| 129 | struct io_uring_sqe *sqe = ur_sqe(st); | ||
| 130 | if (!sqe) return; | ||
| 131 | io_uring_prep_timeout(sqe, &t->ts, 0, 0); | ||
| 132 | UR_SET_DATA(sqe, (__u64)(uintptr_t)t); /* tag 0 (aligned) */ | ||
| 133 | } | ||
| 134 | |||
| 135 | static int ur_add_timer(hm_backend *be, double secs, int oneshot, void *udata) { | ||
| 136 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 137 | ur_timer *t = (ur_timer *)calloc(1, sizeof(ur_timer)); | ||
| 138 | if (!t) return -1; | ||
| 139 | t->ts.tv_sec = (int64_t)secs; | ||
| 140 | t->ts.tv_nsec = (int64_t)((secs - (double)t->ts.tv_sec) * 1e9); | ||
| 141 | if (t->ts.tv_sec == 0 && t->ts.tv_nsec == 0) t->ts.tv_nsec = 1; | ||
| 142 | t->udata = udata; | ||
| 143 | t->interval = !oneshot; | ||
| 144 | t->next = st->timers; | ||
| 145 | if (st->timers) st->timers->prev = t; | ||
| 146 | st->timers = t; | ||
| 147 | ur_arm_timer(st, t); | ||
| 148 | return 0; | ||
| 149 | } | ||
| 150 | |||
| 151 | static void ur_timer_unlink(hm_ur_state *st, ur_timer *t) { | ||
| 152 | if (t->prev) t->prev->next = t->next; | ||
| 153 | else st->timers = t->next; | ||
| 154 | if (t->next) t->next->prev = t->prev; | ||
| 155 | } | ||
| 156 | |||
| 157 | /* Cancel a timer. The timeout is already in flight in the kernel and its | ||
| 158 | * user_data IS this pointer, so freeing here would leave the kernel holding a | ||
| 159 | * dangling reference: the completion still arrives, ur_wait dereferences it | ||
| 160 | * and frees it a second time. Worse, the block is the right size to be handed | ||
| 161 | * straight back to the next ur_add_timer, so the stale completion lands on a | ||
| 162 | * live timer and takes that down instead - which is why this showed up as an | ||
| 163 | * intermittent "double free or corruption" rather than a reliable one. | ||
| 164 | * | ||
| 165 | * So: ask the kernel to remove the timeout and mark the watcher dead, but | ||
| 166 | * leave it linked and allocated. Its completion (-ECANCELED, or a genuine | ||
| 167 | * fire that raced the removal) frees it in ur_wait; anything still linked at | ||
| 168 | * shutdown is freed by ur_destroy. */ | ||
| 169 | static int ur_del_timer(hm_backend *be, void *udata) { | ||
| 170 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 171 | ur_timer *t; | ||
| 172 | for (t = st->timers; t; t = t->next) { | ||
| 173 | if (t->udata == udata && !t->cancelled) { | ||
| 174 | struct io_uring_sqe *sqe; | ||
| 175 | t->cancelled = 1; | ||
| 176 | t->interval = 0; /* never re-arm a cancelled timer */ | ||
| 177 | if ((sqe = ur_sqe(st)) != NULL) { | ||
| 178 | io_uring_prep_timeout_remove(sqe, (__u64)(uintptr_t)t, 0); | ||
| 179 | UR_SET_DATA(sqe, UR_TAG_IGNORE); | ||
| 180 | } | ||
| 181 | return 0; | ||
| 182 | } | ||
| 183 | } | ||
| 184 | return -1; | ||
| 185 | } | ||
| 186 | |||
| 187 | static int ur_add_signal(hm_backend *be, int signo) { | ||
| 188 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 189 | sigaddset(&st->sigs, signo); | ||
| 190 | if (sigprocmask(SIG_BLOCK, &st->sigs, NULL) < 0) return -1; | ||
| 191 | { | ||
| 192 | int sfd = signalfd(st->sfd, &st->sigs, SFD_NONBLOCK | SFD_CLOEXEC); | ||
| 193 | if (sfd < 0) return -1; | ||
| 194 | if (st->sfd < 0) { | ||
| 195 | st->sfd = sfd; | ||
| 196 | st->mask[sfd] |= HM_EV_READ; /* persistent poll on the ring */ | ||
| 197 | ur_arm_poll(st, sfd, HM_EV_READ); | ||
| 198 | } | ||
| 199 | } | ||
| 200 | return 0; | ||
| 201 | } | ||
| 202 | |||
| 203 | static int ur_wait(hm_backend *be, hm_event *out, int max, double timeout) { | ||
| 204 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 205 | struct io_uring_cqe *cqes[HM_MAXEV]; | ||
| 206 | struct io_uring_cqe *cqe; | ||
| 207 | unsigned want, n, i; | ||
| 208 | int nev = 0, ret; | ||
| 209 | |||
| 210 | io_uring_submit(&st->ring); /* flush re-arms, cancels, timers */ | ||
| 211 | |||
| 212 | if (timeout < 0) { | ||
| 213 | ret = io_uring_wait_cqe(&st->ring, &cqe); | ||
| 214 | } else if (timeout == 0) { | ||
| 215 | ret = io_uring_peek_cqe(&st->ring, &cqe); | ||
| 216 | if (ret == -EAGAIN) return 0; | ||
| 217 | } else { | ||
| 218 | struct __kernel_timespec ts; | ||
| 219 | ts.tv_sec = (int64_t)timeout; | ||
| 220 | ts.tv_nsec = (int64_t)((timeout - (double)ts.tv_sec) * 1e9); | ||
| 221 | ret = io_uring_wait_cqe_timeout(&st->ring, &cqe, &ts); | ||
| 222 | if (ret == -ETIME) return 0; | ||
| 223 | } | ||
| 224 | if (ret < 0) return (ret == -EINTR) ? 0 : -1; | ||
| 225 | |||
| 226 | /* headroom so a single signalfd completion (multiple queued signals) | ||
| 227 | * can't overflow the caller's event buffer */ | ||
| 228 | want = (max > 8) ? (unsigned)(max - 8) : 1u; | ||
| 229 | if (want > HM_MAXEV) want = HM_MAXEV; | ||
| 230 | n = io_uring_peek_batch_cqe(&st->ring, cqes, want); | ||
| 231 | |||
| 232 | for (i = 0; i < n; i++) { | ||
| 233 | __u64 data = cqes[i]->user_data; | ||
| 234 | int res = cqes[i]->res; | ||
| 235 | __u64 tag = data & UR_TAG_MASK; | ||
| 236 | |||
| 237 | if (data == LIBURING_UDATA_TIMEOUT) continue; /* liburing internal */ | ||
| 238 | if (tag == UR_TAG_IGNORE) continue; /* poll-remove ack */ | ||
| 239 | |||
| 240 | if (tag == UR_TAG_READ || tag == UR_TAG_WRITE) { | ||
| 241 | int fd = (int)(data >> 3); | ||
| 242 | int bit = (tag == UR_TAG_READ) ? HM_EV_READ : HM_EV_WRITE; | ||
| 243 | st->armed[fd] &= (unsigned char)~bit; | ||
| 244 | if (res == -ECANCELED || !(st->mask[fd] & bit)) continue; | ||
| 245 | |||
| 246 | if (st->sfd >= 0 && fd == st->sfd) { | ||
| 247 | struct signalfd_siginfo si; | ||
| 248 | while (nev < max | ||
| 249 | && read(st->sfd, &si, sizeof(si)) == (ssize_t)sizeof(si)) { | ||
| 250 | out[nev].kind = HM_EV_SIGNAL; | ||
| 251 | out[nev].fd = (int)si.ssi_signo; | ||
| 252 | out[nev].udata = NULL; | ||
| 253 | nev++; | ||
| 254 | } | ||
| 255 | ur_arm_poll(st, st->sfd, HM_EV_READ); /* persistent */ | ||
| 256 | continue; | ||
| 257 | } | ||
| 258 | |||
| 259 | out[nev].kind = bit; | ||
| 260 | out[nev].fd = fd; | ||
| 261 | out[nev].udata = NULL; | ||
| 262 | nev++; | ||
| 263 | if (st->once[fd] & bit) { | ||
| 264 | st->mask[fd] &= (unsigned char)~bit; | ||
| 265 | st->once[fd] &= (unsigned char)~bit; | ||
| 266 | } else { | ||
| 267 | ur_arm_poll(st, fd, bit); /* persistent: re-arm */ | ||
| 268 | } | ||
| 269 | continue; | ||
| 270 | } | ||
| 271 | |||
| 272 | { /* timer: user_data is the ur_timer pointer (tag 0) */ | ||
| 273 | ur_timer *t = (ur_timer *)(uintptr_t)data; | ||
| 274 | if (!t) continue; | ||
| 275 | /* A cancelled timer's completion still arrives; it owns the | ||
| 276 | * struct, and it must not be reported to the loop. -ECANCELED | ||
| 277 | * covers the removal succeeding, t->cancelled the case where the | ||
| 278 | * timeout fired before the removal reached the kernel. */ | ||
| 279 | if (t->cancelled || res == -ECANCELED) { | ||
| 280 | ur_timer_unlink(st, t); | ||
| 281 | free(t); | ||
| 282 | continue; | ||
| 283 | } | ||
| 284 | out[nev].kind = HM_EV_TIMER; | ||
| 285 | out[nev].fd = -1; | ||
| 286 | out[nev].udata = t->udata; | ||
| 287 | nev++; | ||
| 288 | if (t->interval) { | ||
| 289 | ur_arm_timer(st, t); | ||
| 290 | } else { | ||
| 291 | ur_timer_unlink(st, t); | ||
| 292 | free(t); | ||
| 293 | } | ||
| 294 | } | ||
| 295 | } | ||
| 296 | io_uring_cq_advance(&st->ring, n); | ||
| 297 | return nev; | ||
| 298 | } | ||
| 299 | |||
| 300 | static void ur_destroy(hm_backend *be) { | ||
| 301 | hm_ur_state *st = (hm_ur_state *)be->state; | ||
| 302 | if (st) { | ||
| 303 | while (st->timers) { | ||
| 304 | ur_timer *t = st->timers; | ||
| 305 | st->timers = t->next; | ||
| 306 | free(t); | ||
| 307 | } | ||
| 308 | /* be->foreign: inherited across a fork. The signalfd and the ring's | ||
| 309 | * own descriptor are the parent's; io_uring_queue_exit would close | ||
| 310 | * the ring fd and unmap rings this process never set up. */ | ||
| 311 | if (!be->foreign) { | ||
| 312 | if (st->sfd >= 0) close(st->sfd); | ||
| 313 | io_uring_queue_exit(&st->ring); | ||
| 314 | } | ||
| 315 | free(st); | ||
| 316 | } | ||
| 317 | free(be); | ||
| 318 | } | ||
| 319 | |||
| 320 | hm_backend *hm_backend_iouring_new(void) { | ||
| 321 | hm_backend *be = (hm_backend *)calloc(1, sizeof(hm_backend)); | ||
| 322 | hm_ur_state *st = (hm_ur_state *)calloc(1, sizeof(hm_ur_state)); | ||
| 323 | if (!be || !st) { free(be); free(st); return NULL; } | ||
| 324 | st->sfd = -1; | ||
| 325 | sigemptyset(&st->sigs); | ||
| 326 | if (io_uring_queue_init(1024, &st->ring, 0) < 0) { | ||
| 327 | free(be); free(st); return NULL; | ||
| 328 | } | ||
| 329 | be->name = "io_uring"; | ||
| 330 | be->state = st; | ||
| 331 | be->add_io = ur_add_io; | ||
| 332 | be->modify_io = ur_modify_io; | ||
| 333 | be->remove_io = ur_remove_io; | ||
| 334 | be->add_timer = ur_add_timer; | ||
| 335 | be->del_timer = ur_del_timer; | ||
| 336 | be->add_signal = ur_add_signal; | ||
| 337 | be->wait_ev = ur_wait; | ||
| 338 | be->destroy = ur_destroy; | ||
| 339 | return be; | ||
| 340 | } | ||
| 341 | |||
| 342 | int hm_backend_iouring_available(void) { | ||
| 343 | /* liburing is linked; confirm the kernel actually supports io_uring | ||
| 344 | * (not the case under, e.g., default Docker seccomp) */ | ||
| 345 | static int avail = -1; | ||
| 346 | if (avail < 0) { | ||
| 347 | struct io_uring ring; | ||
| 348 | if (io_uring_queue_init(2, &ring, 0) == 0) { | ||
| 349 | io_uring_queue_exit(&ring); | ||
| 350 | avail = 1; | ||
| 351 | } else { | ||
| 352 | avail = 0; | ||
| 353 | } | ||
| 354 | } | ||
| 355 | return avail; | ||
| 356 | } | ||
| 357 | |||
| 358 | #else /* !HM_IOURING_ACTIVE */ | ||
| 359 | |||
| 360 | 44 | hm_backend *hm_backend_iouring_new(void) { return 0; } | |
| 361 | 0 | int hm_backend_iouring_available(void) { return 0; } | |
| 362 | |||
| 363 | #endif |