Implement io_uring zero-copy send support

src/util/ringloop.cpp

@@ -10,6 +10,10 @@
 
 #include "ringloop.h"
 
+#ifndef IORING_CQE_F_MORE
+#define IORING_CQE_F_MORE (1U << 1)
+#endif
+
 ring_loop_t::ring_loop_t(int qd, bool multithreaded)
 {
     mt = multithreaded;
@@ -30,6 +34,16 @@ ring_loop_t::ring_loop_t(int qd, bool multithreaded)
         free_ring_data[i] = i;
     }
     in_loop = false;
+    auto probe = io_uring_get_probe();
+    if (probe)
+    {
+        support_zc = io_uring_opcode_supported(probe, IORING_OP_SENDMSG_ZC);
+#ifdef IORING_SETUP_R_DISABLED /* liburing 2.0 check */
+        io_uring_free_probe(probe);
+#else
+        free(probe);
+#endif
+    }
 }
 
 ring_loop_t::~ring_loop_t()
@@ -108,7 +122,17 @@ void ring_loop_t::loop()
         if (mt)
             mu.lock();
         struct ring_data_t *d = (struct ring_data_t*)cqe->user_data;
-        if (d->callback)
+        if (cqe->flags & IORING_CQE_F_MORE)
+        {
+            // There will be a second notification
+            d->res = cqe->res;
+            d->more = true;
+            if (d->callback)
+                d->callback(d);
+            d->prev = true;
+            d->more = false;
+        }
+        else if (d->callback)
         {
             // First free ring_data item, then call the callback
             // so it has at least 1 free slot for the next event
@@ -116,7 +140,10 @@ void ring_loop_t::loop()
             struct ring_data_t dl;
             dl.iov = d->iov;
             dl.res = cqe->res;
+            dl.more = false;
+            dl.prev = d->prev;
             dl.callback.swap(d->callback);
+            d->prev = d->more = false;
             free_ring_data[free_ring_data_ptr++] = d - ring_datas;
             if (mt)
                 mu.unlock();

src/util/ringloop.h

@@ -18,6 +18,10 @@
 
 #define RINGLOOP_DEFAULT_SIZE 1024
 
+#ifndef IORING_RECV_MULTISHOT /* liburing-2.3 check */
+#define IORING_OP_SENDMSG_ZC 48
+#endif
+
 static inline void my_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd, const void *addr, unsigned len, off_t offset)
 {
     // Prepare a read/write operation without clearing user_data
@@ -62,6 +66,12 @@ static inline void my_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd, const
     sqe->msg_flags = flags;
 }
 
+static inline void my_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd, const struct msghdr *msg, unsigned flags)
+{
+    my_uring_prep_rw(IORING_OP_SENDMSG_ZC, sqe, fd, msg, 1, 0);
+    sqe->msg_flags = flags;
+}
+
 static inline void my_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd, short poll_mask)
 {
     my_uring_prep_rw(IORING_OP_POLL_ADD, sqe, fd, NULL, 0, 0);
@@ -112,6 +122,8 @@ struct ring_data_t
 {
     struct iovec iov; // for single-entry read/write operations
     int res;
+    bool prev: 1;
+    bool more: 1;
     std::function<void(ring_data_t*)> callback;
 };
 
@@ -133,6 +145,7 @@ class ring_loop_t
     bool loop_again;
     struct io_uring ring;
     int ring_eventfd = -1;
+    bool support_zc = false;
 public:
     ring_loop_t(int qd, bool multithreaded = false);
     ~ring_loop_t();
@@ -163,6 +176,10 @@ public:
     {
         return loop_again;
     }
+    inline bool has_sendmsg_zc()
+    {
+        return support_zc;
+    }
 
     void loop();
     void wakeup();