Implement io_uring zero-copy send support

src/client/messenger.cpp

@@ -167,6 +167,10 @@ void osd_messenger_t::init()
         }
     }
 #endif
+    if (ringloop)
+    {
+        has_sendmsg_zc = ringloop->has_sendmsg_zc();
+    }
     if (ringloop && iothread_count > 0)
     {
         for (int i = 0; i < iothread_count; i++)
@@ -329,6 +333,9 @@ void osd_messenger_t::parse_config(const json11::Json & config)
         this->receive_buffer_size = 65536;
     this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
         config["use_sync_send_recv"].uint64_value();
+    this->min_zerocopy_send_size = config["min_zerocopy_send_size"].is_null()
+        ? DEFAULT_MIN_ZEROCOPY_SEND_SIZE
+        : (int)config["min_zerocopy_send_size"].int64_value();
     this->peer_connect_interval = config["peer_connect_interval"].uint64_value();
     if (!this->peer_connect_interval)
         this->peer_connect_interval = 5;
@@ -897,6 +904,7 @@ static const char* local_only_params[] = {
     "tcp_header_buffer_size",
     "use_rdma",
     "use_sync_send_recv",
+    "min_zerocopy_send_size",
 };
 
 static const char **local_only_end = local_only_params + (sizeof(local_only_params)/sizeof(local_only_params[0]));

src/client/messenger.h

@@ -32,6 +32,8 @@
 
 #define VITASTOR_CONFIG_PATH "/etc/vitastor/vitastor.conf"
 
+#define DEFAULT_MIN_ZEROCOPY_SEND_SIZE 32*1024
+
 #define MSGR_SENDP_HDR 1
 #define MSGR_SENDP_FREE 2
 
@@ -88,6 +90,7 @@ struct osd_client_t
     int write_state = 0;
     std::vector<iovec> send_list, next_send_list;
     std::vector<msgr_sendp_t> outbox, next_outbox;
+    std::vector<osd_op_t*> zc_free_list;
 
     ~osd_client_t();
 };
@@ -176,6 +179,7 @@ protected:
     int osd_ping_timeout = 0;
     int log_level = 0;
     bool use_sync_send_recv = false;
+    int min_zerocopy_send_size = DEFAULT_MIN_ZEROCOPY_SEND_SIZE;
     int iothread_count = 0;
 
 #ifdef WITH_RDMA
@@ -202,8 +206,9 @@ protected:
     std::vector<osd_op_t*> set_immediate_ops;
 
 public:
-    timerfd_manager_t *tfd;
-    ring_loop_t *ringloop;
+    timerfd_manager_t *tfd = NULL;
+    ring_loop_t *ringloop = NULL;
+    bool has_sendmsg_zc = false;
     // osd_num_t is only for logging and asserts
     osd_num_t osd_num;
     uint64_t next_subop_id = 1;
@@ -262,7 +267,7 @@ protected:
     void cancel_op(osd_op_t *op);
 
     bool try_send(osd_client_t *cl);
-    void handle_send(int result, osd_client_t *cl);
+    void handle_send(int result, bool prev, bool more, osd_client_t *cl);
 
     bool handle_read(int result, osd_client_t *cl);
     bool handle_read_buffer(osd_client_t *cl, void *curbuf, int remain);

src/client/msgr_send.cpp

@@ -203,8 +203,24 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
         cl->refs++;
         ring_data_t* data = ((ring_data_t*)sqe->user_data);
-        data->callback = [this, cl](ring_data_t *data) { handle_send(data->res, cl); };
-        my_uring_prep_sendmsg(sqe, peer_fd, &cl->write_msg, 0);
+        data->callback = [this, cl](ring_data_t *data) { handle_send(data->res, data->prev, data->more, cl); };
+        bool use_zc = has_sendmsg_zc && min_zerocopy_send_size >= 0;
+        if (use_zc && min_zerocopy_send_size > 0)
+        {
+            size_t avg_size = 0;
+            for (size_t i = 0; i < cl->write_msg.msg_iovlen; i++)
+                avg_size += cl->write_msg.msg_iov[i].iov_len;
+            if (avg_size/cl->write_msg.msg_iovlen < min_zerocopy_send_size)
+                use_zc = false;
+        }
+        if (use_zc)
+        {
+            my_uring_prep_sendmsg_zc(sqe, peer_fd, &cl->write_msg, MSG_WAITALL);
+        }
+        else
+        {
+            my_uring_prep_sendmsg(sqe, peer_fd, &cl->write_msg, MSG_WAITALL);
+        }
         if (iothread)
         {
             iothread->add_sqe(sqe_local);
@@ -220,7 +236,7 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         {
             result = -errno;
         }
-        handle_send(result, cl);
+        handle_send(result, false, false, cl);
     }
     return true;
 }
@@ -240,10 +256,16 @@ void osd_messenger_t::send_replies()
     write_ready_clients.clear();
 }
 
-void osd_messenger_t::handle_send(int result, osd_client_t *cl)
+void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t *cl)
 {
-    cl->write_msg.msg_iovlen = 0;
-    cl->refs--;
+    if (!prev)
+    {
+        cl->write_msg.msg_iovlen = 0;
+    }
+    if (!more)
+    {
+        cl->refs--;
+    }
     if (cl->peer_state == PEER_STOPPED)
     {
         if (cl->refs <= 0)
@@ -261,6 +283,16 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
     }
     if (result >= 0)
     {
+        if (prev)
+        {
+            // Second notification - only free a batch of postponed ops
+            int i = 0;
+            for (; i < cl->zc_free_list.size() && cl->zc_free_list[i]; i++)
+                delete cl->zc_free_list[i];
+            if (i > 0)
+                cl->zc_free_list.erase(cl->zc_free_list.begin(), cl->zc_free_list.begin()+i+1);
+            return;
+        }
         int done = 0;
         while (result > 0 && done < cl->send_list.size())
         {
@@ -270,7 +302,10 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
                 if (cl->outbox[done].flags & MSGR_SENDP_FREE)
                 {
                     // Reply fully sent
-                    delete cl->outbox[done].op;
+                    if (more)
+                        cl->zc_free_list.push_back(cl->outbox[done].op);
+                    else
+                        delete cl->outbox[done].op;
                 }
                 result -= iov.iov_len;
                 done++;
@@ -282,6 +317,11 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
                 break;
             }
         }
+        if (more)
+        {
+            assert(done == cl->send_list.size());
+            cl->zc_free_list.push_back(NULL); // end marker
+        }
         if (done > 0)
         {
             cl->send_list.erase(cl->send_list.begin(), cl->send_list.begin()+done);