WIP Switch back to TCP when RDMA send fails (does not work)

src/client/msgr_rdma.cpp

@@ -643,6 +643,7 @@ static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 
 bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
 {
+    return true;
     auto rc = cl->rdma_conn;
     assert(!rc->recv_buf.buf);
     rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
@@ -698,6 +699,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 continue;
             }
             auto rc = cl->rdma_conn;
+            if (!rc)
+            {
+                // Connection destroyed (fallback to TCP)
+                continue;
+            }
             if (wc[i].status != IBV_WC_SUCCESS)
             {
                 fprintf(stderr, "RDMA work request (%s) failed for client %ju", is_send ? "send" : "recv", client_id);
@@ -727,6 +733,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                         close(cl->peer_fd);
                         cl->peer_fd = -1;
                     }
+                    if (cl->osd_num)
+                    {
+                        osd_peers[cl->osd_num] = cl;
+                        on_connect_peer(cl->osd_num, 0, cl->client_id);
+                    }
                 }
                 rc->cur_recv--;
                 if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))