Add an alternative RDMA implementation via RDMA-CM

Required for non-RoCE cards: iWARP and, possibly, Infiniband

src/osd/CMakeLists.txt

@@ -14,6 +14,7 @@ target_link_libraries(vitastor-osd
 	Jerasure
 	${ISAL_LIBRARIES}
 	${IBVERBS_LIBRARIES}
+	${RDMACM_LIBRARIES}
 )
 
 # osd_rmw_test

src/osd/osd.cpp

@@ -120,6 +120,11 @@ osd_t::~osd_t()
     delete epmgr;
     if (bs)
         delete bs;
+#ifdef WITH_RDMACM
+    for (rdma_cm_id *listener: rdmacm_listeners)
+        msgr.rdmacm_destroy_listener(listener);
+    rdmacm_listeners.clear();
+#endif
     for (auto listen_fd: listen_fds)
         close(listen_fd);
     listen_fds.clear();
@@ -176,6 +181,11 @@ void osd_t::parse_config(bool init)
         bind_port = config["bind_port"].uint64_value();
         if (bind_port <= 0 || bind_port > 65535)
             bind_port = 0;
+#ifdef WITH_RDMACM
+        // Use RDMA CM? (required for iWARP and may be useful for IB)
+        this->use_rdmacm = config["use_rdmacm"].bool_value() || config["use_rdmacm"].uint64_value() != 0;
+        this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
+#endif
         // OSD configuration
         etcd_report_interval = config["etcd_report_interval"].uint64_value();
         if (etcd_report_interval <= 0)
@@ -348,16 +358,35 @@ void osd_t::bind_socket()
     {
         bind_addresses.push_back("0.0.0.0");
     }
-    for (auto & bind_address: bind_addresses)
+    if (!disable_tcp)
     {
-        int listen_fd = create_and_bind_socket(bind_address, listening_port ? listening_port : bind_port, listen_backlog, &listening_port);
-        fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
-        epmgr->set_fd_handler(listen_fd, false, [this](int fd, int events)
+        for (auto & bind_address: bind_addresses)
         {
-            msgr.accept_connections(fd);
-        });
-        listen_fds.push_back(listen_fd);
+            int listen_fd = create_and_bind_socket(bind_address, listening_port ? listening_port : bind_port, listen_backlog, &listening_port);
+            fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
+            epmgr->set_fd_handler(listen_fd, false, [this](int fd, int events)
+            {
+                msgr.accept_connections(fd);
+            });
+            listen_fds.push_back(listen_fd);
+        }
     }
+#ifdef WITH_RDMACM
+    if (use_rdmacm)
+    {
+        for (auto & bind_address: bind_addresses)
+        {
+            auto listener = msgr.rdmacm_listen(bind_address, listening_port, &listening_port, log_level);
+            if (listener)
+                rdmacm_listeners.push_back(listener);
+        }
+        if (!rdmacm_listeners.size() && disable_tcp)
+        {
+            fprintf(stderr, "Failed to create RDMA-CM listeners, exiting\n");
+            force_stop(1);
+        }
+    }
+#endif
 }
 
 bool osd_t::shutdown()

src/osd/osd.h

@@ -109,6 +109,8 @@ class osd_t
     bool allow_net_split = false;
     std::vector<std::string> cfg_bind_addresses;
     int bind_port, listen_backlog = 128;
+    bool use_rdmacm = false;
+    bool disable_tcp = false;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
     bool allow_test_ops = false;
@@ -202,6 +204,9 @@ class osd_t
     int listening_port = 0;
     std::vector<std::string> bind_addresses;
     std::vector<int> listen_fds;
+#ifdef WITH_RDMACM
+    std::vector<rdma_cm_id *> rdmacm_listeners;
+#endif
     ring_consumer_t consumer;
 
     // op statistics

src/osd/osd_cluster.cpp

@@ -172,6 +172,10 @@ json11::Json osd_t::get_osd_state()
     st["host"] = std::string(hostname.data(), hostname.size());
     st["version"] = VITASTOR_VERSION;
     st["port"] = listening_port;
+#ifdef WITH_RDMACM
+    if (rdmacm_listeners.size())
+        st["rdmacm"] = true;
+#endif
     st["primary_enabled"] = run_primary;
     st["blockstore_enabled"] = bs ? true : false;
     return st;