From 3e5f0be52ce1a49b39bfda412d7c12acf78c8276 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 1 Apr 2025 13:51:30 +0300 Subject: [PATCH] Use separate port numbers for RDMA-CM --- docs/config/network.en.md | 6 +++--- docs/config/network.ru.md | 6 +++--- docs/config/src/network.yml | 12 ++++++------ src/client/messenger.cpp | 6 +++--- src/client/messenger.h | 6 ++++-- src/client/msgr_rdmacm.cpp | 22 ++++++++++++---------- src/osd/osd.cpp | 3 ++- src/osd/osd.h | 1 + src/osd/osd_cluster.cpp | 2 +- 9 files changed, 35 insertions(+), 29 deletions(-) diff --git a/docs/config/network.en.md b/docs/config/network.en.md index 6513ee0b..1592bcfa 100644 --- a/docs/config/network.en.md +++ b/docs/config/network.en.md @@ -78,9 +78,9 @@ Consider `use_rdmacm` for such networks. Use an alternative implementation of RDMA through RDMA-CM (Connection Manager). Works with all RDMA networks: Infiniband, iWARP and RoCEv1/RoCEv2, and even allows to disable TCP and run only with RDMA. -When enabled, OSDs listen to the same address(es) and port(s) using -TCP and RDMA-CM. `use_rdma` is automatically disabled when `use_rdmacm` -is enabled. +OSDs always use random port numbers for RDMA-CM listeners, different +from their TCP ports. `use_rdma` is automatically disabled when +`use_rdmacm` is enabled. ## disable_tcp diff --git a/docs/config/network.ru.md b/docs/config/network.ru.md index 1c349abf..5f1f7ea5 100644 --- a/docs/config/network.ru.md +++ b/docs/config/network.ru.md @@ -78,9 +78,9 @@ RDMA-устройства, но они не имеют соединения с Использовать альтернативную реализацию RDMA на основе RDMA-CM (Connection Manager). Работает со всеми типами RDMA-сетей: Infiniband, iWARP и RoCEv1/RoCEv2, и даже позволяет полностью отключить TCP и работать -только на RDMA. Когда опция включена, OSD слушают один и тот же порт -на одних и тех же адресах через TCP и RDMA-CM. Также при включении -автоматически отключается опция `use_rdma`. +только на RDMA. OSD используют случайные номера портов для ожидания +соединений через RDMA-CM, отличающиеся от их TCP-портов. Также при +включении `use_rdmacm` автоматически отключается опция `use_rdma`. ## disable_tcp diff --git a/docs/config/src/network.yml b/docs/config/src/network.yml index 6fb984b7..53af6b72 100644 --- a/docs/config/src/network.yml +++ b/docs/config/src/network.yml @@ -56,16 +56,16 @@ Use an alternative implementation of RDMA through RDMA-CM (Connection Manager). Works with all RDMA networks: Infiniband, iWARP and RoCEv1/RoCEv2, and even allows to disable TCP and run only with RDMA. - When enabled, OSDs listen to the same address(es) and port(s) using - TCP and RDMA-CM. `use_rdma` is automatically disabled when `use_rdmacm` - is enabled. + OSDs always use random port numbers for RDMA-CM listeners, different + from their TCP ports. `use_rdma` is automatically disabled when + `use_rdmacm` is enabled. info_ru: | Использовать альтернативную реализацию RDMA на основе RDMA-CM (Connection Manager). Работает со всеми типами RDMA-сетей: Infiniband, iWARP и RoCEv1/RoCEv2, и даже позволяет полностью отключить TCP и работать - только на RDMA. Когда опция включена, OSD слушают один и тот же порт - на одних и тех же адресах через TCP и RDMA-CM. Также при включении - автоматически отключается опция `use_rdma`. + только на RDMA. OSD используют случайные номера портов для ожидания + соединений через RDMA-CM, отличающиеся от их TCP-портов. Также при + включении `use_rdmacm` автоматически отключается опция `use_rdma`. - name: disable_tcp type: bool default: true diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index 976bfad2..f11dc2fa 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -420,7 +420,7 @@ void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state) wanted_peers[peer_osd].address_changed = true; } #ifdef WITH_RDMACM - wanted_peers[peer_osd].peer_rdmacm = peer_state["rdmacm"].bool_value(); + wanted_peers[peer_osd].rdmacm_port = (int)peer_state["rdmacm_port"].int64_value(); #endif wanted_peers[peer_osd].port = (int)peer_state["port"].int64_value(); try_connect_peer(peer_osd); @@ -448,8 +448,8 @@ void osd_messenger_t::try_connect_peer(uint64_t peer_osd) wp.cur_port = wp.port; wp.connecting = true; #ifdef WITH_RDMACM - if (use_rdmacm && wp.peer_rdmacm) - rdmacm_try_connect_peer(peer_osd, wp.cur_addr.c_str(), wp.cur_port); + if (use_rdmacm && wp.rdmacm_port) + rdmacm_try_connect_peer(peer_osd, wp.cur_addr.c_str(), wp.rdmacm_port, wp.cur_port); else #endif try_connect_peer_tcp(peer_osd, wp.cur_addr.c_str(), wp.cur_port); diff --git a/src/client/messenger.h b/src/client/messenger.h index 55a803f8..a5437706 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -96,8 +96,10 @@ struct osd_wanted_peer_t { json11::Json raw_address_list; json11::Json address_list; - bool peer_rdmacm = false; int port = 0; +#ifdef WITH_RDMACM + int rdmacm_port = 0; +#endif time_t last_connect_attempt = 0; bool connecting = false, address_changed = false; int address_index = 0; @@ -283,7 +285,7 @@ protected: msgr_rdma_context_t* rdmacm_get_context(ibv_context *verbs); msgr_rdma_context_t* rdmacm_create_qp(rdma_cm_id *cmid); void rdmacm_accept(rdma_cm_event *ev); - void rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int peer_port); + void rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int rdmacm_port, int fallback_tcp_port); void rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res); void rdmacm_address_resolved(rdma_cm_event *ev); void rdmacm_route_resolved(rdma_cm_event *ev); diff --git a/src/client/msgr_rdmacm.cpp b/src/client/msgr_rdmacm.cpp index 0e61e710..de27931e 100644 --- a/src/client/msgr_rdmacm.cpp +++ b/src/client/msgr_rdmacm.cpp @@ -15,7 +15,8 @@ struct rdmacm_connecting_t osd_num_t peer_osd = 0; std::string addr; sockaddr_storage parsed_addr = {}; - int peer_port = 0; + int rdmacm_port = 0; + int tcp_port = 0; int timeout_ms = 0; int timeout_id = -1; msgr_rdma_context_t *rdma_context = NULL; @@ -315,7 +316,7 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res) { auto conn = rdmacm_connecting.at(cmid); auto addr = conn->addr; - auto peer_port = conn->peer_port; + auto tcp_port = conn->tcp_port; auto peer_osd = conn->peer_osd; if (conn->timeout_id >= 0) tfd->clear_timer(conn->timeout_id); @@ -334,7 +335,7 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res) if (!disable_tcp) { // Fall back to TCP instead of just reporting the error to on_connect_peer() - try_connect_peer_tcp(peer_osd, addr.c_str(), peer_port); + try_connect_peer_tcp(peer_osd, addr.c_str(), tcp_port); } else { @@ -343,10 +344,10 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res) } } -void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int peer_port) +void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int rdmacm_port, int fallback_tcp_port) { struct sockaddr_storage sa = {}; - if (!string_to_addr(addr, false, peer_port, &sa)) + if (!string_to_addr(addr, false, rdmacm_port, &sa)) { fprintf(stderr, "Address %s is invalid\n", addr.c_str()); on_connect_peer(peer_osd, -EINVAL); @@ -358,7 +359,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri int res = -errno; fprintf(stderr, "Failed to create RDMA-CM ID: %s (code %d), using TCP\n", strerror(errno), errno); if (!disable_tcp) - try_connect_peer_tcp(peer_osd, addr.c_str(), peer_port); + try_connect_peer_tcp(peer_osd, addr.c_str(), fallback_tcp_port); else on_connect_peer(peer_osd, res); return; @@ -380,7 +381,8 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri conn->peer_osd = peer_osd; conn->addr = addr; conn->parsed_addr = sa; - conn->peer_port = peer_port; + conn->rdmacm_port = rdmacm_port; + conn->tcp_port = fallback_tcp_port; conn->timeout_ms = peer_connect_timeout*1000; conn->timeout_id = -1; if (peer_connect_timeout > 0) @@ -465,7 +467,7 @@ void osd_messenger_t::rdmacm_route_resolved(rdma_cm_event *ev) if (rdma_connect(cmid, &conn_params) != 0) { int res = -errno; - fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->peer_port, strerror(errno), errno); + fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->rdmacm_port, strerror(errno), errno); rdmacm_on_connect_peer_error(cmid, res); return; } @@ -484,7 +486,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev) auto peer_osd = conn->peer_osd; if (ev->event != RDMA_CM_EVENT_ESTABLISHED || ev->status != 0) { - fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->peer_port, + fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->rdmacm_port, ev->status > 0 ? "unknown error" : strerror(-ev->status), ev->status); rdmacm_on_connect_peer_error(cmid, ev->status); return; @@ -503,7 +505,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev) // And an osd_client_t auto cl = new osd_client_t(); cl->peer_addr = conn->parsed_addr; - cl->peer_port = conn->peer_port; + cl->peer_port = conn->rdmacm_port; cl->peer_fd = conn->peer_fd; cl->peer_state = PEER_RDMA; cl->connect_timeout_id = -1; diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 6726543f..667e8cad 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -376,7 +376,8 @@ void osd_t::bind_socket() { for (auto & bind_address: bind_addresses) { - auto listener = msgr.rdmacm_listen(bind_address, listening_port, &listening_port, log_level); + // iWARP shares port numbers with TCP traffic and can't listen on the same port with TCP and RDMACM + auto listener = msgr.rdmacm_listen(bind_address, rdmacm_port, &rdmacm_port, log_level); if (listener) rdmacm_listeners.push_back(listener); } diff --git a/src/osd/osd.h b/src/osd/osd.h index cb862ea8..576fa85a 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -206,6 +206,7 @@ class osd_t std::vector listen_fds; #ifdef WITH_RDMACM std::vector rdmacm_listeners; + int rdmacm_port = 0; #endif ring_consumer_t consumer; diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index c2764754..f5583758 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -174,7 +174,7 @@ json11::Json osd_t::get_osd_state() st["port"] = listening_port; #ifdef WITH_RDMACM if (rdmacm_listeners.size()) - st["rdmacm"] = true; + st["rdmacm_port"] = rdmacm_port; #endif st["primary_enabled"] = run_primary; st["blockstore_enabled"] = bs ? true : false;