From 7de38250adf117b2f8f0bc352d1223445b35ebfb Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Wed, 13 Nov 2024 00:56:44 +0300 Subject: [PATCH] Auto-select RDMA device based on osd_network --- src/client/etcd_state_client.cpp | 2 +- src/client/messenger.cpp | 14 +- src/client/messenger.h | 3 +- src/client/msgr_rdma.cpp | 212 ++++++++++++++++++++++++++++--- src/client/msgr_rdma.h | 4 +- src/util/addr_util.cpp | 68 +++++----- src/util/addr_util.h | 12 ++ 7 files changed, 258 insertions(+), 57 deletions(-) diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index b9072c22..1a7be337 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -176,7 +176,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr) exit(1); } if (!local_ips.size()) - local_ips = getifaddr_list(); + local_ips = getifaddr_list(std::vector(), true); std::string check_addr; int pos = addr.find('/'); int pos2 = addr.find(':'); diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index 075a8f7a..5e6023c6 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -121,7 +121,7 @@ void osd_messenger_t::init() if (use_rdma) { rdma_context = msgr_rdma_context_t::create( - rdma_device != "" ? rdma_device.c_str() : NULL, + osd_networks, rdma_device != "" ? rdma_device.c_str() : NULL, rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level ); if (!rdma_context) @@ -266,7 +266,8 @@ void osd_messenger_t::parse_config(const json11::Json & config) this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value(); if (!this->rdma_port_num) this->rdma_port_num = 1; - this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value(); + if (!config["rdma_gid_index"].is_null()) + this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value(); this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value(); this->rdma_max_sge = config["rdma_max_sge"].uint64_value(); if (!this->rdma_max_sge) @@ -281,6 +282,15 @@ void osd_messenger_t::parse_config(const json11::Json & config) if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024) this->rdma_max_msg = 129*1024; this->rdma_odp = config["rdma_odp"].bool_value(); + std::vector mask; + if (config["bind_address"].is_string()) + mask.push_back(config["bind_address"].string_value()); + else if (config["osd_network"].is_string()) + mask.push_back(config["osd_network"].string_value()); + else + for (auto v: config["osd_network"].array_items()) + mask.push_back(v.string_value()); + this->osd_networks = mask; #endif if (!osd_num) this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value(); diff --git a/src/client/messenger.h b/src/client/messenger.h index 1de228d2..8c43552e 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -165,8 +165,9 @@ protected: #ifdef WITH_RDMA bool use_rdma = true; + std::vector osd_networks; std::string rdma_device; - uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0; + uint64_t rdma_port_num = 1, rdma_gid_index = -1, rdma_mtu = 0; msgr_rdma_context_t *rdma_context = NULL; uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0; uint64_t rdma_max_msg = 0; diff --git a/src/client/msgr_rdma.cpp b/src/client/msgr_rdma.cpp index 10d2c408..78e3321c 100644 --- a/src/client/msgr_rdma.cpp +++ b/src/client/msgr_rdma.cpp @@ -3,6 +3,7 @@ #include #include +#include "addr_util.h" #include "msgr_rdma.h" #include "messenger.h" @@ -69,7 +70,126 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t() send_out_size = 0; } -msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level) +static bool is_ipv4_gid(ibv_gid_entry *gidx) +{ + return (((uint64_t*)gidx->gid.raw)[0] == 0 && + ((uint32_t*)gidx->gid.raw)[2] == 0xffff0000); +} + +static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet) +{ + if (gidx->gid_type != IBV_GID_TYPE_ROCE_V1 && + gidx->gid_type != IBV_GID_TYPE_ROCE_V2 || + ((uint64_t*)gidx->gid.raw)[0] == 0 && + ((uint64_t*)gidx->gid.raw)[1] == 0) + { + return false; + } + if (is_ipv4_gid(gidx)) + { + for (int i = 0; i < nnet; i++) + { + if (networks[i].family == AF_INET && cidr_match(*(in_addr*)(gidx->gid.raw+12), networks[i].ipv4, networks[i].bits)) + return true; + } + } + else + { + for (int i = 0; i < nnet; i++) + { + if (networks[i].family == AF_INET6 && cidr6_match(*(in6_addr*)gidx->gid.raw, networks[i].ipv6, networks[i].bits)) + return true; + } + } + return false; +} + +struct matched_dev +{ + int dev = -1; + int port = -1; + int gid = -1; + bool rocev2 = false; +}; + +static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, ibv_gid_entry & gidx) +{ + bool is4 = ((uint64_t*)gidx.gid.raw)[0] == 0 && ((uint32_t*)gidx.gid.raw)[2] == 0xffff0000; + char buf[256]; + inet_ntop(is4 ? AF_INET : AF_INET6, is4 ? gidx.gid.raw+12 : gidx.gid.raw, buf, sizeof(buf)); + fprintf( + stderr, "Auto-selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s\n", + ibv_get_device_name(dev), ib_port, gid_index, + gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf + ); +} + +static matched_dev match_device(ibv_device **dev_list, addr_mask_t *networks, int nnet, int log_level) +{ + matched_dev best; + ibv_device_attr attr; + ibv_port_attr portinfo; + ibv_gid_entry best_gidx; + int res; + for (int i = 0; dev_list[i]; ++i) + { + auto dev = dev_list[i]; + ibv_context *context = ibv_open_device(dev_list[i]); + if ((res = ibv_query_device(context, &attr)) != 0) + { + fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev_list[i]), strerror(res)); + goto cleanup; + } + for (int j = 1; j <= attr.phys_port_cnt; j++) + { + // Try to find a port with matching address + if ((res = ibv_query_port(context, j, &portinfo)) != 0) + { + fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), j, strerror(res)); + goto cleanup; + } + for (int k = 0; k < portinfo.gid_tbl_len; k++) + { + ibv_gid_entry gidx; + if ((res = ibv_query_gid_ex(context, j, k, &gidx, 0)) != 0) + { + if (res != ENODATA) + { + fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(dev), k, strerror(res)); + goto cleanup; + } + else + break; + } + if (match_gid(&gidx, networks, nnet)) + { + // Prefer RoCEv2 + if (!best.rocev2) + { + best.dev = i; + best.port = j; + best.gid = k; + best.rocev2 = (gidx.gid_type == IBV_GID_TYPE_ROCE_V2); + best_gidx = gidx; + } + } + } + } +cleanup: + ibv_close_device(context); + if (best.rocev2) + { + break; + } + } + if (best.dev >= 0 && log_level > 0) + { + log_rdma_dev_port_gid(dev_list[best.dev], best.port, best.gid, best_gidx); + } + return best; +} + +msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector osd_networks, const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level) { int res; ibv_device **dev_list = NULL; @@ -80,28 +200,23 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t clock_gettime(CLOCK_REALTIME, &tv); srand48(tv.tv_sec*1000000000 + tv.tv_nsec); dev_list = ibv_get_device_list(NULL); - if (!dev_list) + if (!dev_list || !*dev_list) { if (errno == -ENOSYS || errno == ENOSYS) { if (log_level > 0) fprintf(stderr, "No RDMA devices found (RDMA device list returned ENOSYS)\n"); } + else if (!*dev_list) + { + if (log_level > 0) + fprintf(stderr, "No RDMA devices found\n"); + } else fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno)); goto cleanup; } - if (!ib_devname) - { - ctx->dev = *dev_list; - if (!ctx->dev) - { - if (log_level > 0) - fprintf(stderr, "No RDMA devices found\n"); - goto cleanup; - } - } - else + if (ib_devname) { int i; for (i = 0; dev_list[i]; ++i) @@ -114,6 +229,31 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t goto cleanup; } } + else if (osd_networks.size()) + { + std::vector nets; + for (auto & netstr: osd_networks) + { + nets.push_back(cidr_parse(netstr)); + } + auto best = match_device(dev_list, nets.data(), nets.size(), log_level); + if (best.dev < 0) + { + if (log_level > 0) + fprintf(stderr, "RDMA device matching osd_network is not found, using first available device\n"); + best.dev = 0; + } + else + { + ib_port = best.port; + gid_index = best.gid; + } + ctx->dev = dev_list[best.dev]; + } + else + { + ctx->dev = *dev_list; + } ctx->context = ibv_open_device(ctx->dev); if (!ctx->context) @@ -123,7 +263,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t } ctx->ib_port = ib_port; - ctx->gid_index = gid_index; if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0) { fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res)); @@ -135,10 +274,47 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev)); goto cleanup; } - if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid)) + + if (gid_index != -1) { - fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index); - goto cleanup; + ctx->gid_index = gid_index; + if (ibv_query_gid_ex(ctx->context, ib_port, gid_index, &ctx->my_gid, 0)) + { + fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index); + goto cleanup; + } + } + else + { + // Auto-guess GID + for (int k = 0; k < ctx->portinfo.gid_tbl_len; k++) + { + ibv_gid_entry gidx; + if (ibv_query_gid_ex(ctx->context, ib_port, k, &gidx, 0) != 0) + { + fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), k); + goto cleanup; + } + // Skip empty GID + if (((uint64_t*)gidx.gid.raw)[0] == 0 && + ((uint64_t*)gidx.gid.raw)[1] == 0) + { + continue; + } + // Prefer IPv4 RoCEv2 GID by default + if (gid_index == -1 || + gidx.gid_type == IBV_GID_TYPE_ROCE_V2 && + (ctx->my_gid.gid_type != IBV_GID_TYPE_ROCE_V2 || is_ipv4_gid(&gidx))) + { + gid_index = k; + ctx->my_gid = gidx; + } + } + ctx->gid_index = gid_index = (gid_index == -1 ? 0 : gid_index); + if (log_level > 0) + { + log_rdma_dev_port_gid(ctx->dev, ctx->ib_port, ctx->gid_index, ctx->my_gid); + } } ctx->pd = ibv_alloc_pd(ctx->context); @@ -255,7 +431,7 @@ msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx, } conn->addr.lid = ctx->my_lid; - conn->addr.gid = ctx->my_gid; + conn->addr.gid = ctx->my_gid.gid; conn->addr.qpn = conn->qp->qp_num; conn->addr.psn = lrand48() & 0xffffff; diff --git a/src/client/msgr_rdma.h b/src/client/msgr_rdma.h index ab8049f4..f798d1ce 100644 --- a/src/client/msgr_rdma.h +++ b/src/client/msgr_rdma.h @@ -31,12 +31,12 @@ struct msgr_rdma_context_t uint8_t ib_port; uint8_t gid_index; uint16_t my_lid; - ibv_gid my_gid; + ibv_gid_entry my_gid; uint32_t mtu; int max_cqe = 0; int used_max_cqe = 0; - static msgr_rdma_context_t *create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level); + static msgr_rdma_context_t *create(std::vector osd_networks, const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level); ~msgr_rdma_context_t(); }; diff --git a/src/util/addr_util.cpp b/src/util/addr_util.cpp index 57e8eed9..84474588 100644 --- a/src/util/addr_util.cpp +++ b/src/util/addr_util.cpp @@ -65,7 +65,7 @@ std::string addr_to_string(const sockaddr_storage &addr) return std::string(peer_str)+":"+std::to_string(port); } -static bool cidr_match(const in_addr &addr, const in_addr &net, uint8_t bits) +bool cidr_match(const in_addr &addr, const in_addr &net, uint8_t bits) { if (bits == 0) { @@ -75,7 +75,7 @@ static bool cidr_match(const in_addr &addr, const in_addr &net, uint8_t bits) return !((addr.s_addr ^ net.s_addr) & htonl(0xFFFFFFFFu << (32 - bits))); } -static bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits) +bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits) { const uint32_t *a = address.s6_addr32; const uint32_t *n = network.s6_addr32; @@ -93,47 +93,49 @@ static bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_ return true; } -struct addr_mask_t +addr_mask_t cidr_parse(std::string mask) { - sa_family_t family; + unsigned bits = 255; + int p = mask.find('/'); + if (p != std::string::npos) + { + char null_byte = 0; + if (sscanf(mask.c_str()+p+1, "%u%c", &bits, &null_byte) != 1 || bits > 128) + throw std::runtime_error("Invalid IP address mask: " + mask); + mask = mask.substr(0, p); + } in_addr ipv4; in6_addr ipv6; - uint8_t bits; -}; + if (inet_pton(AF_INET, mask.c_str(), &ipv4) == 1) + { + if (bits == 255) + bits = 32; + if (bits > 32) + throw std::runtime_error("Invalid IP address mask: " + mask); + return (addr_mask_t){ .family = AF_INET, .ipv4 = ipv4, .bits = (uint8_t)(bits ? bits : 32) }; + } + else if (inet_pton(AF_INET6, mask.c_str(), &ipv6) == 1) + { + if (bits == 255) + bits = 128; + return (addr_mask_t){ .family = AF_INET6, .ipv6 = ipv6, .bits = (uint8_t)bits }; + } + else + { + throw std::runtime_error("Invalid IP address mask: " + mask); + } +} std::vector getifaddr_list(std::vector mask_cfg, bool include_v6) { std::vector masks; for (auto mask: mask_cfg) { - unsigned bits = 0; - int p = mask.find('/'); - if (p != std::string::npos) + masks.push_back(cidr_parse(mask)); + if (masks[masks.size()-1].family == AF_INET6) { - char null_byte = 0; - if (sscanf(mask.c_str()+p+1, "%u%c", &bits, &null_byte) != 1 || bits > 128) - { - throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask); - } - mask = mask.substr(0, p); - } - in_addr ipv4; - in6_addr ipv6; - if (inet_pton(AF_INET, mask.c_str(), &ipv4) == 1) - { - if (bits > 32) - { - throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask); - } - masks.push_back((addr_mask_t){ .family = AF_INET, .ipv4 = ipv4, .bits = (uint8_t)bits }); - } - else if (include_v6 && inet_pton(AF_INET6, mask.c_str(), &ipv6) == 1) - { - masks.push_back((addr_mask_t){ .family = AF_INET6, .ipv6 = ipv6, .bits = (uint8_t)bits }); - } - else - { - throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask); + // Auto-enable IPv6 addresses + include_v6 = true; } } std::set addresses; diff --git a/src/util/addr_util.h b/src/util/addr_util.h index 17f47f54..034733cf 100644 --- a/src/util/addr_util.h +++ b/src/util/addr_util.h @@ -1,10 +1,22 @@ #pragma once +#include #include #include #include +struct addr_mask_t +{ + sa_family_t family; + in_addr ipv4; + in6_addr ipv6; + uint8_t bits; +}; + bool string_to_addr(std::string str, bool parse_port, int default_port, struct sockaddr_storage *addr); std::string addr_to_string(const sockaddr_storage &addr); +addr_mask_t cidr_parse(std::string mask); +bool cidr_match(const in_addr &address, const in_addr &network, uint8_t bits); +bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits); std::vector getifaddr_list(std::vector mask_cfg = std::vector(), bool include_v6 = false); int create_and_bind_socket(std::string bind_address, int bind_port, int listen_backlog, int *listening_port);