From 8b0389b4e8dc5b59a459687c5a1bf0696002d025 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 22 Mar 2025 15:58:13 +0300 Subject: [PATCH] Log RDMA ibv_modify_qp() errors --- src/client/messenger.cpp | 4 ++-- src/client/msgr_rdma.cpp | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index 5e6023c6..cb654e7d 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -582,7 +582,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl) { osd_num_t peer_osd = cl->osd_num; stop_client(op->peer_fd); - on_connect_peer(peer_osd, -1); + on_connect_peer(peer_osd, -EINVAL); delete op; return; } @@ -602,7 +602,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl) // FIXME: Keep TCP connection in this case osd_num_t peer_osd = cl->osd_num; stop_client(cl->peer_fd); - on_connect_peer(peer_osd, -1); + on_connect_peer(peer_osd, -EINVAL); delete op; return; } diff --git a/src/client/msgr_rdma.cpp b/src/client/msgr_rdma.cpp index 6feeeef8..1039f21d 100644 --- a/src/client/msgr_rdma.cpp +++ b/src/client/msgr_rdma.cpp @@ -469,9 +469,10 @@ msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx, .port_num = ctx->ib_port, }; - if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) + int r = 0; + if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) != 0) { - fprintf(stderr, "Failed to switch RDMA queue pair to INIT state\n"); + fprintf(stderr, "Failed to switch RDMA queue pair to INIT state: %s (code %d)\n", strerror(r), r); delete conn; return NULL; } @@ -522,18 +523,19 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest) .rnr_retry = 7, }; // FIXME No idea if ibv_modify_qp is a blocking operation or not. No idea if it has a timeout and what it is. - if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) + int r; + if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | + IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) != 0) { - fprintf(stderr, "Failed to switch RDMA queue pair to RTR (ready-to-receive) state\n"); - return 1; + fprintf(stderr, "Failed to switch RDMA queue pair to RTR (ready-to-receive) state: %s (code %d)\n", strerror(r), r); + return -r; } attr.qp_state = IBV_QPS_RTS; - if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_TIMEOUT | - IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC)) + if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_TIMEOUT | + IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC)) != 0) { - fprintf(stderr, "Failed to switch RDMA queue pair to RTS (ready-to-send) state\n"); - return 1; + fprintf(stderr, "Failed to switch RDMA queue pair to RTS (ready-to-send) state: %s (code %d)\n", strerror(r), r); + return -r; } return 0; }