From 4edda88903f03661caaccc43529a0c2db7298c3d Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Wed, 1 Jan 2025 15:29:42 +0300 Subject: [PATCH] Wait for OSDs to either connect or stop infinitely during listing, not for peer_connect_timeout --- src/client/cluster_client.cpp | 9 ++---- src/client/cluster_client.h | 3 +- src/client/cluster_client_list.cpp | 47 +++++++++++++++--------------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index cf141145..4b94d7eb 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -409,18 +409,15 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co } // client_retry_enospc client_retry_enospc = config["client_retry_enospc"].is_null() ? true : config["client_retry_enospc"].bool_value(); - // peer_connect_timeout, wait_up_timeout - peer_connect_timeout = config["peer_connect_timeout"].uint64_value(); - if (!peer_connect_timeout) - peer_connect_timeout = 5; + // client_wait_up_timeout if (!config["client_wait_up_timeout"].is_null()) - wait_up_timeout = config["client_wait_up_timeout"].uint64_value(); + client_wait_up_timeout = config["client_wait_up_timeout"].uint64_value(); else { auto etcd_report_interval = config["etcd_report_interval"].uint64_value(); if (!etcd_report_interval) etcd_report_interval = 5; - wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000; + client_wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000; } // log_level log_level = config["log_level"].uint64_value(); diff --git a/src/client/cluster_client.h b/src/client/cluster_client.h index 9d5ad70f..c83b6f42 100644 --- a/src/client/cluster_client.h +++ b/src/client/cluster_client.h @@ -94,8 +94,7 @@ class cluster_client_t int client_retry_interval = 50; // ms int client_eio_retry_interval = 1000; // ms bool client_retry_enospc = true; - int peer_connect_timeout = 5; // sec - int wait_up_timeout = 10; // sec (for listings) + int client_wait_up_timeout = 16; // sec (for listings) int retry_timeout_id = -1; int retry_timeout_duration = 0; diff --git a/src/client/cluster_client_list.cpp b/src/client/cluster_client_list.cpp index 4de136b4..c8e8f286 100644 --- a/src/client/cluster_client_list.cpp +++ b/src/client/cluster_client_list.cpp @@ -173,43 +173,42 @@ void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg) } } int new_st = start_pg_listing(pg); - if (new_st == LIST_PG_SENT) + if (new_st == LIST_PG_SENT || new_st == LIST_PG_WAIT_CONNECT) { - pg->state = LIST_PG_SENT; + // sent => wait for completion + // not connected, but OSD state exists => wait for PG or OSD state change infinitely + pg->state = new_st; return; } - if (new_st == LIST_PG_WAIT_ACTIVE && pg->state != LIST_PG_WAIT_ACTIVE || - new_st == LIST_PG_WAIT_CONNECT && pg->state != LIST_PG_WAIT_CONNECT) + if (new_st == LIST_PG_WAIT_ACTIVE && pg->state != LIST_PG_WAIT_ACTIVE) { - int sec = (new_st == LIST_PG_WAIT_ACTIVE ? wait_up_timeout : peer_connect_timeout); - if (sec) + if (!client_wait_up_timeout) { - pg->state = new_st; - clock_gettime(CLOCK_REALTIME, &pg->wait_until); - pg->wait_until.tv_sec += sec; - if (new_st == LIST_PG_WAIT_ACTIVE) - { - if (log_level > 1) - fprintf(stderr, "Waiting for PG %u/%u to become active for %d seconds\n", pg->lst->pool_id, pg->pg_num, wait_up_timeout); - } - else - { - if (log_level > 2) - fprintf(stderr, "Waiting for connection to PG %u/%u OSDs for %d seconds\n", pg->lst->pool_id, pg->pg_num, peer_connect_timeout); - } - set_list_retry_timeout(sec*1000, pg->wait_until); + fprintf(stderr, "PG %u/%u is inactive, skipping listing\n", pg->lst->pool_id, pg->pg_num); + pg->errcode = -EPIPE; + pg->list_osds.clear(); + pg->objects.clear(); + finish_list_pg(pg, false); return; } + pg->state = new_st; + clock_gettime(CLOCK_REALTIME, &pg->wait_until); + pg->wait_until.tv_sec += client_wait_up_timeout; + if (log_level > 1) + { + fprintf(stderr, "Waiting for PG %u/%u to become active for %d seconds\n", pg->lst->pool_id, pg->pg_num, client_wait_up_timeout); + } + set_list_retry_timeout(client_wait_up_timeout*1000, pg->wait_until); + return; } - assert(pg->state == LIST_PG_WAIT_ACTIVE || pg->state == LIST_PG_WAIT_CONNECT); + assert(pg->state == LIST_PG_WAIT_ACTIVE); // Check if the timeout expired timespec tv; clock_gettime(CLOCK_REALTIME, &tv); if (tv.tv_sec > pg->wait_until.tv_sec || tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec >= pg->wait_until.tv_nsec) { - fprintf(stderr, "Failed to wait for PG %u/%u to become %s, skipping listing\n", pg->lst->pool_id, pg->pg_num, - pg->state == LIST_PG_WAIT_ACTIVE ? "active" : "connected"); + fprintf(stderr, "Failed to wait for PG %u/%u to become active, skipping listing\n", pg->lst->pool_id, pg->pg_num); pg->errcode = -EPIPE; pg->list_osds.clear(); pg->objects.clear(); @@ -229,6 +228,8 @@ void cluster_client_t::set_list_retry_timeout(int ms, timespec new_time) } list_retry_timeout_id = tfd->set_timer(ms, false, [this](int timer_id) { + list_retry_timeout_id = -1; + list_retry_time = {}; continue_lists(); }); }