thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Wait for OSDs to either connect or stop infinitely during listing, not for peer_connect_timeout

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-01-01 15:29:42 +03:00
parent 80dda3ca94
commit 4edda88903
3 changed files with 28 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/cluster_client.cpp
+3 -6
View File
@@ -409,18 +409,15 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
}
// client_retry_enospc
client_retry_enospc = config["client_retry_enospc"].is_null() ? true : config["client_retry_enospc"].bool_value();
// peer_connect_timeout, wait_up_timeout
peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
if (!peer_connect_timeout)
peer_connect_timeout = 5;
// client_wait_up_timeout
if (!config["client_wait_up_timeout"].is_null())
wait_up_timeout = config["client_wait_up_timeout"].uint64_value();
client_wait_up_timeout = config["client_wait_up_timeout"].uint64_value();
else
{
auto etcd_report_interval = config["etcd_report_interval"].uint64_value();
if (!etcd_report_interval)
etcd_report_interval = 5;
wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000;
client_wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000;
}
// log_level
log_level = config["log_level"].uint64_value();
src/client/cluster_client.h
+1 -2
View File
@@ -94,8 +94,7 @@ class cluster_client_t
int client_retry_interval = 50; // ms
int client_eio_retry_interval = 1000; // ms
bool client_retry_enospc = true;
int peer_connect_timeout = 5; // sec
int wait_up_timeout = 10; // sec (for listings)
int client_wait_up_timeout = 16; // sec (for listings)
int retry_timeout_id = -1;
int retry_timeout_duration = 0;
src/client/cluster_client_list.cpp
+24 -23
View File
@@ -173,43 +173,42 @@ void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg)
}
}
int new_st = start_pg_listing(pg);
if (new_st == LIST_PG_SENT)
if (new_st == LIST_PG_SENT || new_st == LIST_PG_WAIT_CONNECT)
{
pg->state = LIST_PG_SENT;
// sent => wait for completion
// not connected, but OSD state exists => wait for PG or OSD state change infinitely
pg->state = new_st;
return;
}
if (new_st == LIST_PG_WAIT_ACTIVE && pg->state != LIST_PG_WAIT_ACTIVE ||
new_st == LIST_PG_WAIT_CONNECT && pg->state != LIST_PG_WAIT_CONNECT)
if (new_st == LIST_PG_WAIT_ACTIVE && pg->state != LIST_PG_WAIT_ACTIVE)
{
int sec = (new_st == LIST_PG_WAIT_ACTIVE ? wait_up_timeout : peer_connect_timeout);
if (sec)
if (!client_wait_up_timeout)
{
pg->state = new_st;
clock_gettime(CLOCK_REALTIME, &pg->wait_until);
pg->wait_until.tv_sec += sec;
if (new_st == LIST_PG_WAIT_ACTIVE)
{
if (log_level > 1)
fprintf(stderr, "Waiting for PG %u/%u to become active for %d seconds\n", pg->lst->pool_id, pg->pg_num, wait_up_timeout);
}
else
{
if (log_level > 2)
fprintf(stderr, "Waiting for connection to PG %u/%u OSDs for %d seconds\n", pg->lst->pool_id, pg->pg_num, peer_connect_timeout);
}
set_list_retry_timeout(sec*1000, pg->wait_until);
fprintf(stderr, "PG %u/%u is inactive, skipping listing\n", pg->lst->pool_id, pg->pg_num);
pg->errcode = -EPIPE;
pg->list_osds.clear();
pg->objects.clear();
finish_list_pg(pg, false);
return;
}
pg->state = new_st;
clock_gettime(CLOCK_REALTIME, &pg->wait_until);
pg->wait_until.tv_sec += client_wait_up_timeout;
if (log_level > 1)
{
fprintf(stderr, "Waiting for PG %u/%u to become active for %d seconds\n", pg->lst->pool_id, pg->pg_num, client_wait_up_timeout);
}
set_list_retry_timeout(client_wait_up_timeout*1000, pg->wait_until);
return;
}
assert(pg->state == LIST_PG_WAIT_ACTIVE || pg->state == LIST_PG_WAIT_CONNECT);
assert(pg->state == LIST_PG_WAIT_ACTIVE);
// Check if the timeout expired
timespec tv;
clock_gettime(CLOCK_REALTIME, &tv);
if (tv.tv_sec > pg->wait_until.tv_sec ||
tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec >= pg->wait_until.tv_nsec)
{
fprintf(stderr, "Failed to wait for PG %u/%u to become %s, skipping listing\n", pg->lst->pool_id, pg->pg_num,
pg->state == LIST_PG_WAIT_ACTIVE ? "active" : "connected");
fprintf(stderr, "Failed to wait for PG %u/%u to become active, skipping listing\n", pg->lst->pool_id, pg->pg_num);
pg->errcode = -EPIPE;
pg->list_osds.clear();
pg->objects.clear();
@@ -229,6 +228,8 @@ void cluster_client_t::set_list_retry_timeout(int ms, timespec new_time)
}
list_retry_timeout_id = tfd->set_timer(ms, false, [this](int timer_id)
{
list_retry_timeout_id = -1;
list_retry_time = {};
continue_lists();
});
}