thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support local reads in client

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-05-10 16:42:02 +03:00
parent 7e6a95c678
commit 90b1de307b
21 changed files with 295 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/cluster_client.cpp
+104
View File
@@ -3,6 +3,7 @@
#include <stdexcept>
#include <assert.h>
#include "pg_states.h"
#include "cluster_client_impl.h"
#include "json_util.h"
@@ -57,6 +58,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); };
@@ -470,11 +472,95 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
}
// log_level
log_level = config["log_level"].uint64_value();
// hostname
conf_hostname = config["hostname"].string_value();
auto new_hostname = conf_hostname != "" ? conf_hostname : gethostname_str();
if (new_hostname != client_hostname)
{
self_tree_metrics.clear();
client_hostname = new_hostname;
}
msgr.parse_config(config);
st_cli.parse_config(config);
st_cli.load_pgs();
}
osd_num_t cluster_client_t::select_random_osd(const std::vector<osd_num_t> & osds)
{
osd_num_t alive_set[osds.size()];
int alive_count = 0;
for (auto & osd_num: osds)
{
if (!st_cli.peer_states[osd_num].is_null())
alive_set[alive_count++] = osd_num;
}
if (!alive_count)
return 0;
return alive_set[lrand48() % alive_count];
}
osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & osds)
{
if (!self_tree_metrics.size())
{
std::string cur_id = client_hostname;
int metric = 0;
while (self_tree_metrics.find(cur_id) == self_tree_metrics.end())
{
self_tree_metrics[cur_id] = metric++;
json11::Json cur_placement = st_cli.node_placement[cur_id];
cur_id = cur_placement["parent"].string_value();
}
if (cur_id != "")
{
self_tree_metrics[""] = metric++;
}
}
osd_num_t best_osd = 0;
int best_metric = -1;
for (auto & osd_num: osds)
{
int metric = -1;
auto met_it = osd_tree_metrics.find(osd_num);
if (met_it != osd_tree_metrics.end())
{
metric = met_it->second;
}
else
{
auto & peer_state = st_cli.peer_states[osd_num];
if (!peer_state.is_null())
{
metric = self_tree_metrics[""];
bool first = true;
std::string cur_id = std::to_string(osd_num);
std::set<std::string> seen;
while (seen.find(cur_id) == seen.end())
{
seen.insert(cur_id);
json11::Json cur_placement = st_cli.node_placement[cur_id];
std::string cur_parent = cur_placement["parent"].string_value();
cur_id = (!first || cur_parent != "" ? cur_parent : peer_state["host"].string_value());
first = false;
auto self_it = self_tree_metrics.find(cur_id);
if (self_it != self_tree_metrics.end())
{
metric = self_it->second;
break;
}
}
}
osd_tree_metrics[osd_num] = metric;
}
if (metric >= 0 && (best_metric < 0 || metric < best_metric))
{
best_metric = metric;
best_osd = osd_num;
}
}
return best_osd;
}
void cluster_client_t::on_load_pgs_hook(bool success)
{
for (auto pool_item: st_cli.pool_config)
@@ -546,6 +632,7 @@ bool cluster_client_t::get_immediate_commit(uint64_t inode)
void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
{
osd_tree_metrics.erase(peer_osd);
if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
{
msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
@@ -553,6 +640,12 @@ void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
}
}
void cluster_client_t::on_change_node_placement_hook()
{
osd_tree_metrics.clear();
self_tree_metrics.clear();
}
bool cluster_client_t::is_ready()
{
return pgs_loaded;
@@ -1221,6 +1314,17 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
!pg_it->second.pause && pg_it->second.cur_primary)
{
osd_num_t primary_osd = pg_it->second.cur_primary;
if (pool_cfg.local_reads != POOL_LOCAL_READ_PRIMARY &&
pool_cfg.scheme == POOL_SCHEME_REPLICATED &&
(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) &&
(pg_it->second.cur_state == PG_ACTIVE || pg_it->second.cur_state == (PG_ACTIVE|PG_LEFT_ON_DEAD)))
{
osd_num_t nearest_osd = pool_cfg.local_reads == POOL_LOCAL_READ_NEAREST
? select_nearest_osd(pg_it->second.target_set)
: select_random_osd(pg_it->second.target_set);
if (nearest_osd)
primary_osd = nearest_osd;
}
part->osd_num = primary_osd;
auto peer_it = msgr.osd_peer_fds.find(primary_osd);
if (peer_it != msgr.osd_peer_fds.end())
src/client/cluster_client.h
+11
View File
@@ -100,6 +100,7 @@ public:
uint64_t client_max_buffered_bytes = 0;
uint64_t client_max_buffered_ops = 0;
uint64_t client_max_writeback_iodepth = 0;
std::string conf_hostname;
int log_level = 0;
int client_retry_interval = 50; // ms
@@ -107,6 +108,10 @@ public:
bool client_retry_enospc = true;
int client_wait_up_timeout = 16; // sec (for listings)
std::string client_hostname;
std::map<std::string, int> self_tree_metrics;
std::map<osd_num_t, int> osd_tree_metrics;
int retry_timeout_id = -1;
int retry_timeout_duration = 0;
std::vector<cluster_op_t*> offline_ops;
@@ -161,11 +166,14 @@ protected:
protected:
bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);
void on_load_config_hook(json11::Json::object & config);
void on_load_pgs_hook(bool success);
void on_change_pool_config_hook();
void on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary);
void on_change_osd_state_hook(uint64_t peer_osd);
void on_change_node_placement_hook();
void execute_internal(cluster_op_t *op);
void unshift_op(cluster_op_t *op);
int continue_rw(cluster_op_t *op);
@@ -191,5 +199,8 @@ protected:
bool check_finish_listing(inode_list_t *lst);
void continue_raw_ops(osd_num_t peer_osd);
osd_num_t select_random_osd(const std::vector<osd_num_t> & osds);
osd_num_t select_nearest_osd(const std::vector<osd_num_t> & osds);
friend class writeback_cache_t;
};
src/client/etcd_state_client.cpp
+20
View File
@@ -922,6 +922,19 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
pc.used_for_app = "fs:"+pc.used_for_app;
else
pc.used_for_app = pool_item.second["used_for_app"].as_string();
// Local Read Configuration
std::string local_reads = pool_item.second["local_reads"].string_value();
if (local_reads == "nearest")
pc.local_reads = POOL_LOCAL_READ_NEAREST;
else if (local_reads == "random")
pc.local_reads = POOL_LOCAL_READ_RANDOM;
else if (local_reads == "" || local_reads == "primary")
pc.local_reads = POOL_LOCAL_READ_PRIMARY;
else
{
pc.local_reads = POOL_LOCAL_READ_PRIMARY;
fprintf(stderr, "Warning: Pool %u has invalid local_reads, using 'primary'\n", pool_id);
}
// Immediate Commit Mode
pc.immediate_commit = pool_item.second["immediate_commit"].is_string()
? parse_immediate_commit(pool_item.second["immediate_commit"].string_value(), IMMEDIATE_ALL)
@@ -1256,6 +1269,13 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
}
}
}
else if (key == etcd_prefix+"/config/node_placement")
{
// <etcd_prefix>/config/node_placement
node_placement = value;
if (on_change_node_placement_hook)
on_change_node_placement_hook();
}
}
uint32_t etcd_state_client_t::parse_immediate_commit(const std::string & immediate_commit_str, uint32_t default_value)
src/client/etcd_state_client.h
+18 -11
View File
@@ -25,6 +25,10 @@
#define IMMEDIATE_ALL 2
#endif
#define POOL_LOCAL_READ_PRIMARY 0
#define POOL_LOCAL_READ_NEAREST 1
#define POOL_LOCAL_READ_RANDOM 2
struct etcd_kv_t
{
std::string key;
@@ -48,21 +52,22 @@ struct pg_config_t
struct pool_config_t
{
bool exists;
pool_id_t id;
bool exists = false;
pool_id_t id = 0;
std::string name;
uint64_t scheme;
uint64_t pg_size, pg_minsize, parity_chunks;
uint32_t data_block_size, bitmap_granularity, immediate_commit;
uint64_t pg_count;
uint64_t real_pg_count;
uint64_t scheme = 0;
uint64_t pg_size = 0, pg_minsize = 0, parity_chunks = 0;
uint32_t data_block_size = 0, bitmap_granularity = 0, immediate_commit = 0;
uint64_t pg_count = 0;
uint64_t real_pg_count = 0;
std::string failure_domain;
uint64_t max_osd_combinations;
uint64_t pg_stripe_size;
uint64_t max_osd_combinations = 0;
uint64_t pg_stripe_size = 0;
std::map<pg_num_t, pg_config_t> pg_config;
uint64_t scrub_interval;
uint64_t scrub_interval = 0;
std::string used_for_app;
int backfillfull;
int backfillfull = 0;
int local_reads = 0;
};
struct inode_config_t
@@ -130,6 +135,7 @@ public:
std::set<osd_num_t> seen_peers;
std::map<inode_t, inode_config_t> inode_config;
std::map<std::string, inode_t> inode_by_name;
json11::Json node_placement;
std::function<void(std::map<std::string, etcd_kv_t> &)> on_change_hook;
std::function<void(json11::Json::object &)> on_load_config_hook;
@@ -140,6 +146,7 @@ public:
std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
std::function<void(osd_num_t)> on_change_osd_state_hook;
std::function<void()> on_change_node_placement_hook;
std::function<void()> on_reload_hook;
std::function<void(inode_t, bool)> on_inode_change_hook;
std::function<void(http_co_t *)> on_start_watcher_hook;
src/cmd/cli.cpp
+1
View File
@@ -185,6 +185,7 @@ static const char* help_text =
" --immediate_commit all Put pool only on OSDs with this or larger immediate_commit (none < small < all)\n"
" --level_placement <rules> Use additional failure domain rules (example: \"dc=112233\")\n"
" --raw_placement <rules> Specify raw PG generation rules (see documentation for details)\n"
" --local_reads primary Local read policy for replicated pools: primary, nearest or random\n"
" --primary_affinity_tags tags Prefer to put primary copies on OSDs with all specified tags\n"
" --scrub_interval <time> Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y\n"
" --used_for_app fs:<name> Mark pool as used for VitastorFS with metadata in image <name>\n"
src/cmd/cli_pool_cfg.cpp
+19 -1
View File
@@ -91,7 +91,7 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
}
else if (key == "name" || key == "scheme" || key == "immediate_commit" ||
key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_app" ||
key == "used_for_fs" || key == "raw_placement")
key == "used_for_fs" || key == "raw_placement" || key == "local_reads")
{
if (!value.is_string())
{
@@ -165,6 +165,10 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
new_cfg["used_for_app"] = "fs:"+new_cfg["used_for_fs"].string_value();
new_cfg.erase("used_for_fs");
}
if (new_cfg.find("local_reads") != new_cfg.end() && new_cfg["local_reads"].string_value() == "primary")
{
new_cfg.erase("local_reads");
}
// Prevent autovivification of object keys. Now we don't modify the config, we just check it
json11::Json cfg = new_cfg;
@@ -340,5 +344,19 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
}
}
// local_reads
if (!cfg["local_reads"].is_null())
{
auto lr = cfg["local_reads"].string_value();
if (lr != "" && lr != "primary" && lr != "nearest" && lr != "random")
{
return "local_reads must be '', 'primary', 'nearest' or 'random', but it is "+cfg["local_reads"].string_value();
}
if (lr != "" && lr != "primary" && scheme != POOL_SCHEME_REPLICATED)
{
return "EC pools don't support localized reads, please clear local_reads or set it to 'primary'";
}
}
return "";
}
src/cmd/cli_pool_ls.cpp
+1
View File
@@ -504,6 +504,7 @@ resume_3:
{ "failure_domain", "Failure domain" },
{ "root_node", "Root node" },
{ "osd_tags_fmt", "OSD tags" },
{ "local_reads", "Local read policy" },
{ "primary_affinity_tags_fmt", "Primary affinity" },
{ "block_size_fmt", "Block size" },
{ "bitmap_granularity_fmt", "Bitmap granularity" },