thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Take PG history into account when starting PGs

Browse Source
This commit is contained in:
Vitaliy Filippov
2020-04-19 00:20:18 +03:00
parent 9126ffb0f9
commit f95299b769
5 changed files with 239 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
osd_cluster.cpp
+104 -29
View File
@@ -119,19 +119,54 @@ void osd_t::report_status()
});
}
void osd_t::consul_txn(json11::Json txn, std::function<void(std::string, json11::Json)> callback)
{
std::string req = txn.dump();
req = "PUT /v1/txn HTTP/1.1\r\n"
"Host: "+consul_host+"\r\n"
"Content-Type: application/json\r\n"
"Content-Length: "+std::to_string(req.size())+"\r\n"
"Connection: close\r\n"
"\r\n"+req;
http_request_json(consul_address, req, callback);
}
uint64_t stoull_full(std::string str, int base = 10)
{
if (isspace(str[0]))
{
return 0;
}
size_t end = -1;
uint64_t r = std::stoull(str, &end, base);
if (end < str.length())
{
return 0;
}
return r;
}
// Start -> Load PGs -> Load peers -> Connect to peers -> Peer PGs
// Wait for PG changes -> Start/Stop PGs when requested
// Peer connection is lost -> Reload connection data -> Try to reconnect -> Repeat
void osd_t::load_pgs()
{
assert(this->pgs.size() == 0);
std::string req = "GET /v1/kv/"+consul_prefix+"/config/pgs?raw"+
/*(consul_change_index > 0 ? "&index="+std::to_string(consul_change_index) : "")+*/
" HTTP/1.1\r\n"+
"Host: "+consul_host+"\r\n"+
"Connection: close\r\n"+
"\r\n";
http_request_json(consul_address, req, [this](std::string err, json11::Json data)
json11::Json::array txn = {
json11::Json::object {
{ "KV", json11::Json::object {
{ "Verb", "get" },
{ "Key", consul_prefix+"/config/pgs" },
} }
},
json11::Json::object {
{ "KV", json11::Json::object {
{ "Verb", "get-tree" },
{ "Key", consul_prefix+"/pg/history/" },
} }
},
};
consul_txn(txn, [this](std::string err, json11::Json data)
{
if (err != "")
{
@@ -142,19 +177,43 @@ void osd_t::load_pgs()
});
return;
}
parse_pgs(data);
json11::Json pg_config;
std::map<pg_num_t, json11::Json> pg_history;
for (auto & res: data["Results"].array_items())
{
std::string key = res["KV"]["Key"].string_value();
std::string json_err;
json11::Json value = json11::Json::parse(base64_decode(res["KV"]["Value"].string_value()), json_err);
if (json_err != "")
{
printf("Bad JSON in Consul key %s: %s\n", key.c_str(), json_err.c_str());
}
if (key == consul_prefix+"/config/pgs")
{
pg_config = value;
}
else
{
// <consul_prefix>/pg/history/%d.
pg_num_t pg_num = stoull_full(key.substr(consul_prefix.length()+13, key.length()-consul_prefix.length()-14));
if (pg_num)
{
pg_history[pg_num] = value;
}
}
}
parse_pgs(pg_config, pg_history);
peering_state = OSD_CONNECTING_PEERS;
});
}
void osd_t::parse_pgs(json11::Json data)
void osd_t::parse_pgs(const json11::Json & pg_config, const std::map<pg_num_t, json11::Json> & pg_history)
{
uint64_t pg_count = 0;
for (auto pg_item: data.object_items())
for (auto pg_item: pg_config.object_items())
{
char *pg_num_end = NULL;
pg_num_t pg_num = strtoull(pg_item.first.c_str(), &pg_num_end, 10);
if (!pg_num || *pg_num_end != 0)
pg_num_t pg_num = stoull_full(pg_item.first);
if (!pg_num)
{
throw std::runtime_error("Bad key in PG hash: "+pg_item.first);
}
@@ -163,27 +222,52 @@ void osd_t::parse_pgs(json11::Json data)
if (primary_osd == this->osd_num)
{
// Take this PG
std::set<osd_num_t> all_peers;
std::vector<osd_num_t> target_set;
for (auto pg_osd_num: pg_json["osd_set"].array_items())
{
osd_num_t pg_osd = pg_osd_num.uint64_value();
target_set.push_back(pg_osd);
if (pg_osd != 0)
{
all_peers.insert(pg_osd);
}
}
if (target_set.size() != 3)
{
throw std::runtime_error("Bad PG "+std::to_string(pg_num)+" config format: incorrect osd_set");
}
std::vector<std::vector<osd_num_t>> target_history;
auto hist_it = pg_history.find(pg_num);
if (hist_it != pg_history.end())
{
for (auto hist_item: hist_it->second.array_items())
{
std::vector<osd_num_t> history_set;
for (auto pg_osd_num: hist_item["osd_set"].array_items())
{
osd_num_t pg_osd = pg_osd_num.uint64_value();
history_set.push_back(pg_osd);
if (pg_osd != 0)
{
all_peers.insert(pg_osd);
}
}
target_history.push_back(history_set);
}
}
this->pgs[pg_num] = (pg_t){
.state = PG_PEERING,
.pg_cursize = 0,
.pg_num = pg_num,
.all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end()),
.target_history = target_history,
.target_set = target_set,
};
this->pgs[pg_num].print_state();
// Add peers
for (auto pg_osd: target_set)
for (auto pg_osd: all_peers)
{
// FIXME: Add OSDs from PG history to peers
if (pg_osd != this->osd_num && osd_peer_fds.find(pg_osd) == osd_peer_fds.end())
{
wanted_peers[pg_osd] = { 0 };
@@ -197,7 +281,7 @@ void osd_t::parse_pgs(json11::Json data)
void osd_t::load_and_connect_peers()
{
json11::Json::array consul_txn;
json11::Json::array load_peer_txn;
for (auto wp_it = wanted_peers.begin(); wp_it != wanted_peers.end();)
{
osd_num_t osd_num = wp_it->first;
@@ -217,7 +301,7 @@ void osd_t::load_and_connect_peers()
{
// (Re)load OSD state from Consul
wp_it->second.last_load_attempt = time(NULL);
consul_txn.push_back(json11::Json::object {
load_peer_txn.push_back(json11::Json::object {
{ "KV", json11::Json::object {
{ "Verb", "get-tree" },
{ "Key", consul_prefix+"/osd/state/"+std::to_string(osd_num)+"." },
@@ -255,7 +339,6 @@ void osd_t::load_and_connect_peers()
return;
}
printf("Connected with peer OSD %lu (fd %d)\n", clients[peer_fd].osd_num, peer_fd);
// FIXME: Check peer config after connecting
wanted_peers.erase(osd_num);
if (!wanted_peers.size())
{
@@ -263,7 +346,7 @@ void osd_t::load_and_connect_peers()
printf("Connected to all peers\n");
peering_state = peering_state & ~OSD_CONNECTING_PEERS;
}
repeer_pgs(osd_num, true);
repeer_pgs(osd_num);
});
}
else
@@ -272,17 +355,9 @@ void osd_t::load_and_connect_peers()
wp_it++;
}
}
if (consul_txn.size() > 0)
if (load_peer_txn.size() > 0)
{
std::string req = json11::Json(consul_txn).dump();
req = "PUT /v1/txn HTTP/1.1\r\n"
"Host: "+consul_host+"\r\n"
"Content-Type: application/json\r\n"
"Content-Length: "+std::to_string(req.size())+"\r\n"
"Connection: close\r\n"
"\r\n"+req;
loading_peer_config = true;
http_request_json(consul_address, req, [this](std::string err, json11::Json data)
consul_txn(load_peer_txn, [this](std::string err, json11::Json data)
{
loading_peer_config = false;
if (err != "")