thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use surrogate peer IDs instead of FDs

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-03-30 02:06:10 +03:00
parent 79141eb383
commit 07915c2881
24 changed files with 324 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/osd/osd.cpp
+2 -2
View File
@@ -616,11 +616,11 @@ void osd_t::print_slow()
bufprintf("[OSD %ju] Slow op %jx", osd_num, (uint64_t)op);
if (kv.second->osd_num)
{
bufprintf(" from peer OSD %ju (client %d)", kv.second->osd_num, kv.second->peer_fd);
bufprintf(" from peer OSD %ju (client %ju)", kv.second->osd_num, kv.second->client_id);
}
else
{
bufprintf(" from client %d", kv.second->peer_fd);
bufprintf(" from client %ju", kv.second->client_id);
}
bufprintf(": %s id=%ju", osd_op_names[op->req.hdr.opcode], op->req.hdr.id);
if (op->req.hdr.opcode == OSD_OP_SEC_READ || op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
src/osd/osd_cluster.cpp
+1 -1
View File
@@ -991,7 +991,7 @@ void osd_t::apply_pg_config()
// Add peers
for (auto pg_osd: all_peers)
{
if (pg_osd != this->osd_num && msgr.osd_peer_fds.find(pg_osd) == msgr.osd_peer_fds.end())
if (pg_osd != this->osd_num && msgr.osd_peers.find(pg_osd) == msgr.osd_peers.end())
{
msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
}
src/osd/osd_flush.cpp
+8 -7
View File
@@ -4,6 +4,7 @@
#include "osd.h"
#define FLUSH_BATCH 512
#define SELF_CLIENT 0
void osd_t::submit_pg_flush_ops(pg_t & pg)
{
@@ -91,11 +92,11 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
else
{
printf("Error while doing flush on OSD %ju: %d (%s)\n", osd_num, retval, strerror(-retval));
auto fd_it = msgr.osd_peer_fds.find(peer_osd);
if (fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(peer_osd);
if (peer_it != msgr.osd_peers.end())
{
// Will repeer/stop this PG
msgr.stop_client(fd_it->second);
msgr.stop_client(peer_it->second->client_id);
}
}
}
@@ -212,10 +213,10 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
handle_flush_op(op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK, pool_id, pg_num, fb, peer_osd, op->reply.hdr.retval);
delete op;
};
auto peer_fd_it = msgr.osd_peer_fds.find(peer_osd);
if (peer_fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(peer_osd);
if (peer_it != msgr.osd_peers.end())
{
op->peer_fd = peer_fd_it->second;
op->client_id = peer_it->second->client_id;
msgr.outbox_push(op);
}
else
@@ -307,7 +308,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
{
printf("Submitting recovery operation for %jx:%jx (%s)\n", op->oid.inode, op->oid.stripe, op->degraded ? "degraded" : "misplaced");
}
op->osd_op->peer_fd = -1;
op->osd_op->client_id = SELF_CLIENT;
op->osd_op->callback = [this, op](osd_op_t *osd_op)
{
ringloop->set_immediate([this, op]()
src/osd/osd_peering.cpp
+20 -21
View File
@@ -9,7 +9,7 @@
#include "str_util.h"
#include "osd.h"
#define SELF_FD -1
#define SELF_CLIENT 0
// Peering loop
void osd_t::handle_peers()
@@ -175,17 +175,17 @@ void osd_t::drop_dirty_pg_connections(pool_pg_num_t pg)
{
if (immediate_commit != IMMEDIATE_ALL)
{
std::vector<int> to_stop;
std::vector<uint64_t> to_stop;
for (auto & cp: msgr.clients)
{
if (cp.second->dirty_pgs.find(pg) != cp.second->dirty_pgs.end())
{
to_stop.push_back(cp.first);
to_stop.push_back(cp.second->client_id);
}
}
for (auto peer_fd: to_stop)
for (auto client_id: to_stop)
{
msgr.stop_client(peer_fd);
msgr.stop_client(client_id);
}
}
}
@@ -203,7 +203,7 @@ void osd_t::start_pg_peering(pg_t & pg)
for (auto pg_osd: pg.all_peers)
{
if (pg_osd != this->osd_num &&
msgr.osd_peer_fds.find(pg_osd) == msgr.osd_peer_fds.end())
msgr.osd_peers.find(pg_osd) == msgr.osd_peers.end())
{
if (msgr.wanted_peers.find(pg_osd) == msgr.wanted_peers.end())
msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
@@ -224,7 +224,7 @@ void osd_t::start_pg_peering(pg_t & pg)
for (int role = 0; role < pg.target_set.size(); role++)
{
pg.cur_set[role] = pg.target_set[role] == this->osd_num ||
msgr.osd_peer_fds.find(pg.target_set[role]) != msgr.osd_peer_fds.end() ? pg.target_set[role] : 0;
msgr.osd_peers.find(pg.target_set[role]) != msgr.osd_peers.end() ? pg.target_set[role] : 0;
if (pg.cur_set[role] != 0)
{
pg.pg_cursize++;
@@ -246,7 +246,7 @@ void osd_t::start_pg_peering(pg_t & pg)
std::set<osd_num_t> dead_peers;
for (auto pg_osd: pg.all_peers)
{
if (pg_osd == this->osd_num || msgr.osd_peer_fds.find(pg_osd) != msgr.osd_peer_fds.end())
if (pg_osd == this->osd_num || msgr.osd_peers.find(pg_osd) != msgr.osd_peers.end())
cur_peers.insert(pg_osd);
else
dead_peers.insert(pg_osd);
@@ -266,7 +266,7 @@ void osd_t::start_pg_peering(pg_t & pg)
{
nonzero++;
if (history_osd == this->osd_num ||
msgr.osd_peer_fds.find(history_osd) != msgr.osd_peer_fds.end())
msgr.osd_peers.find(history_osd) != msgr.osd_peers.end())
{
found++;
}
@@ -435,8 +435,8 @@ void osd_t::relock_pg(pg_t & pg)
bool unlock_peer = (i >= relock_osd_count);
uint64_t new_state = unlock_peer ? 0 : pg.state;
auto peer_osd = diff_osds[i];
auto peer_fd_it = msgr.osd_peer_fds.find(peer_osd);
if (peer_fd_it == msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(peer_osd);
if (peer_it == msgr.osd_peers.end())
{
if (unlock_peer)
{
@@ -446,8 +446,7 @@ void osd_t::relock_pg(pg_t & pg)
}
continue;
}
int peer_fd = peer_fd_it->second;
auto cl = msgr.clients.at(peer_fd);
auto cl = peer_it->second;
if (!cl->enable_pg_locks)
{
// Peer does not support locking - just instantly remember the lock as successful
@@ -458,7 +457,7 @@ void osd_t::relock_pg(pg_t & pg)
pg.inflight_locks++;
osd_op_t *op = new osd_op_t();
op->op_type = OSD_OP_OUT;
op->peer_fd = peer_fd;
op->client_id = cl->client_id;
op->req = (osd_any_op_t){
.sec_lock = {
.header = {
@@ -529,7 +528,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
// Self
osd_op_t *op = new osd_op_t();
op->op_type = 0;
op->peer_fd = SELF_FD;
op->client_id = SELF_CLIENT;
clock_gettime(CLOCK_REALTIME, &op->tv_begin);
op->bs_op = new blockstore_op_t();
op->bs_op->opcode = BS_OP_LIST;
@@ -567,8 +566,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
}
else
{
auto role_fd_it = msgr.osd_peer_fds.find(role_osd);
if (role_fd_it == msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(role_osd);
if (peer_it == msgr.osd_peers.end())
{
printf("Failed to get object list from OSD %ju because it is disconnected\n", role_osd);
return;
@@ -576,7 +575,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
// Peer
osd_op_t *op = new osd_op_t();
op->op_type = OSD_OP_OUT;
op->peer_fd = role_fd_it->second;
op->client_id = peer_it->second->client_id;
op->req = (osd_any_op_t){
.sec_list = {
.header = {
@@ -595,10 +594,10 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
if (op->reply.hdr.retval < 0)
{
printf("Failed to get object list from OSD %ju (retval=%jd), disconnecting peer\n", role_osd, op->reply.hdr.retval);
int fail_fd = op->peer_fd;
uint64_t fail_client_id = op->client_id;
ps->list_ops.erase(role_osd);
delete op;
msgr.stop_client(fail_fd);
msgr.stop_client(fail_client_id);
return;
}
printf(
@@ -622,7 +621,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
void osd_t::discard_list_subop(osd_op_t *list_op)
{
if (list_op->peer_fd == SELF_FD)
if (list_op->client_id == SELF_CLIENT)
{
// Self
list_op->bs_op->callback = [list_op](blockstore_op_t *bs_op)
src/osd/osd_primary.cpp
+1 -1
View File
@@ -795,7 +795,7 @@ resume_5:
{
this->dirty_osds.insert(chunk.osd_num);
}
for (auto cl_it = msgr.clients.find(cur_op->peer_fd); cl_it != msgr.clients.end(); )
for (auto cl_it = msgr.clients.find(cur_op->client_id); cl_it != msgr.clients.end(); )
{
cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
break;
src/osd/osd_primary_chain.cpp
+5 -5
View File
@@ -312,16 +312,16 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
}
handle_primary_subop(subop, cur_op);
};
auto peer_fd_it = msgr.osd_peer_fds.find(subop_osd_num);
if (peer_fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(subop_osd_num);
if (peer_it != msgr.osd_peers.end())
{
subop->peer_fd = peer_fd_it->second;
subop->client_id = peer_it->second->client_id;
msgr.outbox_push(subop);
}
else
{
// Fail it immediately
subop->peer_fd = -1;
subop->client_id = 0;
subop->reply.hdr.retval = -EPIPE;
ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
}
@@ -607,7 +607,7 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
{
if (cur > prev)
{
// Send buffer in parts to avoid copying
// Send buffer in parts to avoid copying
if (!prev_set)
{
while ((cur-prev) > zero_buffer_size/bs_bitmap_granularity)
src/osd/osd_primary_subops.cpp
+45 -41
View File
@@ -3,7 +3,7 @@
#include "osd_primary.h"
#define SELF_FD -1
#define SELF_CLIENT 0
void osd_t::autosync()
{
@@ -15,7 +15,7 @@ void osd_t::autosync()
}
autosync_op = new osd_op_t();
autosync_op->op_type = OSD_OP_IN;
autosync_op->peer_fd = SELF_FD;
autosync_op->client_id = SELF_CLIENT;
autosync_op->req = (osd_any_op_t){
.sync = {
.header = {
@@ -90,7 +90,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
cur_op->reply.hdr.id = cur_op->req.hdr.id;
cur_op->reply.hdr.opcode = cur_op->req.hdr.opcode;
cur_op->reply.hdr.retval = retval;
if (cur_op->peer_fd == SELF_FD)
if (cur_op->client_id == SELF_CLIENT)
{
// Do not include internal primary writes (recovery/rebalance) into client op statistics
if (cur_op->req.hdr.opcode != OSD_OP_WRITE)
@@ -103,7 +103,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
else
{
// FIXME add separate magic number for primary ops
auto cl_it = msgr.clients.find(cur_op->peer_fd);
auto cl_it = msgr.clients.find(cur_op->client_id);
if (cl_it != msgr.clients.end())
{
msgr.outbox_push(cur_op);
@@ -243,7 +243,7 @@ void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
.offset = wr ? si->write_start : si->read_start,
.len = subop_len,
.attr_len = wr ? clean_entry_bitmap_size : 0,
.flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
.flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
};
#ifdef OSD_DEBUG
printf(
@@ -270,16 +270,16 @@ void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
{
handle_primary_subop(subop, cur_op);
};
auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num);
if (peer_fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(si->osd_num);
if (peer_it != msgr.osd_peers.end())
{
subop->peer_fd = peer_fd_it->second;
subop->client_id = peer_it->second->client_id;
msgr.outbox_push(subop);
}
else
{
// Fail it immediately
subop->peer_fd = -1;
subop->client_id = 0;
subop->reply.hdr.retval = -EPIPE;
ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
}
@@ -326,7 +326,7 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
}
throw std::runtime_error("local blockstore modification failed");
}
bool recovery_related = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB;
bool recovery_related = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB;
add_bs_subop_stats(subop, recovery_related);
subop->req.hdr.opcode = bs_op_to_osd_op[bs_op->opcode];
subop->reply.hdr.retval = bs_op->retval;
@@ -339,7 +339,7 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
}
delete bs_op;
subop->bs_op = NULL;
subop->peer_fd = SELF_FD;
subop->client_id = SELF_CLIENT;
if (recovery_related && recovery_target_sleep_us)
{
tfd->set_timer_us(recovery_target_sleep_us, false, [=](int timer_id)
@@ -399,11 +399,15 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
{
uint64_t version = subop->reply.sec_rw.version;
#ifdef OSD_DEBUG
int64_t peer_osd = subop->peer_fd == SELF_FD ? osd_num :
(msgr.clients.find(subop->peer_fd) != msgr.clients.end()
? msgr.clients[subop->peer_fd]->osd_num : -subop->peer_fd);
printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version);
if (subop->client_id == SELF_CLIENT)
printf("subop %s %jx:%jx from local: version = %ju\n", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, version);
else if (msgr.clients.find(subop->client_id) != msgr.clients.end())
printf("subop %s %jx:%jx from osd %ju: version = %ju\n", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, msgr.clients.at(subop->client_id)->osd_num, version);
else
printf("subop %s %jx:%jx from client %ju: version = %ju\n", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->client_id, version);
#endif
if (version != 0 && op_data->fact_ver != UINT64_MAX)
{
@@ -422,16 +426,16 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
}
if (retval != expected)
{
int64_t peer_osd = (msgr.clients.find(subop->peer_fd) != msgr.clients.end()
? msgr.clients[subop->peer_fd]->osd_num : 0);
int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end()
? msgr.clients.at(subop->client_id)->osd_num : 0);
if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
{
printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode],
subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version);
if (subop->peer_fd >= 0 && peer_osd > 0)
if (subop->client_id && peer_osd > 0)
printf("on osd %ju: retval = %d (expected %d)\n", peer_osd, retval, expected);
else if (peer_osd > 0)
printf("on peer %d: retval = %d (expected %d)\n", subop->peer_fd, retval, expected);
else if (subop->client_id)
printf("on client %ju: retval = %d (expected %d)\n", subop->client_id, retval, expected);
else
printf("locally: retval = %d (expected %d)\n", retval, expected);
}
@@ -446,7 +450,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
else
{
printf(
"%s subop failed on osd %jd: retval = %d (expected %d)\n",
"%s subop failed on osd %ju: retval = %d (expected %d)\n",
osd_op_names[opcode], peer_osd, retval, expected
);
}
@@ -460,12 +464,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
{
op_data->errcode = retval;
}
if (subop->peer_fd >= 0 && retval != -EDOM && retval != -ERANGE &&
if (subop->client_id && retval != -EDOM && retval != -ERANGE &&
(retval != -ENOSPC || opcode != OSD_OP_SEC_WRITE && opcode != OSD_OP_SEC_WRITE_STABLE) &&
(retval != -EIO || opcode != OSD_OP_SEC_READ))
{
// Drop connection on unexpected errors
msgr.stop_client(subop->peer_fd);
msgr.stop_client(subop->client_id);
op_data->drops++;
}
// Increase op_data->errors after stop_client to prevent >= n_subops running twice
@@ -606,22 +610,22 @@ void osd_t::submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_
},
.oid = chunk.oid,
.version = chunk.version,
.flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
.flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
} };
subops[i].callback = [cur_op, this](osd_op_t *subop)
{
handle_primary_subop(subop, cur_op);
};
auto peer_fd_it = msgr.osd_peer_fds.find(chunk.osd_num);
if (peer_fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(chunk.osd_num);
if (peer_it != msgr.osd_peers.end())
{
subops[i].peer_fd = peer_fd_it->second;
subops[i].client_id = peer_it->second->client_id;
msgr.outbox_push(&subops[i]);
}
else
{
// Fail it immediately
subops[i].peer_fd = -1;
subops[i].client_id = 0;
subops[i].reply.hdr.retval = -EPIPE;
ringloop->set_immediate([subop = &subops[i]]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
}
@@ -637,7 +641,7 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
op_data->done = op_data->errors = op_data->errcode = 0;
op_data->n_subops = n_osds;
op_data->subops = subops;
std::map<uint64_t, int>::iterator peer_it;
std::map<uint64_t, osd_client_t*>::iterator peer_it;
for (int i = 0; i < n_osds; i++)
{
osd_num_t sync_osd = op_data->dirty_osds[i];
@@ -654,16 +658,16 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
});
bs->enqueue_op(subops[i].bs_op);
}
else if ((peer_it = msgr.osd_peer_fds.find(sync_osd)) != msgr.osd_peer_fds.end())
else if ((peer_it = msgr.osd_peers.find(sync_osd)) != msgr.osd_peers.end())
{
subops[i].op_type = OSD_OP_OUT;
subops[i].peer_fd = peer_it->second;
subops[i].client_id = peer_it->second->client_id;
subops[i].req = (osd_any_op_t){ .sec_sync = {
.header = {
.magic = SECONDARY_OSD_OP_MAGIC,
.opcode = OSD_OP_SEC_SYNC,
},
.flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
.flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
} };
subops[i].callback = [cur_op, this](osd_op_t *subop)
{
@@ -722,23 +726,23 @@ void osd_t::submit_primary_stab_subops(osd_op_t *cur_op)
.opcode = OSD_OP_SEC_STABILIZE,
},
.len = (uint64_t)(stab_osd.len * sizeof(obj_ver_id)),
.flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
.flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
} };
subops[i].iov.push_back(op_data->unstable_writes + stab_osd.start, stab_osd.len * sizeof(obj_ver_id));
subops[i].callback = [cur_op, this](osd_op_t *subop)
{
handle_primary_subop(subop, cur_op);
};
auto peer_fd_it = msgr.osd_peer_fds.find(stab_osd.osd_num);
if (peer_fd_it != msgr.osd_peer_fds.end())
auto peer_it = msgr.osd_peers.find(stab_osd.osd_num);
if (peer_it != msgr.osd_peers.end())
{
subops[i].peer_fd = peer_fd_it->second;
subops[i].client_id = peer_it->second->client_id;
msgr.outbox_push(&subops[i]);
}
else
{
// Fail it immediately
subops[i].peer_fd = -1;
subops[i].client_id = 0;
subops[i].reply.hdr.retval = -EPIPE;
ringloop->set_immediate([subop = &subops[i]]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
}
@@ -756,7 +760,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
for (int role = 0; role < op_data->pg->pg_size; role++)
{
if (osd_set[role] != 0 && !stripes[role].read_error &&
(osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
(osd_set[role] == this->osd_num || msgr.osd_peers.find(osd_set[role]) != msgr.osd_peers.end()))
{
n_subops++;
}
@@ -773,7 +777,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
for (int role = 0; role < op_data->pg->pg_size; role++)
{
if (osd_set[role] != 0 && !stripes[role].read_error &&
(osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
(osd_set[role] == this->osd_num || msgr.osd_peers.find(osd_set[role]) != msgr.osd_peers.end()))
{
osd_op_t *subop = &op_data->subops[i];
op_data->unstable_writes[i] = (obj_ver_id){
@@ -827,7 +831,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
op_data->oid.inode, op_data->oid.stripe | role, op_data->target_ver-1
);
#endif
subop->peer_fd = msgr.osd_peer_fds.at(osd_set[role]);
subop->client_id = msgr.osd_peers.at(osd_set[role])->client_id;
msgr.outbox_push(subop);
}
i++;
src/osd/osd_primary_sync.cpp
+2 -2
View File
@@ -247,9 +247,9 @@ resume_8:
else
{
finish:
if (cur_op->peer_fd)
if (cur_op->client_id)
{
auto it = msgr.clients.find(cur_op->peer_fd);
auto it = msgr.clients.find(cur_op->client_id);
if (it != msgr.clients.end())
it->second->dirty_pgs.clear();
}
src/osd/osd_primary_write.cpp
+1 -1
View File
@@ -545,7 +545,7 @@ lazy:
}
// Remember PG as dirty to drop the connection when PG goes offline
// (this is required because of the "lazy sync")
auto cl_it = msgr.clients.find(cur_op->peer_fd);
auto cl_it = msgr.clients.find(cur_op->client_id);
if (cl_it != msgr.clients.end())
{
cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
src/osd/osd_scrub.cpp
+6 -6
View File
@@ -3,7 +3,7 @@
#include "osd_primary.h"
#define SELF_FD -1
#define SELF_CLIENT 0
void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oid)
{
@@ -16,7 +16,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
// Self
osd_op_t *op = new osd_op_t();
op->op_type = 0;
op->peer_fd = SELF_FD;
op->client_id = SELF_CLIENT;
clock_gettime(CLOCK_REALTIME, &op->tv_begin);
op->bs_op = new blockstore_op_t();
op->bs_op->opcode = BS_OP_LIST;
@@ -61,7 +61,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
// Peer
osd_op_t *op = new osd_op_t();
op->op_type = OSD_OP_OUT;
op->peer_fd = msgr.osd_peer_fds.at(role_osd);
op->client_id = msgr.osd_peers.at(role_osd)->client_id;
op->req = (osd_any_op_t){
.sec_list = {
.header = {
@@ -83,9 +83,9 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
if (op->reply.hdr.retval < 0)
{
printf("Failed to get object list from OSD %ju (retval=%jd), disconnecting peer\n", role_osd, op->reply.hdr.retval);
int fail_fd = op->peer_fd;
uint64_t fail_client_id = op->client_id;
delete op;
msgr.stop_client(fail_fd);
msgr.stop_client(fail_client_id);
return;
}
scrub_cur_list = {
@@ -224,7 +224,7 @@ void osd_t::submit_scrub_op(object_id oid)
{
auto osd_op = new osd_op_t();
osd_op->op_type = OSD_OP_OUT;
osd_op->peer_fd = -1;
osd_op->client_id = SELF_CLIENT;
osd_op->req = (osd_any_op_t){
.rw = {
.header = {
src/osd/osd_secondary.cpp
+5 -5
View File
@@ -128,7 +128,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
exec_sec_lock(cur_op);
return;
}
auto cl = msgr.clients.at(cur_op->peer_fd);
osd_client_t *cl = msgr.clients.at(cur_op->client_id);
cur_op->bs_op = new blockstore_op_t();
cur_op->bs_op->callback = [this, cur_op](blockstore_op_t* bs_op) { secondary_op_callback(cur_op); };
cur_op->bs_op->opcode = (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ? BS_OP_READ
@@ -249,7 +249,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
{
auto cl = msgr.clients.at(cur_op->peer_fd);
auto cl = msgr.clients.at(cur_op->client_id);
int n = cur_op->req.sec_read_bmp.len / sizeof(obj_ver_id);
if (n > 0)
{
@@ -278,7 +278,7 @@ void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
void osd_t::exec_sec_lock(osd_op_t *cur_op)
{
cur_op->reply.sec_lock.cur_primary = 0;
auto cl = msgr.clients.at(cur_op->peer_fd);
auto cl = msgr.clients.at(cur_op->client_id);
if (!cl->in_osd_num ||
cur_op->req.sec_lock.flags != OSD_SEC_LOCK_PG &&
cur_op->req.sec_lock.flags != OSD_SEC_UNLOCK_PG ||
@@ -340,7 +340,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
? json11::Json::parse(std::string((char *)cur_op->buf), json_err)
: json11::Json();
auto peer_osd_num = req_json["osd_num"].uint64_value();
auto cl = msgr.clients.at(cur_op->peer_fd);
auto cl = msgr.clients.at(cur_op->client_id);
cl->in_osd_num = peer_osd_num;
if (req_json["features"]["check_sequencing"].bool_value())
{
@@ -369,7 +369,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
if (req_json["connect_rdma"].is_string())
{
// Peer is trying to connect using RDMA, try to satisfy him
bool ok = msgr.connect_rdma(cur_op->peer_fd, req_json["connect_rdma"].string_value(), req_json["rdma_max_msg"].uint64_value());
bool ok = msgr.connect_rdma(cur_op->client_id, req_json["connect_rdma"].string_value(), req_json["rdma_max_msg"].uint64_value());
if (ok)
{
auto rc = cl->rdma_conn;