thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement flushing (stabilize/rollback) of unstable entries on start of the PG

Browse Source
This commit is contained in:
Vitaliy Filippov
2020-03-14 02:49:34 +03:00
parent 46f9bd2a69
commit 21d0b06959
10 changed files with 275 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
osd_peering.cpp
+239 -23
View File
@@ -21,15 +21,15 @@ void osd_t::init_primary()
}
if (peers.size() < 2)
throw std::runtime_error("run_primary requires at least 2 peers");
pgs.push_back((pg_t){
pgs[1] = (pg_t){
.state = PG_OFFLINE,
.pg_cursize = 0,
.pg_num = 1,
.target_set = { 1, 2, 3 },
.cur_set = { 1, 0, 0 },
});
};
pg_count = 1;
peering_state = OSD_PEERING_PEERS;
peering_state = OSD_CONNECTING_PEERS;
}
osd_peer_def_t osd_t::parse_peer(std::string peer)
@@ -132,7 +132,7 @@ void osd_t::handle_connect_result(int peer_fd)
// Peering loop
void osd_t::handle_peers()
{
if (peering_state & OSD_PEERING_PEERS)
if (peering_state & OSD_CONNECTING_PEERS)
{
for (int i = 0; i < peers.size(); i++)
{
@@ -158,7 +158,7 @@ void osd_t::handle_peers()
if (i >= peers.size())
{
// Connected to all peers
peering_state = peering_state & ~OSD_PEERING_PEERS;
peering_state = peering_state & ~OSD_CONNECTING_PEERS;
}
repeer_pgs(osd_num, true);
});
@@ -167,27 +167,239 @@ void osd_t::handle_peers()
}
if (peering_state & OSD_PEERING_PGS)
{
bool still_doing_pgs = false;
for (int i = 0; i < pgs.size(); i++)
bool still = false;
for (auto & p: pgs)
{
if (pgs[i].state == PG_PEERING)
if (p.second.state == PG_PEERING)
{
if (!pgs[i].peering_state->list_ops.size())
if (!p.second.peering_state->list_ops.size())
{
pgs[i].calc_object_states();
p.second.calc_object_states();
if (p.second.state & PG_HAS_UNCLEAN)
{
peering_state = peering_state | OSD_FLUSHING_PGS;
}
}
else
{
still_doing_pgs = true;
still = true;
}
}
}
if (!still_doing_pgs)
if (!still)
{
// Done all PGs
peering_state = peering_state & ~OSD_PEERING_PGS;
}
}
if (peering_state & OSD_FLUSHING_PGS)
{
bool still = false;
for (auto & p: pgs)
{
if (p.second.state & PG_HAS_UNCLEAN)
{
if (!p.second.flush_batch)
{
submit_pg_flush_ops(p.first);
}
still = true;
}
}
if (!still)
{
peering_state = peering_state & ~OSD_FLUSHING_PGS;
}
}
}
#define FLUSH_BATCH 512
struct pg_flush_batch_t
{
std::map<osd_num_t, std::vector<obj_ver_id>> rollback_lists;
std::map<osd_num_t, std::vector<obj_ver_id>> stable_lists;
int flush_ops = 0, flush_done = 0;
int flush_objects = 0;
};
void osd_t::submit_pg_flush_ops(pg_num_t pg_num)
{
pg_t & pg = pgs[pg_num];
pg_flush_batch_t *fb = new pg_flush_batch_t();
pg.flush_batch = fb;
auto it = pg.flush_actions.begin(), prev_it = pg.flush_actions.begin();
bool first = true;
while (it != pg.flush_actions.end())
{
if (!first && (it->first.oid.inode != prev_it->first.oid.inode ||
(it->first.oid.stripe & ~STRIPE_MASK) != (prev_it->first.oid.stripe & ~STRIPE_MASK)) &&
fb->rollback_lists[it->first.osd_num].size() >= FLUSH_BATCH ||
fb->stable_lists[it->first.osd_num].size() >= FLUSH_BATCH)
{
// Stop only at the object boundary
break;
}
it->second.submitted = true;
if (it->second.rollback)
{
fb->flush_objects++;
fb->rollback_lists[it->first.osd_num].push_back((obj_ver_id){
.oid = it->first.oid,
.version = it->second.rollback_to,
});
}
if (it->second.make_stable)
{
fb->flush_objects++;
fb->stable_lists[it->first.osd_num].push_back((obj_ver_id){
.oid = it->first.oid,
.version = it->second.stable_to,
});
}
prev_it = it;
first = false;
it++;
}
for (auto & l: fb->rollback_lists)
{
if (l.second.size() > 0)
{
fb->flush_ops++;
submit_flush_op(pg.pg_num, fb, true, l.first, l.second.size(), l.second.data());
}
}
for (auto & l: fb->stable_lists)
{
if (l.second.size() > 0)
{
fb->flush_ops++;
submit_flush_op(pg.pg_num, fb, false, l.first, l.second.size(), l.second.data());
}
}
}
void osd_t::handle_flush_op(pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t osd_num, bool ok)
{
if (pgs.find(pg_num) == pgs.end() || pgs[pg_num].flush_batch != fb)
{
// Throw the result away
return;
}
if (!ok)
{
if (osd_num == this->osd_num)
throw std::runtime_error("Error while doing local flush operation");
else
{
assert(osd_peer_fds.find(osd_num) != osd_peer_fds.end());
stop_client(osd_peer_fds[osd_num]);
return;
}
}
fb->flush_done++;
if (fb->flush_done == fb->flush_ops)
{
// This flush batch is done
std::vector<osd_op_t*> continue_ops;
auto & pg = pgs[pg_num];
auto it = pg.flush_actions.begin(), prev_it = it;
auto erase_start = it;
while (1)
{
if (it == pg.flush_actions.end() ||
it->first.oid.inode != prev_it->first.oid.inode ||
(it->first.oid.stripe & ~STRIPE_MASK) != (prev_it->first.oid.stripe & ~STRIPE_MASK))
{
auto wr_it = pg.write_queue.find((object_id){
.inode = prev_it->first.oid.inode,
.stripe = (prev_it->first.oid.stripe & ~STRIPE_MASK),
});
if (wr_it != pg.write_queue.end())
{
continue_ops.push_back(wr_it->second);
pg.write_queue.erase(wr_it);
}
}
if ((it == pg.flush_actions.end() || !it->second.submitted) &&
erase_start != it)
{
pg.flush_actions.erase(erase_start, it);
}
if (it == pg.flush_actions.end())
{
break;
}
prev_it = it;
if (!it->second.submitted)
{
it++;
erase_start = it;
}
else
{
it++;
}
}
delete fb;
pg.flush_batch = NULL;
if (!pg.flush_actions.size())
{
pg.state = pg.state & ~PG_HAS_UNCLEAN;
}
for (osd_op_t *op: continue_ops)
{
continue_primary_write(op);
}
}
}
void osd_t::submit_flush_op(pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t osd_num, int count, obj_ver_id *data)
{
osd_op_t *op = new osd_op_t();
// Copy buffer so it gets freed along with the operation
op->buf = malloc(sizeof(obj_ver_id) * count);
memcpy(op->buf, data, sizeof(obj_ver_id) * count);
if (osd_num == this->osd_num)
{
// local
op->bs_op = new blockstore_op_t({
.opcode = (uint64_t)(rollback ? BS_OP_ROLLBACK : BS_OP_STABLE),
.callback = [this, op, pg_num, fb](blockstore_op_t *bs_op)
{
handle_flush_op(pg_num, fb, this->osd_num, bs_op->retval == 0);
delete op;
},
.len = (uint32_t)count,
.buf = op->buf,
});
bs->enqueue_op(op->bs_op);
}
else
{
// Peer
int peer_fd = osd_peer_fds[osd_num];
op->op_type = OSD_OP_OUT;
op->send_list.push_back(op->req.buf, OSD_PACKET_SIZE);
op->send_list.push_back(op->buf, count * sizeof(obj_ver_id));
op->peer_fd = peer_fd;
op->req = {
.sec_stab = {
.header = {
.magic = SECONDARY_OSD_OP_MAGIC,
.id = this->next_subop_id++,
.opcode = (uint64_t)(rollback ? OSD_OP_SECONDARY_ROLLBACK : OSD_OP_SECONDARY_STABILIZE),
},
.len = count * sizeof(obj_ver_id),
},
};
op->callback = [this, pg_num, fb](osd_op_t *op)
{
handle_flush_op(pg_num, fb, clients[op->peer_fd].osd_num, op->reply.hdr.retval == 0);
delete op;
};
outbox_push(clients[peer_fd], op);
}
}
void osd_t::repeer_pgs(osd_num_t osd_num, bool is_connected)
@@ -195,15 +407,15 @@ void osd_t::repeer_pgs(osd_num_t osd_num, bool is_connected)
// Re-peer affected PGs
// FIXME: We shouldn't rely just on target_set. Other OSDs may also contain PG data.
osd_num_t real_osd = (is_connected ? osd_num : 0);
for (int i = 0; i < pgs.size(); i++)
for (auto & p: pgs)
{
bool repeer = false;
for (int r = 0; r < pgs[i].target_set.size(); r++)
for (int r = 0; r < p.second.target_set.size(); r++)
{
if (pgs[i].target_set[r] == osd_num &&
pgs[i].cur_set[r] != real_osd)
if (p.second.target_set[r] == osd_num &&
p.second.cur_set[r] != real_osd)
{
pgs[i].cur_set[r] = real_osd;
p.second.cur_set[r] = real_osd;
repeer = true;
break;
}
@@ -211,21 +423,25 @@ void osd_t::repeer_pgs(osd_num_t osd_num, bool is_connected)
if (repeer)
{
// Repeer this pg
printf("Repeer PG %d because of OSD %lu\n", i, osd_num);
start_pg_peering(i);
printf("Repeer PG %d because of OSD %lu\n", p.second.pg_num, osd_num);
start_pg_peering(p.second.pg_num);
peering_state |= OSD_PEERING_PGS;
}
}
}
// Repeer on each connect/disconnect peer event
void osd_t::start_pg_peering(int pg_idx)
void osd_t::start_pg_peering(pg_num_t pg_num)
{
auto & pg = pgs[pg_idx];
auto & pg = pgs[pg_num];
pg.state = PG_PEERING;
pg.state_dict.clear();
pg.obj_states.clear();
pg.ver_override.clear();
pg.flush_actions.clear();
if (pg.flush_batch)
delete pg.flush_batch;
pg.flush_batch = NULL;
pg.pg_cursize = 0;
for (int role = 0; role < pg.cur_set.size(); role++)
{
@@ -330,8 +546,8 @@ void osd_t::start_pg_peering(int pg_idx)
op->bs_op = new blockstore_op_t();
op->bs_op->opcode = BS_OP_LIST;
op->bs_op->oid.stripe = parity_block_size;
op->bs_op->len = pg_count,
op->bs_op->offset = pg.pg_num-1,
op->bs_op->len = pg_count;
op->bs_op->offset = pg.pg_num-1;
op->bs_op->callback = [ps, op, role_osd](blockstore_op_t *bs_op)
{
if (op->bs_op->retval < 0)