thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(Almost) Implement misplaced recovery, integrating it into calc_rmw()

Browse Source
This commit is contained in:
Vitaliy Filippov
2020-04-05 15:50:53 +03:00
parent 6212195440
commit cf7de0f181
5 changed files with 274 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
osd_primary.cpp
+14 -91
View File
@@ -31,8 +31,9 @@ struct osd_primary_op_data_t
int degraded = 0, pg_size, pg_minsize;
osd_rmw_stripe_t *stripes;
osd_op_t *subops = NULL;
void *recovery_buf = NULL;
uint64_t *prev_set = NULL;
uint64_t object_state = 0;
// for sync. oops, requires freeing
std::vector<unstable_osd_num_t> *unstable_write_osds = NULL;
pg_num_t *dirty_pgs = NULL;
@@ -117,27 +118,32 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
return true;
}
uint64_t* get_object_osd_set(pg_t &pg, object_id &oid, uint64_t *def)
uint64_t* get_object_osd_set(pg_t &pg, object_id &oid, uint64_t *def, uint64_t &object_state)
{
if (!(pg.state & (PG_HAS_INCOMPLETE | PG_HAS_DEGRADED | PG_HAS_MISPLACED)))
{
object_state = 0;
return def;
}
auto st_it = pg.incomplete_objects.find(oid);
if (st_it != pg.incomplete_objects.end())
{
object_state = st_it->second->state;
return st_it->second->read_target.data();
}
st_it = pg.degraded_objects.find(oid);
if (st_it != pg.degraded_objects.end())
{
object_state = st_it->second->state;
return st_it->second->read_target.data();
}
st_it = pg.misplaced_objects.find(oid);
if (st_it != pg.misplaced_objects.end())
{
object_state = st_it->second->state;
return st_it->second->read_target.data();
}
object_state = 0;
return def;
}
@@ -171,7 +177,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
else
{
// PG may be degraded or have misplaced objects
uint64_t* cur_set = get_object_osd_set(pg, op_data->oid, pg.cur_set.data());
uint64_t* cur_set = get_object_osd_set(pg, op_data->oid, pg.cur_set.data(), op_data->object_state);
if (extend_missing_stripes(op_data->stripes, cur_set, pg.pg_minsize, pg.pg_size) < 0)
{
finish_op(cur_op, -EIO);
@@ -404,7 +410,6 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
return;
}
osd_primary_op_data_t *op_data = cur_op->op_data;
// FIXME: Handle operation cancel
auto & pg = pgs[op_data->pg_num];
if (op_data->st == 1) goto resume_1;
else if (op_data->st == 2) goto resume_2;
@@ -413,8 +418,6 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
else if (op_data->st == 5) goto resume_5;
else if (op_data->st == 6) goto resume_6;
else if (op_data->st == 7) goto resume_7;
else if (op_data->st == 8) goto resume_8;
else if (op_data->st == 9) goto resume_9;
assert(op_data->st == 0);
// Check if actions are pending for this object
{
@@ -441,91 +444,12 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
}
pg.write_queue.emplace(op_data->oid, cur_op);
}
if (pg.state & PG_HAS_DEGRADED)
{
op_data->prev_set = NULL;
{
auto st_it = pg.degraded_objects.find(op_data->oid);
if (st_it != pg.degraded_objects.end())
op_data->prev_set = st_it->second->read_target.data();
}
if (op_data->prev_set != NULL)
{
// Read the whole object
op_data->recovery_buf = memalign(512, pg.pg_size * bs_block_size);
for (int i = 0; i < pg.pg_size; i++)
{
op_data->stripes[i].read_buf = op_data->recovery_buf + i*bs_block_size;
op_data->stripes[i].read_start = 0;
op_data->stripes[i].read_end = bs_block_size;
op_data->stripes[i].missing = op_data->prev_set[i] == 0;
op_data->stripes[i].write_end = 0;
}
op_data->degraded = 1;
submit_primary_subops(SUBMIT_READ, pg.pg_size, op_data->prev_set, cur_op);
resume_8:
op_data->st = 8;
return;
resume_9:
if (op_data->errors > 0)
{
pg_cancel_write_queue(pg, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
return;
}
reconstruct_stripes(op_data->stripes, pg.pg_size);
if (cur_op->req.rw.len > 0)
{
memcpy(
op_data->recovery_buf + cur_op->req.rw.offset - op_data->oid.stripe,
cur_op->buf, cur_op->req.rw.len
);
}
free(cur_op->buf);
cur_op->buf = op_data->recovery_buf;
op_data->recovery_buf = NULL;
if (cur_op->req.rw.len > 0)
{
// Write modified parts
uint32_t start = 0, end = 0;
for (int role = 0; role < pg.pg_minsize; role++)
{
if (op_data->stripes[role].req_end != 0)
{
start = !end || op_data->stripes[role].req_start < start ? op_data->stripes[role].req_start : start;
end = std::max(op_data->stripes[role].req_end, end);
op_data->stripes[role].write_start = op_data->stripes[role].req_start;
op_data->stripes[role].write_end = op_data->stripes[role].req_end;
op_data->stripes[role].write_buf = cur_op->buf + role*bs_block_size + op_data->stripes[role].write_start;
}
}
for (int role = pg.pg_minsize; role < pg.pg_size; role++)
{
op_data->stripes[role].write_start = start;
op_data->stripes[role].write_end = end;
op_data->stripes[role].write_buf = cur_op->buf + role*bs_block_size + op_data->stripes[role].write_start;
}
}
// Also write recovered parts
uint64_t *cur_set = pg.cur_set.data();
for (int role = 0; role < pg.pg_size; role++)
{
if (cur_set[role] != op_data->prev_set[role])
{
op_data->stripes[role].write_start = 0;
op_data->stripes[role].write_end = bs_block_size;
op_data->stripes[role].write_buf = cur_op->buf + role*bs_block_size;
}
}
pg.ver_override[op_data->oid] = op_data->fact_ver;
// Send writes
submit_primary_subops(SUBMIT_WRITE, pg.pg_size, cur_set, cur_op);
op_data->st = 4;
return;
}
}
resume_1:
// Determine blocks to read and write
cur_op->rmw_buf = calc_rmw_reads(cur_op->buf, op_data->stripes, pg.cur_set.data(), pg.pg_size, pg.pg_minsize, pg.pg_cursize);
// Missing chunks are allowed to be overwritten even in incomplete objects
op_data->prev_set = get_object_osd_set(pg, op_data->oid, pg.cur_set.data(), op_data->object_state);
cur_op->rmw_buf = calc_rmw(cur_op->buf, op_data->stripes, op_data->prev_set,
pg.pg_size, pg.pg_minsize, pg.pg_cursize, pg.cur_set.data(), bs_block_size);
// Read required blocks
submit_primary_subops(SUBMIT_RMW_READ, pg.pg_size, pg.cur_set.data(), cur_op);
resume_2:
@@ -540,7 +464,7 @@ resume_3:
// Save version override for parallel reads
pg.ver_override[op_data->oid] = op_data->fact_ver;
// Recover missing stripes, calculate parity
calc_rmw_parity(op_data->stripes, pg.pg_size);
calc_rmw_parity(op_data->stripes, pg.pg_size, op_data->prev_set, pg.cur_set.data(), bs_block_size);
// Send writes
submit_primary_subops(SUBMIT_WRITE, pg.pg_size, pg.cur_set.data(), cur_op);
resume_4:
@@ -693,7 +617,6 @@ resume_1:
syncs_in_progress.push_back(cur_op);
}
resume_2:
// FIXME: Handle operation cancel
if (unstable_writes.size() == 0)
{
// Nothing to sync