diff --git a/src/osd/osd.h b/src/osd/osd.h index 33eb7694..1347df0b 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -281,6 +281,8 @@ class osd_t int pick_next_scrub(object_id & next_oid); void submit_scrub_op(object_id oid); bool continue_scrub(); + void submit_scrub_subops(osd_op_t *cur_op); + void scrub_check_results(osd_op_t *cur_op); void plan_scrub(pg_t & pg, bool report_state = true); void schedule_scrub(pg_t & pg); @@ -313,7 +315,7 @@ class osd_t pg_osd_set_state_t *mark_object(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, bool ref, std::function calc_set); pg_osd_set_state_t *mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, - osd_rmw_stripe_t *stripes, bool ref, bool inconsistent); + osd_rmw_stripe_t *stripes, bool ref); pg_osd_set_state_t *mark_partial_write(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, osd_rmw_stripe_t *stripes, bool ref); void deref_object_state(pg_t & pg, pg_osd_set_state_t **object_state, bool deref); @@ -326,6 +328,8 @@ class osd_t void submit_primary_subops(int submit_type, uint64_t op_version, const uint64_t* osd_set, osd_op_t *cur_op); int submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version, osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read); + void submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop, + osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version); void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set); void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count); int submit_primary_sync_subops(osd_op_t *cur_op); diff --git a/src/osd/osd_primary.cpp b/src/osd/osd_primary.cpp index e7a1cbcc..98bf966a 100644 --- a/src/osd/osd_primary.cpp +++ b/src/osd/osd_primary.cpp @@ -51,9 +51,8 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) finish_op(cur_op, -EINVAL); return false; } - // Scrub is similar to r/w, so it's also handled here - int stripe_count = (pool_cfg.scheme == POOL_SCHEME_REPLICATED - && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? 1 : pg_it->second.pg_size); + int stripe_count = (cur_op->req.hdr.opcode == OSD_OP_SCRUB ? 0 : + (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg_it->second.pg_size)); int chain_size = 0; if (cur_op->req.hdr.opcode == OSD_OP_READ && cur_op->req.rw.meta_revision > 0) { @@ -112,15 +111,19 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) op_data->pg = &pg_it->second; op_data->oid = oid; op_data->stripes = (osd_rmw_stripe_t*)data_buf; + op_data->stripe_count = stripe_count; data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count; cur_op->op_data = op_data; - split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes); - // Resulting bitmaps have to survive op_data and be freed with the op itself - assert(!cur_op->bitmap_buf); - cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count); - for (int i = 0; i < stripe_count; i++) + if (cur_op->req.hdr.opcode != OSD_OP_SCRUB) { - op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i; + split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes); + // Resulting bitmaps have to survive op_data and be freed with the op itself + assert(!cur_op->bitmap_buf); + cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count); + for (int i = 0; i < stripe_count; i++) + { + op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i; + } } op_data->chain_size = chain_size; if (chain_size > 0) @@ -261,7 +264,7 @@ resume_2: { // I/O or checksum error // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated - op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false, false); + op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false); goto resume_0; } finish_op(cur_op, op_data->errcode); @@ -354,10 +357,10 @@ pg_osd_set_state_t *osd_t::mark_object(pg_t & pg, object_id oid, pg_osd_set_stat return object_state; } -pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, - osd_rmw_stripe_t *stripes, bool ref, bool inconsistent) +pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, + pg_osd_set_state_t *prev_object_state, osd_rmw_stripe_t *stripes, bool ref) { - return mark_object(pg, oid, prev_object_state, ref, [stripes, inconsistent](pg_osd_set_t & new_set) + return mark_object(pg, oid, prev_object_state, ref, [stripes](pg_osd_set_t & new_set) { // Mark object chunk(s) as corrupted int changes = 0; @@ -384,16 +387,6 @@ pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_os chunk.loc_bad &= ~LOC_CORRUPTED; } } - if (inconsistent && !chunk.loc_bad) - { - changes++; - chunk.loc_bad |= LOC_INCONSISTENT; - } - else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT)) - { - changes++; - chunk.loc_bad &= ~LOC_INCONSISTENT; - } chunk_it++; } return changes; diff --git a/src/osd/osd_primary.h b/src/osd/osd_primary.h index a8600be1..161421a1 100644 --- a/src/osd/osd_primary.h +++ b/src/osd/osd_primary.h @@ -26,6 +26,7 @@ struct osd_primary_op_data_t uint64_t orig_ver = 0, fact_ver = 0; int n_subops = 0, done = 0, errors = 0, drops = 0, errcode = 0; int degraded = 0; + int stripe_count = 0; osd_rmw_stripe_t *stripes = NULL; pg_t *pg = NULL; osd_op_t *subops = NULL; diff --git a/src/osd/osd_primary_chain.cpp b/src/osd/osd_primary_chain.cpp index 4f826798..94375daa 100644 --- a/src/osd/osd_primary_chain.cpp +++ b/src/osd/osd_primary_chain.cpp @@ -535,7 +535,7 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op) } if (corrupted) { - mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false, false); + mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false); } } } diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index dfec8669..9ca11485 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -152,9 +152,9 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version, osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read) { + bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED; bool wr = submit_type == SUBMIT_WRITE; osd_primary_op_data_t *op_data = cur_op->op_data; - bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED; int i = subop_idx; for (int role = 0; role < op_data->pg->pg_size; role++) { @@ -168,109 +168,9 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o osd_rmw_stripe_t *si = stripes + (submit_type == SUBMIT_SCRUB_READ ? role : stripe_num); if (role_osd_num != 0) { - osd_op_t *subop = op_data->subops + i; - uint32_t subop_len = wr - ? si->write_end - si->write_start - : si->read_end - si->read_start; - if (!wr && si->read_end == UINT32_MAX) - { - subop_len = 0; - } si->osd_num = role_osd_num; - si->read_error = false; - subop->bitmap = si->bmp_buf; - subop->bitmap_len = clean_entry_bitmap_size; - // Using rmw_buf to pass pointer to stripes. Dirty but should work - subop->rmw_buf = si; - if (role_osd_num == this->osd_num) - { - clock_gettime(CLOCK_REALTIME, &subop->tv_begin); - subop->op_type = (uint64_t)cur_op; - subop->bs_op = new blockstore_op_t((blockstore_op_t){ - .opcode = (uint64_t)(wr ? (rep ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ), - .callback = [subop, this](blockstore_op_t *bs_subop) - { - handle_primary_bs_subop(subop); - }, - { { - .oid = (object_id){ - .inode = inode, - .stripe = op_data->oid.stripe | stripe_num, - }, - .version = op_version, - .offset = wr ? si->write_start : si->read_start, - .len = subop_len, - } }, - .buf = wr ? si->write_buf : si->read_buf, - .bitmap = si->bmp_buf, - }); -#ifdef OSD_DEBUG - printf( - "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", - inode, op_data->oid.stripe | stripe_num, op_version, - subop->bs_op->offset, subop->bs_op->len - ); -#endif - bs->enqueue_op(subop->bs_op); - } - else - { - subop->op_type = OSD_OP_OUT; - subop->req.sec_rw = { - .header = { - .magic = SECONDARY_OSD_OP_MAGIC, - .id = msgr.next_subop_id++, - .opcode = (uint64_t)(wr ? (rep ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ), - }, - .oid = { - .inode = inode, - .stripe = op_data->oid.stripe | stripe_num, - }, - .version = op_version, - .offset = wr ? si->write_start : si->read_start, - .len = subop_len, - .attr_len = wr ? clean_entry_bitmap_size : 0, - .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0, - }; -#ifdef OSD_DEBUG - printf( - "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", role_osd_num, - inode, op_data->oid.stripe | stripe_num, op_version, - subop->req.sec_rw.offset, subop->req.sec_rw.len - ); -#endif - if (wr) - { - if (si->write_end > si->write_start) - { - subop->iov.push_back(si->write_buf, si->write_end - si->write_start); - } - } - else - { - if (subop_len > 0) - { - subop->iov.push_back(si->read_buf, subop_len); - } - } - subop->callback = [cur_op, this](osd_op_t *subop) - { - handle_primary_subop(subop, cur_op); - }; - auto peer_fd_it = msgr.osd_peer_fds.find(role_osd_num); - if (peer_fd_it != msgr.osd_peer_fds.end()) - { - subop->peer_fd = peer_fd_it->second; - msgr.outbox_push(subop); - } - else - { - // Fail it immediately - subop->peer_fd = -1; - subop->reply.hdr.retval = -EPIPE; - ringloop->set_immediate([subop]() { std::function(subop->callback)(subop); }); - } - } + si->role = stripe_num; + submit_primary_subop(cur_op, &op_data->subops[i], si, wr, inode, op_version); i++; } else @@ -281,6 +181,112 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o return i-subop_idx; } +void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop, + osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version) +{ + uint32_t subop_len = wr + ? si->write_end - si->write_start + : si->read_end - si->read_start; + if (!wr && si->read_end == UINT32_MAX) + { + subop_len = 0; + } + si->read_error = false; + subop->bitmap = si->bmp_buf; + subop->bitmap_len = clean_entry_bitmap_size; + // Using rmw_buf to pass pointer to stripes. Dirty but works + subop->rmw_buf = si; + if (si->osd_num == this->osd_num) + { + clock_gettime(CLOCK_REALTIME, &subop->tv_begin); + subop->op_type = (uint64_t)cur_op; // also dirty + subop->bs_op = new blockstore_op_t((blockstore_op_t){ + .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ), + .callback = [subop, this](blockstore_op_t *bs_subop) + { + handle_primary_bs_subop(subop); + }, + { { + .oid = (object_id){ + .inode = inode, + .stripe = cur_op->op_data->oid.stripe | si->role, + }, + .version = op_version, + .offset = wr ? si->write_start : si->read_start, + .len = subop_len, + } }, + .buf = wr ? si->write_buf : si->read_buf, + .bitmap = si->bmp_buf, + }); +#ifdef OSD_DEBUG + printf( + "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", + inode, op_data->oid.stripe | si->role, op_version, + subop->bs_op->offset, subop->bs_op->len + ); +#endif + bs->enqueue_op(subop->bs_op); + } + else + { + subop->op_type = OSD_OP_OUT; + subop->req.sec_rw = (osd_op_sec_rw_t){ + .header = { + .magic = SECONDARY_OSD_OP_MAGIC, + .id = msgr.next_subop_id++, + .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ), + }, + .oid = { + .inode = inode, + .stripe = cur_op->op_data->oid.stripe | si->role, + }, + .version = op_version, + .offset = wr ? si->write_start : si->read_start, + .len = subop_len, + .attr_len = wr ? clean_entry_bitmap_size : 0, + .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0, + }; +#ifdef OSD_DEBUG + printf( + "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", si->osd_num, + inode, op_data->oid.stripe | si->role, op_version, + subop->req.sec_rw.offset, subop->req.sec_rw.len + ); +#endif + if (wr) + { + if (si->write_end > si->write_start) + { + subop->iov.push_back(si->write_buf, si->write_end - si->write_start); + } + } + else + { + if (subop_len > 0) + { + subop->iov.push_back(si->read_buf, subop_len); + } + } + subop->callback = [cur_op, this](osd_op_t *subop) + { + handle_primary_subop(subop, cur_op); + }; + auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num); + if (peer_fd_it != msgr.osd_peer_fds.end()) + { + subop->peer_fd = peer_fd_it->second; + msgr.outbox_push(subop); + } + else + { + // Fail it immediately + subop->peer_fd = -1; + subop->reply.hdr.retval = -EPIPE; + ringloop->set_immediate([subop]() { std::function(subop->callback)(subop); }); + } + } +} + static uint64_t bs_op_to_osd_op[] = { 0, OSD_OP_SEC_READ, // BS_OP_READ = 1 @@ -401,7 +407,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op) printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version); #endif - if (op_data->fact_ver != UINT64_MAX) + if (version != 0 && op_data->fact_ver != UINT64_MAX) { if (op_data->fact_ver != 0 && op_data->fact_ver != version) { diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index ab54cdfe..f2c263fa 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -151,7 +151,7 @@ resume_3: if (op_data->errcode == -EIO || op_data->errcode == -EDOM) { // Mark object corrupted and retry - op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true, false); + op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true); op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data(); if (cur_op->rmw_buf) { diff --git a/src/osd/osd_rmw.cpp b/src/osd/osd_rmw.cpp index 3485043d..50702a59 100644 --- a/src/osd/osd_rmw.cpp +++ b/src/osd/osd_rmw.cpp @@ -1118,139 +1118,230 @@ static bool next_combination(int *subset, int k, int n) return true; } -static int c_n_k(int n, int k) +static uint64_t c_n_k(uint64_t n, uint64_t k) { - int c = 1; - for (int i = n; i > k; i--) + uint64_t c = 1; + for (uint64_t i = n; i > k; i--) + { + if ((c*i) < i) + return UINT64_MAX; c *= i; - for (int i = 2; i <= (n-k); i++) + } + for (uint64_t i = 2; i <= (n-k); i++) c /= i; return c; } -std::vector ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor, - uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce) +static std::vector ec_check_combination(osd_rmw_stripe_t *stripes, int stripe_count, + int *subset, int pg_size, int pg_minsize, bool is_xor, + uint32_t chunk_size, uint32_t bitmap_size, uint8_t *tmp_buf) +{ + osd_num_t fake_osd_set[pg_size]; + for (int i = 0; i < pg_size; i++) + { + fake_osd_set[i] = i+1; + } + osd_rmw_stripe_t brute_stripes[pg_size]; + memset(brute_stripes, 0, sizeof(osd_rmw_stripe_t)*pg_size); + for (int i = 0; i < pg_size; i++) + { + auto & bs = brute_stripes[i]; + bs.req_end = bs.read_end = chunk_size; + } + for (int i = 0; i < pg_minsize; i++) + { + auto & src = stripes[subset[i]]; + auto & bs = brute_stripes[src.role]; + bs.bmp_buf = src.bmp_buf; + bs.write_buf = bs.read_buf = src.read_buf; + } + for (int i = 0; i < pg_size; i++) + { + auto & bs = brute_stripes[i]; + if (!bs.read_buf) + { + // missing chunks are recovered in read_bufs and write_bufs are used as source for parity + bs.missing = true; + bs.read_buf = bs.write_buf = tmp_buf+i*chunk_size; + bs.bmp_buf = tmp_buf + stripe_count*chunk_size + i*bitmap_size; + } + else if (i >= pg_minsize) + { + // parity chunks are regenerated in their write_bufs, so use a temporary buffer + bs.write_buf = tmp_buf+i*chunk_size; + } + } + if (is_xor) + { + assert(pg_size == pg_minsize+1); + reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size); + } + else + { + reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size); + calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size); + } + bool matched_other = false; + std::vector good_set; + for (int i = 0; i < stripe_count; i++) + { + if (stripes[i].read_error || stripes[i].not_exists) + { + continue; + } + auto & bs = brute_stripes[stripes[i].role]; + if (!bs.missing && bs.read_buf == stripes[i].read_buf) + { + // source chunk, mark OK + good_set.push_back(i); + } + else if (memcmp(stripes[i].role < pg_minsize ? bs.read_buf : bs.write_buf, stripes[i].read_buf, chunk_size) == 0) + { + // matching chunk, mark OK + good_set.push_back(i); + matched_other = true; + } + } + if (!matched_other) + { + good_set.clear(); + } + return good_set; +} + +static int count_roles(osd_rmw_stripe_t *stripes, std::vector & valid_chunks, int pg_size) +{ + bool role_ok[pg_size]; + for (int i = 0; i < pg_size; i++) + { + role_ok[i] = false; + } + for (int idx: valid_chunks) + { + role_ok[stripes[idx].role] = true; + } + int ok_count = 0; + for (int i = 0; i < pg_size; i++) + { + if (role_ok[i]) + ok_count++; + } + return ok_count; +} + +std::vector ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor, + uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best) { std::vector found_valid; - int cur_live[pg_size], live_count = 0, exists_count = 0; - osd_num_t fake_osd_set[pg_size]; - for (int role = 0; role < pg_size; role++) + std::vector> live_variants(pg_size); + int eq_to[stripe_count]; + int live_roles = 0, live_total = 0; + for (int i = 0; i < pg_size; i++) { - if (!stripes[role].missing) - { - if (!stripes[role].not_exists) - exists_count++; - cur_live[live_count++] = role; - fake_osd_set[role] = role+1; - } + eq_to[i] = i; } - if (live_count <= pg_minsize) + for (int i = 0; i < stripe_count; i++) { - return std::vector(); - } - if (exists_count <= pg_minsize) - { - // Special case: user manually deleted some chunks - for (int role = 0; role < pg_size; role++) - if (!stripes[role].missing && !stripes[role].not_exists) - found_valid.push_back(role); - return found_valid; - } - // Try to locate errors using brute force if there isn't too many combinations - osd_rmw_stripe_t brute_stripes[pg_size]; - int out_count = live_count-pg_minsize; - bool brute_force = out_count > 1 && c_n_k(live_count-1, out_count-1) <= max_bruteforce; - int subset[pg_minsize], outset[out_count]; - // Select all combinations with items except the last one (== anything to compare) - first_combination(subset, pg_minsize, live_count-1); - uint8_t *tmp_buf = (uint8_t*)malloc_or_die(pg_size*chunk_size); - do - { - memcpy(brute_stripes, stripes, sizeof(osd_rmw_stripe_t)*pg_size); - int i = 0, j = 0, k = 0; - for (; i < pg_minsize; i++, j++) - while (j < subset[i]) - outset[k++] = j++; - while (j < pg_size) - outset[k++] = j++; - for (int i = 0; i < out_count; i++) + if (!stripes[i].read_error && !stripes[i].not_exists) { - brute_stripes[cur_live[outset[i]]].missing = true; - brute_stripes[cur_live[outset[i]]].read_buf = tmp_buf+cur_live[outset[i]]*chunk_size; - } - for (int i = 0; i < pg_minsize; i++) - { - brute_stripes[i].write_buf = brute_stripes[i].read_buf; - brute_stripes[i].req_start = 0; - brute_stripes[i].req_end = chunk_size; - } - for (int i = pg_minsize; i < pg_size; i++) - { - brute_stripes[i].write_buf = tmp_buf+i*chunk_size; - } - if (is_xor) - { - assert(pg_size == pg_minsize+1); - reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size); - } - else - { - reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size); - calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size); - } - for (int i = pg_minsize; i < pg_size; i++) - { - brute_stripes[i].read_buf = brute_stripes[i].write_buf; - } - int valid_count = 0; - for (int i = 0; i < out_count; i++) - { - if (memcmp(brute_stripes[cur_live[outset[i]]].read_buf, - stripes[cur_live[outset[i]]].read_buf, chunk_size) == 0) + if (live_variants[stripes[i].role].size() > 0) { - brute_stripes[cur_live[outset[i]]].missing = false; - valid_count++; - } - } - if (valid_count > 0) - { - if (found_valid.size()) - { - // Check if we found the same set from the different point of view, - // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5 - for (int i = 0, j = 0; i < pg_size; i++) + for (int j = 0; j < i; j++) { - if (!brute_stripes[i].missing) + if (stripes[j].role == stripes[i].role && + memcmp(stripes[i].read_buf, stripes[j].read_buf, chunk_size) == 0) { - if (j >= found_valid.size() || found_valid[j] != i) - { - // Ambiguity: we found multiple valid sets and don't know which one is correct - found_valid.clear(); - break; - } - j++; + eq_to[i] = eq_to[j]; + break; } } - if (!found_valid.size()) - { - break; - } } else { - for (int i = 0; i < pg_size; i++) + live_roles++; + } + if (eq_to[i] == i) + { + live_variants[stripes[i].role].push_back(i); + live_total++; + } + } + } + if (live_roles == pg_minsize && live_total > pg_minsize) + { + // Nothing to validate and there are chunks with different data => object is inconsistent + return std::vector(); + } + if (live_roles <= pg_minsize) + { + // Nothing to validate, just return all live chunks + for (int i = 0; i < stripe_count; i++) + if (!stripes[i].read_error) + found_valid.push_back(i); + return found_valid; + } + // Try to locate errors using brute force if there isn't too many combinations + bool brute_force = c_n_k(live_roles, pg_minsize) <= max_bruteforce; + int combination[pg_minsize], subset[pg_minsize], subvar[pg_minsize]; + // To translate 0..live_roles into 0..pg_size + int comb_to_subset[live_roles]; + for (int i = 0, r = 0; i < pg_size; i++) + { + if (live_variants[i].size() > 0) + comb_to_subset[r++] = i; + } + // Select all combinations with items except the last one (== anything to compare) + first_combination(combination, pg_minsize, live_roles); + uint8_t *tmp_buf = (uint8_t*)malloc_or_die(stripe_count*(chunk_size+bitmap_size)); + do + { + // Then loop over all subvariants (if some roles have multiple diverged variants of data) + for (int i = 0; i < pg_minsize; i++) + { + subvar[i] = 0; + } + while (true) + { + // Transform combination[] + subvar[] into subset[] + for (int i = 0; i < pg_minsize; i++) + { + subset[i] = live_variants[comb_to_subset[combination[i]]][subvar[i]]; + } + // Check the combination + auto valid_chunks = ec_check_combination(stripes, stripe_count, subset, pg_size, pg_minsize, is_xor, chunk_size, bitmap_size, tmp_buf); + // The same set may be found from different points of view, + // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5 + if (valid_chunks.size() > 0) + { + if (found_valid.size() >= valid_chunks.size() && found_valid != valid_chunks) { - if (!brute_stripes[i].missing) - { - found_valid.push_back(i); - } + // Ambiguity: we found multiple valid sets and don't know which one is correct + printf("Scrub found 2 different correct chunk subsets: OSD "); + for (int i = 0; i < found_valid.size(); i++) + printf(i > 0 ? ", %ju" : "%ju", stripes[found_valid[i]].osd_num); + printf(" and OSD "); + for (int i = 0; i < valid_chunks.size(); i++) + printf(i > 0 ? ", %ju" : "%ju", stripes[valid_chunks[i]].osd_num); + printf("\n"); + found_valid.clear(); + goto out; + } + else if (!found_valid.size() && (find_best || count_roles(stripes, valid_chunks, pg_size) >= pg_size)) + { + found_valid = valid_chunks; } } - if (valid_count == out_count) + // Select next subvariant + int i = 0; + for (i = 0; i < pg_minsize; i++) { - // All chunks are good - break; + subvar[i]++; + if (subvar[i] < live_variants[combination[i]].size()) + break; + subvar[i] = 0; } + if (i >= pg_minsize) + break; } if (!brute_force) { @@ -1258,7 +1349,8 @@ std::vector ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_min // if we find it we won't be able to check that it's the only good one break; } - } while (out_count > 1 && next_combination(subset, pg_minsize, live_count-1)); + } while (next_combination(combination, pg_minsize, live_roles)); +out: free(tmp_buf); return found_valid; } diff --git a/src/osd/osd_rmw.h b/src/osd/osd_rmw.h index 60255b2b..474973c9 100644 --- a/src/osd/osd_rmw.h +++ b/src/osd/osd_rmw.h @@ -28,6 +28,7 @@ struct osd_rmw_stripe_t uint32_t read_start, read_end; uint32_t write_start, write_end; osd_num_t osd_num; + int role; bool missing: 1; bool read_error: 1; bool not_exists: 1; @@ -57,5 +58,5 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, uint64_t *read_osd_set, uint64_t *write_osd_set, uint32_t chunk_size, uint32_t bitmap_size); -std::vector ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor, - uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce); +std::vector ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor, + uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best); diff --git a/src/osd/osd_rmw_test.cpp b/src/osd/osd_rmw_test.cpp index a71aab3d..f7002ef0 100644 --- a/src/osd/osd_rmw_test.cpp +++ b/src/osd/osd_rmw_test.cpp @@ -1160,24 +1160,26 @@ void test_ec43_error_bruteforce() stripes[i].read_end = 4096; stripes[i].read_buf = write_buf+i*4096; stripes[i].write_buf = NULL; + stripes[i].role = i; + stripes[i].osd_num = i+1; } // All good chunks - auto res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100); + auto res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true); assert_eq_vec(res, std::vector({0, 1, 2, 3, 4, 5, 6})); // 1 missing chunk set_pattern(write_buf+1*4096, 4096, 0); - res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100); + res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true); assert_eq_vec(res, std::vector({0, 2, 3, 4, 5, 6})); // 2 missing chunks set_pattern(write_buf+1*4096, 4096, 0); set_pattern(write_buf+5*4096, 4096, 0); - res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100); + res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true); assert_eq_vec(res, std::vector({0, 2, 3, 4, 6})); // 3 missing chunks set_pattern(write_buf+1*4096, 4096, 0); set_pattern(write_buf+5*4096, 4096, 0); set_pattern(write_buf+6*4096, 4096, 0); - res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100); + res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true); assert_eq_vec(res, std::vector()); // Done free(rmw_buf); diff --git a/src/osd/osd_scrub.cpp b/src/osd/osd_scrub.cpp index 0460c9c3..bee5616a 100644 --- a/src/osd/osd_scrub.cpp +++ b/src/osd/osd_scrub.cpp @@ -369,113 +369,102 @@ void osd_t::schedule_scrub(pg_t & pg) } } -void osd_t::continue_primary_scrub(osd_op_t *cur_op) +void osd_t::submit_scrub_subops(osd_op_t *cur_op) { - if (!cur_op->op_data && !prepare_primary_rw(cur_op)) - return; osd_primary_op_data_t *op_data = cur_op->op_data; - if (op_data->st == 1) - goto resume_1; - else if (op_data->st == 2) - goto resume_2; + assert(!op_data->stripe_count); + cur_op->req.rw.len = bs_block_size * op_data->pg->pg_data_size; + // Determine version + auto vo_it = op_data->pg->ver_override.find(op_data->oid); + op_data->target_ver = vo_it != op_data->pg->ver_override.end() ? vo_it->second : UINT64_MAX; + // Find object state + op_data->prev_set = get_object_osd_set(*op_data->pg, op_data->oid, &op_data->object_state); + if (!op_data->object_state) { - auto & pg = *cur_op->op_data->pg; - cur_op->req.rw.len = bs_block_size * pg.pg_data_size; - // Determine version - auto vo_it = pg.ver_override.find(op_data->oid); - op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX; - // PG may have degraded or misplaced objects - op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state); - // Read all available chunks - int n_copies = 0; - op_data->degraded = false; - for (int role = 0; role < pg.pg_size; role++) + op_data->stripe_count = op_data->pg->pg_size; + op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t)); + for (int i = 0; i < op_data->pg->pg_size; i++) { - op_data->stripes[role].write_buf = NULL; - op_data->stripes[role].read_start = 0; - op_data->stripes[role].read_end = bs_block_size; - if (op_data->prev_set[role] != 0) - { - n_copies++; - } - else - { - op_data->stripes[role].missing = true; - if (pg.scheme != POOL_SCHEME_REPLICATED && role < pg.pg_data_size) - { - op_data->degraded = true; - } - } - } - if (n_copies <= pg.pg_data_size) - { - // Nothing to compare, even if we'd like to - finish_op(cur_op, 0); - return; - } - cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0); - // Submit reads - osd_op_t *subops = new osd_op_t[n_copies]; - op_data->fact_ver = 0; - op_data->done = op_data->errors = op_data->errcode = 0; - op_data->n_subops = n_copies; - op_data->subops = subops; - int sent = submit_primary_subop_batch(SUBMIT_SCRUB_READ, op_data->oid.inode, op_data->target_ver, - op_data->stripes, op_data->prev_set, cur_op, 0, -1); - assert(sent == n_copies); - op_data->st = 1; - } -resume_1: - return; -resume_2: - if (op_data->errors > 0) - { - if (op_data->errcode == -EIO || op_data->errcode == -EDOM) - { - // I/O or checksum error - int n_copies = 0; - for (int role = 0; role < op_data->pg->pg_size; role++) - { - if (op_data->stripes[role].read_error) - { - op_data->stripes[role].missing = true; - if (op_data->pg->scheme != POOL_SCHEME_REPLICATED && role < op_data->pg->pg_data_size) - { - op_data->degraded = true; - } - } - else if (!op_data->stripes[role].missing) - { - n_copies++; - } - } - if (n_copies <= op_data->pg->pg_data_size) - { - // Nothing to compare, just mark the object as corrupted - // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated - op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false, false); - // Operation is treated as unsuccessful only if the object becomes unreadable - finish_op(cur_op, n_copies < op_data->pg->pg_data_size ? op_data->errcode : 0); - return; - } - // Proceed, we can still compare chunks that were successfully read - } - else - { - finish_op(cur_op, op_data->errcode); - return; + op_data->stripes[i].osd_num = op_data->prev_set[i]; + op_data->stripes[i].role = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : i); + op_data->stripes[i].read_end = bs_block_size; } } + else + { + op_data->stripe_count = 0; + for (auto & chunk: op_data->object_state->osd_set) + { + // Read all chunks except outdated + if (!(chunk.loc_bad & LOC_OUTDATED)) + op_data->stripe_count++; + } + op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t)); + int i = 0; + for (auto & chunk: op_data->object_state->osd_set) + { + if (!(chunk.loc_bad & LOC_OUTDATED)) + { + op_data->stripes[i].osd_num = chunk.osd_num; + op_data->stripes[i].role = chunk.role; + op_data->stripes[i].read_end = bs_block_size; + i++; + } + } + } + assert(!cur_op->bitmap_buf); + cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * op_data->stripe_count); + for (int i = 0; i < op_data->stripe_count; i++) + { + op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i; + } + cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->stripe_count, 0); + op_data->fact_ver = 0; + op_data->done = op_data->errors = op_data->errcode = 0; + op_data->n_subops = op_data->stripe_count; + op_data->subops = new osd_op_t[op_data->stripe_count]; + op_data->st = 1; + for (int i = 0; i < op_data->stripe_count; i++) + { + submit_primary_subop(cur_op, &op_data->subops[i], &op_data->stripes[i], + false, op_data->oid.inode, op_data->target_ver); + } +} + +// The idea is that scrub should not only find out if the object +// is corrupted, but it should also verify availability of all copies +void osd_t::scrub_check_results(osd_op_t *cur_op) +{ + osd_primary_op_data_t *op_data = cur_op->op_data; bool inconsistent = false; + int total = 0; + for (int role = 0; role < op_data->stripe_count; role++) + { + if (!op_data->stripes[role].not_exists) + total++; + } + if (!total) + { + // Object is deleted manually from all OSDs, forget it + printf( + "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n", + INODE_POOL(op_data->oid.inode), op_data->pg_num, + op_data->oid.inode, op_data->oid.stripe + ); + remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false); + deref_object_state(*op_data->pg, &op_data->object_state, true); + return; + } if (op_data->pg->scheme == POOL_SCHEME_REPLICATED) { // Check that all chunks have returned the same data int total = 0; - int eq_to[op_data->pg->pg_size]; - for (int role = 0; role < op_data->pg->pg_size; role++) + int eq_to[op_data->stripe_count]; + for (int role = 0; role < op_data->stripe_count; role++) { eq_to[role] = -1; - if (op_data->stripes[role].read_end != 0 && !op_data->stripes[role].missing && + if (op_data->stripes[role].read_end != 0 && + !op_data->stripes[role].read_error && !op_data->stripes[role].not_exists) { total++; @@ -491,16 +480,16 @@ resume_2: } } } - int votes[op_data->pg->pg_size]; - for (int role = 0; role < op_data->pg->pg_size; role++) + int votes[op_data->stripe_count]; + for (int role = 0; role < op_data->stripe_count; role++) votes[role] = 0; - for (int role = 0; role < op_data->pg->pg_size; role++) + for (int role = 0; role < op_data->stripe_count; role++) { if (eq_to[role] != -1) votes[eq_to[role]]++; } int best = -1; - for (int role = 0; role < op_data->pg->pg_size; role++) + for (int role = 0; role < op_data->stripe_count; role++) { if (votes[role] > (best >= 0 ? votes[best] : 0)) best = role; @@ -508,7 +497,7 @@ resume_2: if (best >= 0 && votes[best] < total) { bool unknown = false; - for (int role = 0; role < op_data->pg->pg_size; role++) + for (int role = 0; role < op_data->stripe_count; role++) { if (role != best && votes[role] == votes[best]) { @@ -551,8 +540,9 @@ resume_2: { assert(op_data->pg->scheme == POOL_SCHEME_EC || op_data->pg->scheme == POOL_SCHEME_XOR); auto good_subset = ec_find_good( - op_data->stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, op_data->pg->scheme == POOL_SCHEME_XOR, - bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce + op_data->stripes, op_data->stripe_count, + op_data->pg->pg_size, op_data->pg->pg_data_size, op_data->pg->scheme == POOL_SCHEME_XOR, + bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce, scrub_find_best ); if (!good_subset.size()) { @@ -566,61 +556,115 @@ resume_2: else { int total = 0; - for (int role = 0; role < op_data->pg->pg_size; role++) + for (int i = 0; i < op_data->stripe_count; i++) { - if (!op_data->stripes[role].missing) + if (!op_data->stripes[i].not_exists) { + // use "missing" flag to distinguish actual read errors and inconsistent chunks total++; - op_data->stripes[role].read_error = true; + op_data->stripes[i].missing = true; } } - for (int role: good_subset) + for (int i: good_subset) { - op_data->stripes[role].read_error = false; + op_data->stripes[i].missing = false; } - for (int role = 0; role < op_data->pg->pg_size; role++) + for (int i = 0; i < op_data->stripe_count; i++) { - if (!op_data->stripes[role].missing && op_data->stripes[role].read_error) + if (op_data->stripes[i].missing) { + op_data->stripes[i].read_error = true; printf( "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n", INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver, - role, op_data->stripes[role].osd_num, + op_data->stripes[i].role, op_data->stripes[i].osd_num, scrub_find_best ? ", marking it as corrupted" : "" ); } } - if (!scrub_find_best && good_subset.size() < total) + } + } + bool mark = inconsistent; + for (int role = 0; !mark && role < op_data->stripe_count; role++) + { + if (op_data->stripes[role].read_error || op_data->stripes[role].not_exists) + mark = true; + } + if (!mark) + { + return; + } + // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated + op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false /*ref*/, [op_data, inconsistent](pg_osd_set_t & new_set) + { + // Mark object chunk(s) as corrupted and/or missing and/or inconsistent + int changes = 0; + for (int i = 0; i < op_data->stripe_count; i++) + { + // Find the same stripe in new_set + int set_pos = 0; + while (set_pos < new_set.size() && (op_data->stripes[i].osd_num != new_set[set_pos].osd_num || + op_data->stripes[i].role != new_set[set_pos].role)) { - inconsistent = true; - printf( - "[PG %u/%u] Object %jx:%jx v%ju is marked as inconsistent because scrub_find_best is turned off. Use vitastor-cli fix to fix it\n", - INODE_POOL(op_data->oid.inode), op_data->pg_num, - op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver - ); - for (int role = 0; role < op_data->pg->pg_size; role++) - { - if (!op_data->stripes[role].missing && op_data->stripes[role].read_error) - { - // Undo error locator marking chunk as bad - op_data->stripes[role].read_error = false; - } - } + set_pos++; + } + if (set_pos >= new_set.size()) + { + continue; + } + if (op_data->stripes[i].not_exists) + { + changes++; + new_set.erase(new_set.begin()+set_pos, new_set.begin()+set_pos+1); + continue; + } + auto & chunk = new_set[set_pos]; + if (op_data->stripes[i].read_error && chunk.loc_bad != LOC_CORRUPTED) + { + changes++; + chunk.loc_bad = LOC_CORRUPTED; + } + else if (op_data->stripes[i].read_end > 0 && !op_data->stripes[chunk.role].missing && + (chunk.loc_bad & LOC_CORRUPTED)) + { + changes++; + chunk.loc_bad &= ~LOC_CORRUPTED; + } + if (inconsistent && !(chunk.loc_bad & LOC_INCONSISTENT)) + { + changes++; + chunk.loc_bad |= LOC_INCONSISTENT; + } + else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT)) + { + changes++; + chunk.loc_bad &= ~LOC_INCONSISTENT; } } - } - for (int role = 0; role < op_data->pg->pg_size; role++) + return changes; + }); +} + +void osd_t::continue_primary_scrub(osd_op_t *cur_op) +{ + if (!cur_op->op_data && !prepare_primary_rw(cur_op)) + return; + if (cur_op->op_data->st == 1) + goto resume_1; + else if (cur_op->op_data->st == 2) + goto resume_2; + submit_scrub_subops(cur_op); +resume_1: + return; +resume_2: + if (cur_op->op_data->errors > 0 && + // I/O and checksum errors (represented by stripes[i].read_error) are OK + (cur_op->op_data->errcode != -EIO && cur_op->op_data->errcode != -EDOM)) { - if (op_data->stripes[role].osd_num != 0 && - (op_data->stripes[role].read_error || op_data->stripes[role].not_exists) || - inconsistent) - { - // Got at least 1 read error or mismatch, mark the object as corrupted - // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated - op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false, inconsistent); - break; - } + finish_op(cur_op, cur_op->op_data->errcode); + return; } + scrub_check_results(cur_op); finish_op(cur_op, 0); }