diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index b52b6b9d..5cdaa399 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -962,8 +962,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) if (pc.pg_stripe_size < min_stripe_size) pc.pg_stripe_size = min_stripe_size; // Save - pc.real_pg_count = this->pool_config[pool_id].real_pg_count; - std::swap(pc.pg_config, this->pool_config[pool_id].pg_config); + auto & old_pc = this->pool_config[pool_id]; + pc.real_pg_count = old_pc.real_pg_count; + pc.applied_pg_count = old_pc.applied_pg_count; + pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size; + pc.reshard_state = old_pc.reshard_state; + std::swap(pc.pg_config, old_pc.pg_config); std::swap(this->pool_config[pool_id], pc); auto & parsed_cfg = this->pool_config[pool_id]; parsed_cfg.exists = true; diff --git a/src/osd/osd.h b/src/osd/osd.h index b8cd63a7..e3ee6527 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -171,7 +171,6 @@ class osd_t // peers and PGs std::map pg_locks; - std::map pg_counts; std::map pgs; std::set dirty_pgs; std::set dirty_osds; @@ -384,12 +383,12 @@ class osd_t int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg); int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state); - inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size) + inline pg_num_t map_to_pg(object_id oid) { - uint64_t pg_count = pg_counts[INODE_POOL(oid.inode)]; - if (!pg_count) - pg_count = 1; - return (oid.stripe / pg_stripe_size) % pg_count + 1; + auto pool_it = st_cli.pool_config.find(INODE_POOL(oid.inode)); + if (pool_it == st_cli.pool_config.end()) + return 1; + return (oid.stripe / pool_it->second.applied_pg_stripe_size) % pool_it->second.applied_pg_count + 1; } public: diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index 663b3319..7cf25a84 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -55,9 +55,10 @@ void osd_t::init_cluster() .pg_minsize = 2, .pg_count = 1, .real_pg_count = 1, + .applied_pg_count = 1, + .applied_pg_stripe_size = bs_block_size*2, }; report_pg_state(pgs[{ 1, 1 }]); - pg_counts[1] = 1; } bind_socket(); } @@ -728,8 +729,12 @@ void osd_t::apply_pg_count() for (auto & pool_item: st_cli.pool_config) { auto & pool_cfg = pool_item.second; - if (pool_cfg.real_pg_count != 0 && - pool_cfg.real_pg_count != pg_counts[pool_item.first]) + if (pool_cfg.real_pg_count == 0) + { + continue; + } + if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count || + pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) { // Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online // The external tool must wait for all PGs to come down before changing PG count @@ -751,29 +756,38 @@ void osd_t::apply_pg_count() } if (still_active_primary > 0 || still_active_secondary > 0) { - printf( - "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u)," - " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n", - this->osd_num, pool_item.first, pool_cfg.real_pg_count, pg_counts[pool_item.first], - still_active_primary, still_active_secondary - ); + if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count) + { + printf( + "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %ju)," + " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n", + this->osd_num, pool_item.first, pool_cfg.real_pg_count, pool_cfg.applied_pg_count, + still_active_primary, still_active_secondary + ); + } + else + { + printf( + "[OSD %ju] PG stripe change detected for pool %u (new is %ju, old is %ju)," + " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n", + this->osd_num, pool_item.first, pool_cfg.pg_stripe_size, pool_cfg.applied_pg_stripe_size, + still_active_primary, still_active_secondary + ); + } force_stop(1); return; } - } - if (bs && (pool_cfg.real_pg_count != pool_cfg.applied_pg_count || - pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) && - !pool_cfg.reshard_state) - { pool_cfg.applied_pg_count = pool_cfg.real_pg_count; pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size; - pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size); - if (pool_cfg.reshard_state) + if (bs && !pool_cfg.reshard_state) { - reshard_pools.push_back(pool_item.first); + pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size); + if (pool_cfg.reshard_state) + { + reshard_pools.push_back(pool_item.first); + } } } - this->pg_counts[pool_item.first] = pool_cfg.real_pg_count; } if (reshard_pools.size() && reshard_timer_id < 0) { diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index ea4b912b..f1ae4353 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -318,8 +318,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op) // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not printf( "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n", - INODE_POOL(op->oid.inode), - map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size), + INODE_POOL(op->oid.inode), map_to_pg(op->oid), op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval ); } diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index 294ab391..38d2177a 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -159,11 +159,10 @@ void osd_t::reset_pg(pg_t & pg) cancel_primary_write(p.second); } pg.write_queue.clear(); - uint64_t pg_stripe_size = st_cli.pool_config[pg.pool_id].pg_stripe_size; for (auto it = unstable_writes.begin(); it != unstable_writes.end(); ) { // Forget this PG's unstable writes - if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid, pg_stripe_size) == pg.pg_num) + if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid) == pg.pg_num) unstable_writes.erase(it++); else it++; @@ -524,6 +523,7 @@ void osd_t::relock_pg(pg_t & pg) void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps) { + auto & pool_cfg = st_cli.pool_config.at(ps->pool_id); if (role_osd == this->osd_num) { // Self @@ -533,11 +533,11 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps) clock_gettime(CLOCK_REALTIME, &op->tv_begin); op->bs_op = new blockstore_op_t(); op->bs_op->opcode = BS_OP_LIST; - op->bs_op->pg_alignment = st_cli.pool_config[ps->pool_id].pg_stripe_size; + op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size; op->bs_op->min_oid.inode = ((uint64_t)ps->pool_id << (64 - POOL_ID_BITS)); op->bs_op->max_oid.inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1; op->bs_op->max_oid.stripe = UINT64_MAX; - op->bs_op->pg_count = pg_counts[ps->pool_id]; + op->bs_op->pg_count = pool_cfg.applied_pg_count; op->bs_op->pg_number = ps->pg_num-1; op->bs_op->callback = [this, ps, op, role_osd](blockstore_op_t *bs_op) { @@ -584,8 +584,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps) .opcode = OSD_OP_SEC_LIST, }, .list_pg = ps->pg_num, - .pg_count = pg_counts[ps->pool_id], - .pg_stripe_size = st_cli.pool_config[ps->pool_id].pg_stripe_size, + .pg_count = pool_cfg.applied_pg_count, + .pg_stripe_size = pool_cfg.applied_pg_stripe_size, .min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)), .max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1, }, diff --git a/src/osd/osd_primary.cpp b/src/osd/osd_primary.cpp index 761e965a..f3ed5d52 100644 --- a/src/osd/osd_primary.cpp +++ b/src/osd/osd_primary.cpp @@ -35,15 +35,14 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) // oid.stripe = starting offset of the parity stripe .stripe = (cur_op->req.rw.offset/pg_block_size)*pg_block_size, }; - // FIXME: pg_counts should be stored in pool_config, not in a separate map - auto pg_count = pg_counts[pool_id]; + auto pg_count = pool_cfg.applied_pg_count; if (!pg_count) { // Pool config is not loaded yet finish_op(cur_op, -EPIPE); return false; } - pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_count + 1; // like map_to_pg() + pg_num_t pg_num = (oid.stripe/pool_cfg.applied_pg_stripe_size) % pg_count + 1; // like map_to_pg() auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num }); if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE) { diff --git a/src/osd/osd_primary_sync.cpp b/src/osd/osd_primary_sync.cpp index aecffa90..6a56274d 100644 --- a/src/osd/osd_primary_sync.cpp +++ b/src/osd/osd_primary_sync.cpp @@ -171,7 +171,7 @@ resume_6: auto & w = op_data->unstable_writes[unstable_osd.start + i]; pool_pg_num_t wpg = { .pool_id = INODE_POOL(w.oid.inode), - .pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size), + .pg_num = map_to_pg(w.oid), }; if (pgs.at(wpg).state & PG_ACTIVE) { diff --git a/src/osd/osd_scrub.cpp b/src/osd/osd_scrub.cpp index b4480068..caa46557 100644 --- a/src/osd/osd_scrub.cpp +++ b/src/osd/osd_scrub.cpp @@ -9,6 +9,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi { pool_id_t pool_id = pg_id.pool_id; pg_num_t pg_num = pg_id.pg_num; + auto & pool_cfg = st_cli.pool_config.at(pool_id); assert(!scrub_list_op); if (role_osd == this->osd_num) { @@ -19,7 +20,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi clock_gettime(CLOCK_REALTIME, &op->tv_begin); op->bs_op = new blockstore_op_t(); op->bs_op->opcode = BS_OP_LIST; - op->bs_op->pg_alignment = st_cli.pool_config[pool_id].pg_stripe_size; + op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size; if (min_oid.inode != 0 || min_oid.stripe != 0) op->bs_op->min_oid = min_oid; else @@ -30,7 +31,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi op->bs_op->max_oid.inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1; op->bs_op->max_oid.stripe = UINT64_MAX; op->bs_op->list_stable_limit = scrub_list_limit; - op->bs_op->pg_count = pg_counts[pool_id]; + op->bs_op->pg_count = pool_cfg.applied_pg_count; op->bs_op->pg_number = pg_num-1; op->bs_op->callback = [this, op](blockstore_op_t *bs_op) { @@ -68,8 +69,8 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi .opcode = OSD_OP_SEC_LIST, }, .list_pg = pg_num, - .pg_count = pg_counts[pool_id], - .pg_stripe_size = st_cli.pool_config[pool_id].pg_stripe_size, + .pg_count = (uint32_t)pool_cfg.applied_pg_count, + .pg_stripe_size = pool_cfg.applied_pg_stripe_size, .min_inode = min_oid.inode ? min_oid.inode : ((uint64_t)(pool_id) << (64 - POOL_ID_BITS)), .max_inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1, .min_stripe = min_oid.stripe, @@ -249,7 +250,7 @@ void osd_t::submit_scrub_op(object_id oid) printf( "Scrub failed with object %jx:%jx (PG %u/%u): error %jd\n", oid.inode, oid.stripe, INODE_POOL(oid.inode), - map_to_pg(oid, st_cli.pool_config.at(INODE_POOL(oid.inode)).pg_stripe_size), + map_to_pg(oid), osd_op->reply.hdr.retval ); } diff --git a/src/osd/osd_secondary.cpp b/src/osd/osd_secondary.cpp index e082f23d..db2e28c3 100644 --- a/src/osd/osd_secondary.cpp +++ b/src/osd/osd_secondary.cpp @@ -96,7 +96,7 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid) { return true; } - auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) }; + auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid) }; auto pg_it = pgs.find(ppg); if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE) {