thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove separate storage of PG counts in pg_counts

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-01-25 12:26:49 +03:00
parent c4c17ee6fb
commit 9a6a7b7f75
9 changed files with 60 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/etcd_state_client.cpp
+6 -2
View File
@@ -962,8 +962,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
if (pc.pg_stripe_size < min_stripe_size)
pc.pg_stripe_size = min_stripe_size;
// Save
pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
auto & old_pc = this->pool_config[pool_id];
pc.real_pg_count = old_pc.real_pg_count;
pc.applied_pg_count = old_pc.applied_pg_count;
pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
pc.reshard_state = old_pc.reshard_state;
std::swap(pc.pg_config, old_pc.pg_config);
std::swap(this->pool_config[pool_id], pc);
auto & parsed_cfg = this->pool_config[pool_id];
parsed_cfg.exists = true;
src/osd/osd.h
+5 -6
View File
@@ -171,7 +171,6 @@ class osd_t
// peers and PGs
std::map<pool_pg_num_t, osd_pg_lock_t> pg_locks;
std::map<pool_id_t, pg_num_t> pg_counts;
std::map<pool_pg_num_t, pg_t> pgs;
std::set<pool_pg_num_t> dirty_pgs;
std::set<osd_num_t> dirty_osds;
@@ -384,12 +383,12 @@ class osd_t
int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg);
int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state);
inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
inline pg_num_t map_to_pg(object_id oid)
{
uint64_t pg_count = pg_counts[INODE_POOL(oid.inode)];
if (!pg_count)
pg_count = 1;
return (oid.stripe / pg_stripe_size) % pg_count + 1;
auto pool_it = st_cli.pool_config.find(INODE_POOL(oid.inode));
if (pool_it == st_cli.pool_config.end())
return 1;
return (oid.stripe / pool_it->second.applied_pg_stripe_size) % pool_it->second.applied_pg_count + 1;
}
public:
src/osd/osd_cluster.cpp
+32 -18
View File
@@ -55,9 +55,10 @@ void osd_t::init_cluster()
.pg_minsize = 2,
.pg_count = 1,
.real_pg_count = 1,
.applied_pg_count = 1,
.applied_pg_stripe_size = bs_block_size*2,
};
report_pg_state(pgs[{ 1, 1 }]);
pg_counts[1] = 1;
}
bind_socket();
}
@@ -728,8 +729,12 @@ void osd_t::apply_pg_count()
for (auto & pool_item: st_cli.pool_config)
{
auto & pool_cfg = pool_item.second;
if (pool_cfg.real_pg_count != 0 &&
pool_cfg.real_pg_count != pg_counts[pool_item.first])
if (pool_cfg.real_pg_count == 0)
{
continue;
}
if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
{
// Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online
// The external tool must wait for all PGs to come down before changing PG count
@@ -751,29 +756,38 @@ void osd_t::apply_pg_count()
}
if (still_active_primary > 0 || still_active_secondary > 0)
{
printf(
"[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u),"
" but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
this->osd_num, pool_item.first, pool_cfg.real_pg_count, pg_counts[pool_item.first],
still_active_primary, still_active_secondary
);
if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count)
{
printf(
"[OSD %ju] PG count change detected for pool %u (new is %ju, old is %ju),"
" but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
this->osd_num, pool_item.first, pool_cfg.real_pg_count, pool_cfg.applied_pg_count,
still_active_primary, still_active_secondary
);
}
else
{
printf(
"[OSD %ju] PG stripe change detected for pool %u (new is %ju, old is %ju),"
" but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
this->osd_num, pool_item.first, pool_cfg.pg_stripe_size, pool_cfg.applied_pg_stripe_size,
still_active_primary, still_active_secondary
);
}
force_stop(1);
return;
}
}
if (bs && (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) &&
!pool_cfg.reshard_state)
{
pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
if (pool_cfg.reshard_state)
if (bs && !pool_cfg.reshard_state)
{
reshard_pools.push_back(pool_item.first);
pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
if (pool_cfg.reshard_state)
{
reshard_pools.push_back(pool_item.first);
}
}
}
this->pg_counts[pool_item.first] = pool_cfg.real_pg_count;
}
if (reshard_pools.size() && reshard_timer_id < 0)
{
src/osd/osd_flush.cpp
+1 -2
View File
@@ -318,8 +318,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
// EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not
printf(
"[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n",
INODE_POOL(op->oid.inode),
map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size),
INODE_POOL(op->oid.inode), map_to_pg(op->oid),
op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval
);
}
src/osd/osd_peering.cpp
+6 -6
View File
@@ -159,11 +159,10 @@ void osd_t::reset_pg(pg_t & pg)
cancel_primary_write(p.second);
}
pg.write_queue.clear();
uint64_t pg_stripe_size = st_cli.pool_config[pg.pool_id].pg_stripe_size;
for (auto it = unstable_writes.begin(); it != unstable_writes.end(); )
{
// Forget this PG's unstable writes
if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid, pg_stripe_size) == pg.pg_num)
if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid) == pg.pg_num)
unstable_writes.erase(it++);
else
it++;
@@ -524,6 +523,7 @@ void osd_t::relock_pg(pg_t & pg)
void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
{
auto & pool_cfg = st_cli.pool_config.at(ps->pool_id);
if (role_osd == this->osd_num)
{
// Self
@@ -533,11 +533,11 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
clock_gettime(CLOCK_REALTIME, &op->tv_begin);
op->bs_op = new blockstore_op_t();
op->bs_op->opcode = BS_OP_LIST;
op->bs_op->pg_alignment = st_cli.pool_config[ps->pool_id].pg_stripe_size;
op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
op->bs_op->min_oid.inode = ((uint64_t)ps->pool_id << (64 - POOL_ID_BITS));
op->bs_op->max_oid.inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1;
op->bs_op->max_oid.stripe = UINT64_MAX;
op->bs_op->pg_count = pg_counts[ps->pool_id];
op->bs_op->pg_count = pool_cfg.applied_pg_count;
op->bs_op->pg_number = ps->pg_num-1;
op->bs_op->callback = [this, ps, op, role_osd](blockstore_op_t *bs_op)
{
@@ -584,8 +584,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
.opcode = OSD_OP_SEC_LIST,
},
.list_pg = ps->pg_num,
.pg_count = pg_counts[ps->pool_id],
.pg_stripe_size = st_cli.pool_config[ps->pool_id].pg_stripe_size,
.pg_count = pool_cfg.applied_pg_count,
.pg_stripe_size = pool_cfg.applied_pg_stripe_size,
.min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)),
.max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1,
},
src/osd/osd_primary.cpp
+2 -3
View File
@@ -35,15 +35,14 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
// oid.stripe = starting offset of the parity stripe
.stripe = (cur_op->req.rw.offset/pg_block_size)*pg_block_size,
};
// FIXME: pg_counts should be stored in pool_config, not in a separate map
auto pg_count = pg_counts[pool_id];
auto pg_count = pool_cfg.applied_pg_count;
if (!pg_count)
{
// Pool config is not loaded yet
finish_op(cur_op, -EPIPE);
return false;
}
pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_count + 1; // like map_to_pg()
pg_num_t pg_num = (oid.stripe/pool_cfg.applied_pg_stripe_size) % pg_count + 1; // like map_to_pg()
auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE)
{
src/osd/osd_primary_sync.cpp
+1 -1
View File
@@ -171,7 +171,7 @@ resume_6:
auto & w = op_data->unstable_writes[unstable_osd.start + i];
pool_pg_num_t wpg = {
.pool_id = INODE_POOL(w.oid.inode),
.pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size),
.pg_num = map_to_pg(w.oid),
};
if (pgs.at(wpg).state & PG_ACTIVE)
{
src/osd/osd_scrub.cpp
+6 -5
View File
@@ -9,6 +9,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
{
pool_id_t pool_id = pg_id.pool_id;
pg_num_t pg_num = pg_id.pg_num;
auto & pool_cfg = st_cli.pool_config.at(pool_id);
assert(!scrub_list_op);
if (role_osd == this->osd_num)
{
@@ -19,7 +20,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
clock_gettime(CLOCK_REALTIME, &op->tv_begin);
op->bs_op = new blockstore_op_t();
op->bs_op->opcode = BS_OP_LIST;
op->bs_op->pg_alignment = st_cli.pool_config[pool_id].pg_stripe_size;
op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
if (min_oid.inode != 0 || min_oid.stripe != 0)
op->bs_op->min_oid = min_oid;
else
@@ -30,7 +31,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
op->bs_op->max_oid.inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1;
op->bs_op->max_oid.stripe = UINT64_MAX;
op->bs_op->list_stable_limit = scrub_list_limit;
op->bs_op->pg_count = pg_counts[pool_id];
op->bs_op->pg_count = pool_cfg.applied_pg_count;
op->bs_op->pg_number = pg_num-1;
op->bs_op->callback = [this, op](blockstore_op_t *bs_op)
{
@@ -68,8 +69,8 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
.opcode = OSD_OP_SEC_LIST,
},
.list_pg = pg_num,
.pg_count = pg_counts[pool_id],
.pg_stripe_size = st_cli.pool_config[pool_id].pg_stripe_size,
.pg_count = (uint32_t)pool_cfg.applied_pg_count,
.pg_stripe_size = pool_cfg.applied_pg_stripe_size,
.min_inode = min_oid.inode ? min_oid.inode : ((uint64_t)(pool_id) << (64 - POOL_ID_BITS)),
.max_inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1,
.min_stripe = min_oid.stripe,
@@ -249,7 +250,7 @@ void osd_t::submit_scrub_op(object_id oid)
printf(
"Scrub failed with object %jx:%jx (PG %u/%u): error %jd\n",
oid.inode, oid.stripe, INODE_POOL(oid.inode),
map_to_pg(oid, st_cli.pool_config.at(INODE_POOL(oid.inode)).pg_stripe_size),
map_to_pg(oid),
osd_op->reply.hdr.retval
);
}
src/osd/osd_secondary.cpp
+1 -1
View File
@@ -96,7 +96,7 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
{
return true;
}
auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid) };
auto pg_it = pgs.find(ppg);
if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
{