Remove separate storage of PG counts in pg_counts

src/client/etcd_state_client.cpp

@@ -962,8 +962,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_stripe_size < min_stripe_size)
                 pc.pg_stripe_size = min_stripe_size;
             // Save
-            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
+            auto & old_pc = this->pool_config[pool_id];
-            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
+            pc.real_pg_count = old_pc.real_pg_count;
+            pc.applied_pg_count = old_pc.applied_pg_count;
+            pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
+            pc.reshard_state = old_pc.reshard_state;
+            std::swap(pc.pg_config, old_pc.pg_config);
             std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;

src/osd/osd.h

@@ -171,7 +171,6 @@ class osd_t
     // peers and PGs
 
     std::map<pool_pg_num_t, osd_pg_lock_t> pg_locks;
-    std::map<pool_id_t, pg_num_t> pg_counts;
     std::map<pool_pg_num_t, pg_t> pgs;
     std::set<pool_pg_num_t> dirty_pgs;
     std::set<osd_num_t> dirty_osds;
@@ -384,12 +383,12 @@ class osd_t
     int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg);
     int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state);
 
-    inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
+    inline pg_num_t map_to_pg(object_id oid)
     {
-        uint64_t pg_count = pg_counts[INODE_POOL(oid.inode)];
+        auto pool_it = st_cli.pool_config.find(INODE_POOL(oid.inode));
-        if (!pg_count)
+        if (pool_it == st_cli.pool_config.end())
-            pg_count = 1;
+            return 1;
-        return (oid.stripe / pg_stripe_size) % pg_count + 1;
+        return (oid.stripe / pool_it->second.applied_pg_stripe_size) % pool_it->second.applied_pg_count + 1;
     }
 
 public:

src/osd/osd_cluster.cpp

@@ -55,9 +55,10 @@ void osd_t::init_cluster()
                 .pg_minsize = 2,
                 .pg_count = 1,
                 .real_pg_count = 1,
+                .applied_pg_count = 1,
+                .applied_pg_stripe_size = bs_block_size*2,
             };
             report_pg_state(pgs[{ 1, 1 }]);
-            pg_counts[1] = 1;
         }
         bind_socket();
     }
@@ -728,8 +729,12 @@ void osd_t::apply_pg_count()
     for (auto & pool_item: st_cli.pool_config)
     {
         auto & pool_cfg = pool_item.second;
-        if (pool_cfg.real_pg_count != 0 &&
+        if (pool_cfg.real_pg_count == 0)
-            pool_cfg.real_pg_count != pg_counts[pool_item.first])
+        {
+            continue;
+        }
+        if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+            pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
         {
             // Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online
             // The external tool must wait for all PGs to come down before changing PG count
@@ -751,29 +756,38 @@ void osd_t::apply_pg_count()
             }
             if (still_active_primary > 0 || still_active_secondary > 0)
             {
-                printf(
+                if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count)
-                    "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u),"
+                {
-                    " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
+                    printf(
-                    this->osd_num, pool_item.first, pool_cfg.real_pg_count, pg_counts[pool_item.first],
+                        "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %ju),"
-                    still_active_primary, still_active_secondary
+                        " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
-                );
+                        this->osd_num, pool_item.first, pool_cfg.real_pg_count, pool_cfg.applied_pg_count,
+                        still_active_primary, still_active_secondary
+                    );
+                }
+                else
+                {
+                    printf(
+                        "[OSD %ju] PG stripe change detected for pool %u (new is %ju, old is %ju),"
+                        " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
+                        this->osd_num, pool_item.first, pool_cfg.pg_stripe_size, pool_cfg.applied_pg_stripe_size,
+                        still_active_primary, still_active_secondary
+                    );
+                }
                 force_stop(1);
                 return;
             }
-        }
-        if (bs && (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
-            pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) &&
-            !pool_cfg.reshard_state)
-        {
             pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
             pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
-            pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+            if (bs && !pool_cfg.reshard_state)
-            if (pool_cfg.reshard_state)
             {
-                reshard_pools.push_back(pool_item.first);
+                pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+                if (pool_cfg.reshard_state)
+                {
+                    reshard_pools.push_back(pool_item.first);
+                }
             }
         }
-        this->pg_counts[pool_item.first] = pool_cfg.real_pg_count;
     }
     if (reshard_pools.size() && reshard_timer_id < 0)
     {

src/osd/osd_flush.cpp

@@ -318,8 +318,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
                 // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not
                 printf(
                     "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n",
-                    INODE_POOL(op->oid.inode),
+                    INODE_POOL(op->oid.inode), map_to_pg(op->oid),
-                    map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size),
                     op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval
                 );
             }

src/osd/osd_peering.cpp

@@ -159,11 +159,10 @@ void osd_t::reset_pg(pg_t & pg)
         cancel_primary_write(p.second);
     }
     pg.write_queue.clear();
-    uint64_t pg_stripe_size = st_cli.pool_config[pg.pool_id].pg_stripe_size;
     for (auto it = unstable_writes.begin(); it != unstable_writes.end(); )
     {
         // Forget this PG's unstable writes
-        if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid, pg_stripe_size) == pg.pg_num)
+        if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid) == pg.pg_num)
             unstable_writes.erase(it++);
         else
             it++;
@@ -524,6 +523,7 @@ void osd_t::relock_pg(pg_t & pg)
 
 void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
 {
+    auto & pool_cfg = st_cli.pool_config.at(ps->pool_id);
     if (role_osd == this->osd_num)
     {
         // Self
@@ -533,11 +533,11 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
         op->bs_op = new blockstore_op_t();
         op->bs_op->opcode = BS_OP_LIST;
-        op->bs_op->pg_alignment = st_cli.pool_config[ps->pool_id].pg_stripe_size;
+        op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
         op->bs_op->min_oid.inode = ((uint64_t)ps->pool_id << (64 - POOL_ID_BITS));
         op->bs_op->max_oid.inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1;
         op->bs_op->max_oid.stripe = UINT64_MAX;
-        op->bs_op->pg_count = pg_counts[ps->pool_id];
+        op->bs_op->pg_count = pool_cfg.applied_pg_count;
         op->bs_op->pg_number = ps->pg_num-1;
         op->bs_op->callback = [this, ps, op, role_osd](blockstore_op_t *bs_op)
         {
@@ -584,8 +584,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = ps->pg_num,
-                .pg_count = pg_counts[ps->pool_id],
+                .pg_count = pool_cfg.applied_pg_count,
-                .pg_stripe_size = st_cli.pool_config[ps->pool_id].pg_stripe_size,
+                .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1,
             },

src/osd/osd_primary.cpp

@@ -35,15 +35,14 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         // oid.stripe = starting offset of the parity stripe
         .stripe = (cur_op->req.rw.offset/pg_block_size)*pg_block_size,
     };
-    // FIXME: pg_counts should be stored in pool_config, not in a separate map
+    auto pg_count = pool_cfg.applied_pg_count;
-    auto pg_count = pg_counts[pool_id];
     if (!pg_count)
     {
         // Pool config is not loaded yet
         finish_op(cur_op, -EPIPE);
         return false;
     }
-    pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_count + 1; // like map_to_pg()
+    pg_num_t pg_num = (oid.stripe/pool_cfg.applied_pg_stripe_size) % pg_count + 1; // like map_to_pg()
     auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
     if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE)
     {

src/osd/osd_primary_sync.cpp

@@ -171,7 +171,7 @@ resume_6:
                     auto & w = op_data->unstable_writes[unstable_osd.start + i];
                     pool_pg_num_t wpg = {
                         .pool_id = INODE_POOL(w.oid.inode),
-                        .pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size),
+                        .pg_num = map_to_pg(w.oid),
                     };
                     if (pgs.at(wpg).state & PG_ACTIVE)
                     {

src/osd/osd_scrub.cpp

@@ -9,6 +9,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
 {
     pool_id_t pool_id = pg_id.pool_id;
     pg_num_t pg_num = pg_id.pg_num;
+    auto & pool_cfg = st_cli.pool_config.at(pool_id);
     assert(!scrub_list_op);
     if (role_osd == this->osd_num)
     {
@@ -19,7 +20,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
         op->bs_op = new blockstore_op_t();
         op->bs_op->opcode = BS_OP_LIST;
-        op->bs_op->pg_alignment = st_cli.pool_config[pool_id].pg_stripe_size;
+        op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
         if (min_oid.inode != 0 || min_oid.stripe != 0)
             op->bs_op->min_oid = min_oid;
         else
@@ -30,7 +31,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
         op->bs_op->max_oid.inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1;
         op->bs_op->max_oid.stripe = UINT64_MAX;
         op->bs_op->list_stable_limit = scrub_list_limit;
-        op->bs_op->pg_count = pg_counts[pool_id];
+        op->bs_op->pg_count = pool_cfg.applied_pg_count;
         op->bs_op->pg_number = pg_num-1;
         op->bs_op->callback = [this, op](blockstore_op_t *bs_op)
         {
@@ -68,8 +69,8 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = pg_num,
-                .pg_count = pg_counts[pool_id],
+                .pg_count = (uint32_t)pool_cfg.applied_pg_count,
-                .pg_stripe_size = st_cli.pool_config[pool_id].pg_stripe_size,
+                .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = min_oid.inode ? min_oid.inode : ((uint64_t)(pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1,
                 .min_stripe = min_oid.stripe,
@@ -249,7 +250,7 @@ void osd_t::submit_scrub_op(object_id oid)
             printf(
                 "Scrub failed with object %jx:%jx (PG %u/%u): error %jd\n",
                 oid.inode, oid.stripe, INODE_POOL(oid.inode),
-                map_to_pg(oid, st_cli.pool_config.at(INODE_POOL(oid.inode)).pg_stripe_size),
+                map_to_pg(oid),
                 osd_op->reply.hdr.retval
             );
         }

src/osd/osd_secondary.cpp

@@ -96,7 +96,7 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
     {
         return true;
     }
-    auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
+    auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid) };
     auto pg_it = pgs.find(ppg);
     if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
     {