Fix bad resharding due to the lack of iteration order in a hashmap

src/blockstore/blockstore_heap.cpp

@@ -817,31 +817,32 @@ void blockstore_heap_t::reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_s
     {
         return;
     }
+    uint32_t old_pg_count = !pool_settings.pg_count ? 1 : pool_settings.pg_count;
     uint64_t pool_id = (uint64_t)pool;
     emhash7::HashMap<uint64_t, emhash7::HashMap<inode_t, emhash5::HashMap<uint64_t, uint64_t>>> new_shards;
-    auto sh_it = block_index.begin();
+    for (uint32_t pg_num = 0; pg_num <= old_pg_count; pg_num++)
-    while (sh_it != block_index.end())
     {
-        if ((sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
+        auto sh_it = block_index.find((pool_id << (64-POOL_ID_BITS)) | pg_num);
+        if (sh_it == block_index.end())
         {
-            for (auto & inode_pair: sh_it->second)
+            continue;
-            {
-                inode_t inode = inode_pair.first;
-                for (auto & pair: inode_pair.second)
-                {
-                    // like map_to_pg()
-                    uint64_t pg_num = (pair.first / pg_stripe_size) % pg_count + 1;
-                    uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
-                    new_shards[shard_id][inode][pair.first] = std::move(pair.second);
-                }
-            }
-            block_index.erase(sh_it++);
         }
+        for (auto & inode_pair: sh_it->second)
+        {
+            inode_t inode = inode_pair.first;
+            for (auto & pair: inode_pair.second)
+            {
+                // like map_to_pg()
+                uint64_t pg_num = (pair.first / pg_stripe_size) % pg_count + 1;
+                uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
+                new_shards[shard_id][inode][pair.first] = std::move(pair.second);
+            }
+        }
+        block_index.erase(sh_it);
     }
-    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
+    for (auto sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
     {
-        auto & to = block_index[sh_it->first];
+        block_index[sh_it->first] = std::move(sh_it->second);
-        to.swap(sh_it->second);
     }
     pool_settings = (pool_shard_settings_t){
         .pg_count = pg_count,