Fix online PG count change, add a test for it

.gitea/workflows/test.yml

@@ -144,6 +144,24 @@ jobs:
           echo ""
         done
 
+  test_change_pg_count_online:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_change_pg_count_online.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_change_pg_size:
     runs-on: ubuntu-latest
     needs: build

src/client/cluster_client.cpp

@@ -57,6 +57,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
     st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
     st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
+    st_cli.on_change_pg_config_hook = [this]() { on_change_pool_config_hook(); };
     st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
     st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
     st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
@@ -563,7 +564,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
 
 void cluster_client_t::on_load_pgs_hook(bool success)
 {
-    for (auto pool_item: st_cli.pool_config)
+    for (auto & pool_item: st_cli.pool_config)
     {
         pg_counts[pool_item.first] = pool_item.second.real_pg_count;
     }
@@ -583,10 +584,15 @@ void cluster_client_t::on_load_pgs_hook(bool success)
 
 void cluster_client_t::on_change_pool_config_hook()
 {
-    for (auto pool_item: st_cli.pool_config)
+    for (auto & pool_item: st_cli.pool_config)
     {
         if (pg_counts[pool_item.first] != pool_item.second.real_pg_count)
         {
+            if (log_level > 2 && pg_counts[pool_item.first])
+            {
+                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                    pg_counts[pool_item.first], pool_item.second.real_pg_count);
+            }
             // At this point, all pool operations should have been suspended
             // And now they have to be resliced!
             for (auto op = op_queue_head; op; op = op->next)

src/client/etcd_state_client.cpp

@@ -1056,6 +1056,10 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             }
             pool_item.second.real_pg_count = n;
         }
+        if (on_change_pg_config_hook)
+        {
+            on_change_pg_config_hook();
+        }
     }
     else if (key.substr(0, etcd_prefix.length()+12) == etcd_prefix+"/pg/history/")
     {

src/client/etcd_state_client.h

@@ -142,6 +142,7 @@ public:
     std::function<json11::Json()> load_pgs_checks_hook;
     std::function<void(bool)> on_load_pgs_hook;
     std::function<void()> on_change_pool_config_hook;
+    std::function<void()> on_change_pg_config_hook;
     std::function<void(pool_id_t)> on_change_backfillfull_hook;
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;

tests/run_tests.sh

@@ -10,6 +10,8 @@ cd $(dirname $0)
 ./test_change_pg_count.sh
 SCHEME=ec ./test_change_pg_count.sh
 
+./test_change_pg_count_online.sh
+
 ./test_change_pg_size.sh
 
 ./test_create_nomaxid.sh

tests/test_change_pg_count_online.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -ex
+
+OSD_COUNT=${OSD_COUNT:-6}
+PG_COUNT=16
+GLOBAL_CONFIG=',"client_retry_interval":1000'
+
+. `dirname $0`/run_3osds.sh
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randrw \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -loops=100 -cluster_log_level=3 -runtime=60 &>./testdata/fio.log &
+FIO_PID=$!
+
+try_change()
+{
+    n=$1
+
+    for i in {1..6}; do
+        echo --- Change PG count to $n --- >>testdata/osd$i.log
+    done
+    echo --- Change PG count to $n --- >>testdata/mon.log
+
+    $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
+    echo "Pool 1 (testpool) PG count changed from $PG_COUNT to $n" >> testdata/pgr.log
+    PG_COUNT=$n
+
+    sleep 10
+}
+
+sleep 5
+
+# 16 -> 32
+
+try_change 32
+
+# 32 -> 16
+
+try_change 16
+
+# 16 -> 25
+
+try_change 25
+
+# 25 -> 17
+
+try_change 17
+
+# 17 -> 16
+
+try_change 16
+
+wait $FIO_PID
+
+grep -P 'Pool 1 \(testpool\) PG count changed from \d+ to \d+' testdata/fio.log > testdata/pgc.log
+diff testdata/pgr.log testdata/pgc.log
+
+format_green OK