diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 7f3236fd..cf503b47 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -144,6 +144,24 @@ jobs: echo "" done + test_change_pg_count_online: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_change_pg_count_online.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_change_pg_size: runs-on: ubuntu-latest needs: build diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index c170c438..beebdc37 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -57,6 +57,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); }; st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); }; st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); }; + st_cli.on_change_pg_config_hook = [this]() { on_change_pool_config_hook(); }; st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); }; st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); }; st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); }; @@ -563,7 +564,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector & os void cluster_client_t::on_load_pgs_hook(bool success) { - for (auto pool_item: st_cli.pool_config) + for (auto & pool_item: st_cli.pool_config) { pg_counts[pool_item.first] = pool_item.second.real_pg_count; } @@ -583,10 +584,15 @@ void cluster_client_t::on_load_pgs_hook(bool success) void cluster_client_t::on_change_pool_config_hook() { - for (auto pool_item: st_cli.pool_config) + for (auto & pool_item: st_cli.pool_config) { if (pg_counts[pool_item.first] != pool_item.second.real_pg_count) { + if (log_level > 2 && pg_counts[pool_item.first]) + { + printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(), + pg_counts[pool_item.first], pool_item.second.real_pg_count); + } // At this point, all pool operations should have been suspended // And now they have to be resliced! for (auto op = op_queue_head; op; op = op->next) diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index 59cab19c..3e60688f 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -1056,6 +1056,10 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) } pool_item.second.real_pg_count = n; } + if (on_change_pg_config_hook) + { + on_change_pg_config_hook(); + } } else if (key.substr(0, etcd_prefix.length()+12) == etcd_prefix+"/pg/history/") { diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index 4d9e8953..a664ec35 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -142,6 +142,7 @@ public: std::function load_pgs_checks_hook; std::function on_load_pgs_hook; std::function on_change_pool_config_hook; + std::function on_change_pg_config_hook; std::function on_change_backfillfull_hook; std::function on_change_pg_state_hook; std::function on_change_pg_history_hook; diff --git a/tests/run_tests.sh b/tests/run_tests.sh index a445beb5..734d3858 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -10,6 +10,8 @@ cd $(dirname $0) ./test_change_pg_count.sh SCHEME=ec ./test_change_pg_count.sh +./test_change_pg_count_online.sh + ./test_change_pg_size.sh ./test_create_nomaxid.sh diff --git a/tests/test_change_pg_count_online.sh b/tests/test_change_pg_count_online.sh new file mode 100755 index 00000000..e6d4f085 --- /dev/null +++ b/tests/test_change_pg_count_online.sh @@ -0,0 +1,61 @@ +#!/bin/bash -ex + +OSD_COUNT=${OSD_COUNT:-6} +PG_COUNT=16 +GLOBAL_CONFIG=',"client_retry_interval":1000' + +. `dirname $0`/run_3osds.sh + +LD_PRELOAD="build/src/client/libfio_vitastor.so" \ +fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ + -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M + +LD_PRELOAD="build/src/client/libfio_vitastor.so" \ +fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randrw \ + -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -loops=100 -cluster_log_level=3 -runtime=60 &>./testdata/fio.log & +FIO_PID=$! + +try_change() +{ + n=$1 + + for i in {1..6}; do + echo --- Change PG count to $n --- >>testdata/osd$i.log + done + echo --- Change PG count to $n --- >>testdata/mon.log + + $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' + echo "Pool 1 (testpool) PG count changed from $PG_COUNT to $n" >> testdata/pgr.log + PG_COUNT=$n + + sleep 10 +} + +sleep 5 + +# 16 -> 32 + +try_change 32 + +# 32 -> 16 + +try_change 16 + +# 16 -> 25 + +try_change 25 + +# 25 -> 17 + +try_change 17 + +# 17 -> 16 + +try_change 16 + +wait $FIO_PID + +grep -P 'Pool 1 \(testpool\) PG count changed from \d+ to \d+' testdata/fio.log > testdata/pgc.log +diff testdata/pgr.log testdata/pgc.log + +format_green OK