diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 25c3d904..e8a0db00 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -414,6 +414,24 @@ jobs: echo "" done + test_degraded: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_degraded.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_rm: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index fe59b2b7..23b8f661 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -410,7 +410,11 @@ void osd_t::report_statistics() void osd_t::on_change_osd_state_hook(osd_num_t peer_osd) { - if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end()) + if (st_cli.peer_states[peer_osd].is_null()) + { + repeer_pgs(peer_osd); + } + else if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end()) { msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]); } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 5c762bca..76a5afe3 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -36,6 +36,8 @@ SCHEME=ec ./test_snapshot.sh ./test_move_reappear.sh +./test_degraded.sh + ./test_rm.sh ./test_rm_degraded.sh diff --git a/tests/test_degraded.sh b/tests/test_degraded.sh new file mode 100755 index 00000000..f070e021 --- /dev/null +++ b/tests/test_degraded.sh @@ -0,0 +1,28 @@ +#!/bin/bash -ex + +OSD_COUNT=0 +PG_COUNT=32 +PG_SIZE=3 +PG_MINSIZE=2 +POOLCFG='"failure_domain":"host",' +. `dirname $0`/run_3osds.sh + +$ETCDCTL put /vitastor/config/node_placement '{"1":{"parent":"host1"},"2":{"parent":"host2"},"3":{"parent":"host3"},"4":{"parent":"host1"},"5":{"parent":"host2"},"6":{"parent":"host3"},"host1":{"level":"host"},"host2":{"level":"host"}}' + +OSD_COUNT=6 +for i in $(seq 1 $OSD_COUNT); do + start_osd $i +done +wait_up 120 + +LD_PRELOAD="build/src/client/libfio_vitastor.so" \ +fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \ + -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 & +FIO_PID=$! + +sleep 15 +kill -9 $OSD1_PID $OSD4_PID + +wait $FIO_PID + +format_green OK