Fix PG_INCOMPLETE on node outage when allow_net_split is false

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_rm:
     runs-on: ubuntu-latest
     needs: build

src/osd/osd_cluster.cpp

@@ -410,7 +410,11 @@ void osd_t::report_statistics()
 
 void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 {
-    if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
+    if (st_cli.peer_states[peer_osd].is_null())
+    {
+        repeer_pgs(peer_osd);
+    }
+    else if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
         msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
     }

tests/run_tests.sh

@@ -36,6 +36,8 @@ SCHEME=ec ./test_snapshot.sh
 
 ./test_move_reappear.sh
 
+./test_degraded.sh
+
 ./test_rm.sh
 
 ./test_rm_degraded.sh

tests/test_degraded.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+OSD_COUNT=0
+PG_COUNT=32
+PG_SIZE=3
+PG_MINSIZE=2
+POOLCFG='"failure_domain":"host",'
+. `dirname $0`/run_3osds.sh
+
+$ETCDCTL put /vitastor/config/node_placement '{"1":{"parent":"host1"},"2":{"parent":"host2"},"3":{"parent":"host3"},"4":{"parent":"host1"},"5":{"parent":"host2"},"6":{"parent":"host3"},"host1":{"level":"host"},"host2":{"level":"host"}}'
+
+OSD_COUNT=6
+for i in $(seq 1 $OSD_COUNT); do
+    start_osd $i
+done
+wait_up 120
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
+FIO_PID=$!
+
+sleep 15
+kill -9 $OSD1_PID $OSD4_PID
+
+wait $FIO_PID
+
+format_green OK