Add a test for simple master switching without PG reconfiguration

Also use osd_out_time:1 only in select tests and restart mon in tests only on connection errors

.gitea/workflows/test.yml

@@ -532,6 +532,24 @@ jobs:
           echo ""
         done
 
+  test_switch_primary:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_switch_primary.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_write:
     runs-on: ubuntu-latest
     needs: build

mon/mon.js

@@ -390,7 +390,8 @@ class Mon
 {
     constructor(config)
     {
-        this.die = (e) => this._die(e);
+        this.failconnect = (e) => this._die(e, 2);
+        this.die = (e) => this._die(e, 1);
         if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
         {
             config = {
@@ -604,7 +605,7 @@ class Mon
         }
         if (!this.ws)
         {
-            this.die('Failed to open etcd watch websocket');
+            this.failconnect('Failed to open etcd watch websocket');
         }
         const cur_addr = this.selected_etcd_url;
         this.ws_alive = true;
@@ -791,7 +792,7 @@ class Mon
             const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
             if (!res.result.TTL)
             {
-                this.die('Lease expired');
+                this.failconnect('Lease expired');
             }
         }, this.config.etcd_mon_timeout);
         if (!this.signals_set)
@@ -1997,14 +1998,14 @@ class Mon
                 return res.json;
             }
         }
-        this.die();
+        this.failconnect();
     }
 
-    _die(err)
+    _die(err, code)
     {
         // In fact we can just try to rejoin
         console.error(new Error(err || 'Cluster connection failed'));
-        process.exit(1);
+        process.exit(code || 2);
     }
 
     local_ips(all)

tests/run_3osds.sh

@@ -10,6 +10,7 @@ SCHEME=${SCHEME:-replicated}
 # OFFSET_ARGS
 # PG_SIZE
 # PG_MINSIZE
+# GLOBAL_CONFIG
 
 if [ "$SCHEME" = "ec" ]; then
     OSD_COUNT=${OSD_COUNT:-5}
@@ -19,10 +20,10 @@ fi
 
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"immediate_commit":"all","client_enable_writeback":true}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"client_enable_writeback":true}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
 start_osd_on()
@@ -53,7 +54,7 @@ for i in $(seq 1 $OSD_COUNT); do
     start_osd $i
 done
 
-(while true; do node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 || true; done) >>./testdata/mon.log 2>&1 &
+(while true; do set +e; node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1; if [[ $? -ne 2 ]]; then break; fi; done) >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 
 if [ "$SCHEME" = "ec" ]; then

tests/run_tests.sh

@@ -45,6 +45,8 @@ IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 SCHEME=ec ./test_rebalance_verify.sh
 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 
+./test_switch_primary.sh
+
 ./test_write.sh
 SCHEME=xor ./test_write.sh
 

tests/test_add_osd.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
 PG_COUNT=2048
-
+GLOBAL_CONFIG=',"osd_out_time":1'
 . `dirname $0`/run_3osds.sh
 
 LD_PRELOAD="build/src/libfio_vitastor.so" \

tests/test_heal.sh

@@ -9,6 +9,7 @@ if [[ "$SCHEME" = "ec" ]]; then
 fi
 OSD_COUNT=${OSD_COUNT:-7}
 PG_COUNT=32
+GLOBAL_CONFIG=',"osd_out_time":1'
 . `dirname $0`/run_3osds.sh
 check_qemu
 

tests/test_minsize_1.sh

@@ -2,6 +2,7 @@
 
 PG_MINSIZE=1
 SCHEME=replicated
+GLOBAL_CONFIG=',"osd_out_time":1'
 
 . `dirname $0`/run_3osds.sh
 

tests/test_splitbrain.sh

@@ -4,6 +4,7 @@ OSD_COUNT=2
 PG_SIZE=2
 PG_MINSIZE=1
 SCHEME=replicated
+GLOBAL_CONFIG=',"osd_out_time":1'
 
 . `dirname $0`/run_3osds.sh
 

tests/test_switch_primary.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_3osds.sh
+
+primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+primary_pid=OSD${primary}_PID
+kill -9 ${!primary_pid}
+
+sleep 15
+wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'"
+
+newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+
+if [ "$newprim" = "$primary" ]; then
+    format_error Primary not switched
+fi
+
+format_green OK