Release 1.4.1

- Fix a monitor crash on primary OSD switching introduced in 1.4.0
- Fix "partly outside array bounds" warnings for GCC 12 in cpp-btree
- Fix a realloc memory leak in theory possible with too large listings (OSD_OP_LIST)

.gitea/workflows/test.yml

@@ -532,6 +532,24 @@ jobs:
           echo ""
         done
 
+  test_switch_primary:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_switch_primary.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_write:
     runs-on: ubuntu-latest
     needs: build

CLA-en.md

@@ -38,7 +38,7 @@ in the offer.
         on behalf of third parties, including on behalf of his employer.
 
 2. Subject of the Agreement. \
-   2.1. Subject of the Agreement shall be the Contributions sent to the Author by Contributors.
+   2.1. Subject of the Agreement shall be the Contributions sent to the Author by Contributors. \
    2.2. The Contributor grants to the Author the right to use Contributions at his own
         discretion and without any necessity to get a prior approval from Contributor or
         any other third party in any way, under a simple (non-exclusive), royalty-free,
@@ -86,7 +86,7 @@ in the offer.
         of their provision to the Author. \
    5.2. The Contributor represents and warrants that he legally owns exclusive
         intellectual property rights to the Contributions. \
-   5.3. The Contributor represents and warrants that any further use of \
+   5.3. The Contributor represents and warrants that any further use of
         Contributions by the Author as provided by Contributor under the terms
         of the Agreement does not infringe on intellectual and other rights and
         legitimate interests of third parties. \

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.4.0")
+set(VERSION "1.4.1")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.4.0
+VERSION ?= v1.4.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.4.0
+          image: vitalif/vitastor-csi:v1.4.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.4.0
+          image: vitalif/vitastor-csi:v1.4.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.4.0"
+    vitastorCSIDriverVersion = "1.4.1"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.4.0-1) unstable; urgency=medium
+vitastor (1.4.1-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.4.0; \
+    cp -r /root/vitastor vitastor-1.4.1; \
-    cd vitastor-1.4.0; \
+    cd vitastor-1.4.1; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.0.orig.tar.xz vitastor-1.4.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.1.orig.tar.xz vitastor-1.4.1; \
-    cd vitastor-1.4.0; \
+    cd vitastor-1.4.1; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/performance/theoretical.en.md

@@ -11,19 +11,26 @@ Replicated setups:
 - Single-threaded write+fsync latency:
   - With immediate commit: 2 network roundtrips + 1 disk write.
   - With lazy commit: 4 network roundtrips + 1 disk write + 1 disk flush.
-- Saturated parallel read iops: min(network bandwidth, sum(disk read iops)).
+- Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
-- Saturated parallel write iops: min(network bandwidth, sum(disk write iops / number of replicas / write amplification)).
+- Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
+- Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.
 
-EC/XOR setups:
+EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
 - Single-threaded write+fsync latency:
   - With immediate commit: 3.5 network roundtrips + 1 disk read + 2 disk writes.
   - With lazy commit: 5.5 network roundtrips + 1 disk read + 2 disk writes + 2 disk fsyncs.
-  - 0.5 in actually (k-1)/k which means that an additional roundtrip doesn't happen when
+  - 0.5 in actually `(N-1)/N` which means that an additional roundtrip doesn't happen when
     the read sub-operation can be served locally.
-- Saturated parallel read iops: min(network bandwidth, sum(disk read iops)).
+- Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
-- Saturated parallel write iops: min(network bandwidth, sum(disk write iops * number of data drives / (number of data + parity drives) / write amplification)).
+- Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
-  In fact, you should put disk write iops under the condition of ~10% reads / ~90% writes in this formula.
+- Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
+  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
+  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
+  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
+  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.
 
 Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
 1. Journal block write

docs/performance/theoretical.ru.md

@@ -11,20 +11,27 @@
 - Запись+fsync в 1 поток:
   - С мгновенным сбросом: 2 RTT + 1 запись.
   - С отложенным ("ленивым") сбросом: 4 RTT + 1 запись + 1 fsync.
-- Параллельное чтение: сумма IOPS всех дисков либо производительность сети, если в сеть упрётся раньше.
+- Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети (сумма пропускной способности сети всех нод), если в сеть упрётся раньше.
-- Параллельная запись: сумма IOPS всех дисков / число реплик / WA либо производительность сети, если в сеть упрётся раньше.
+- Линейная запись: сумма МБ/с записи всех дисков / число реплик, либо производительность сети / число реплик, если в сеть упрётся раньше.
+- Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков, либо производительность сети, если в сеть упрётся раньше.
+- Параллельная случайная мелкая запись: сумма IOPS записи всех дисков / число реплик / WA, либо производительность сети / число реплик, если в сеть упрётся раньше.
 
-При использовании кодов коррекции ошибок (EC):
+При использовании кодов коррекции ошибок (EC N+K):
 - Задержка чтения в 1 поток (T1Q1): 1.5 RTT + 1 чтение.
 - Запись+fsync в 1 поток:
   - С мгновенным сбросом: 3.5 RTT + 1 чтение + 2 записи.
   - С отложенным ("ленивым") сбросом: 5.5 RTT + 1 чтение + 2 записи + 2 fsync.
-- Под 0.5 на самом деле подразумевается (k-1)/k, где k - число дисков данных,
+- Под 0.5 на самом деле подразумевается (N-1)/N, где N - число дисков данных,
   что означает, что дополнительное обращение по сети не нужно, когда операция
   чтения обслуживается локально.
-- Параллельное чтение: сумма IOPS всех дисков либо производительность сети, если в сеть упрётся раньше.
+- Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
-- Параллельная запись: сумма IOPS всех дисков / общее число дисков данных и чётности / WA либо производительность сети, если в сеть упрётся раньше.
+- Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
-  Примечание: IOPS дисков в данном случае надо брать в смешанном режиме чтения/записи в пропорции, аналогичной формулам выше.
+- Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
+  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
+  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
+  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
+  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.
 
 WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
 1. Запись метаданных в журнал

mon/mon.js

@@ -390,7 +390,8 @@ class Mon
 {
     constructor(config)
     {
-        this.die = (e) => this._die(e);
+        this.failconnect = (e) => this._die(e, 2);
+        this.die = (e) => this._die(e, 1);
         if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
         {
             config = {
@@ -604,7 +605,7 @@ class Mon
         }
         if (!this.ws)
         {
-            this.die('Failed to open etcd watch websocket');
+            this.failconnect('Failed to open etcd watch websocket');
         }
         const cur_addr = this.selected_etcd_url;
         this.ws_alive = true;
@@ -791,7 +792,7 @@ class Mon
             const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
             if (!res.result.TTL)
             {
-                this.die('Lease expired');
+                this.failconnect('Lease expired');
             }
         }, this.config.etcd_mon_timeout);
         if (!this.signals_set)
@@ -1414,7 +1415,14 @@ class Mon
             }
             if (changed)
             {
-                await this.save_pg_config(new_config_pgs);
+                const ok = await this.save_pg_config(new_config_pgs);
+                if (ok)
+                    console.log('PG configuration successfully changed');
+                else
+                {
+                    console.log('Someone changed PG configuration while we also tried to change it. Retrying in '+this.config.mon_change_timeout+' ms');
+                    this.schedule_recheck();
+                }
             }
         }
         this.recheck_pgs_active = false;
@@ -1495,6 +1503,11 @@ class Mon
             this.save_new_pgs_txn(new_config_pgs, etcd_request, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
         new_config_pgs.hash = tree_hash;
+        return await this.save_pg_config(new_config_pgs, etcd_request);
+    }
+
+    async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
+    {
         etcd_request.compare.push(
             { key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
             { key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
@@ -1985,14 +1998,14 @@ class Mon
                 return res.json;
             }
         }
-        this.die();
+        this.failconnect();
     }
 
-    _die(err)
+    _die(err, code)
     {
         // In fact we can just try to rejoin
         console.error(new Error(err || 'Cluster connection failed'));
-        process.exit(1);
+        process.exit(code || 2);
     }
 
     local_ips(all)

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.4.0'
+VERSION = '1.4.1'
 
 LOG = logging.getLogger(__name__)
 

pull_request_template.yml

@@ -0,0 +1,28 @@
+name: Pull Request
+about: Submit a pull request
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: Describe your pull request
+      placeholder: ""
+      value: ""
+    validations:
+      required: true
+  - type: input
+    id: author
+    attributes:
+      label: Contributor Name
+      description: Contributor Name or Company Details if the Contributor is a company
+      placeholder: ""
+    validations:
+      required: false
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: CLA
+      description: By submitting this pull request, I accept [Vitastor CLA](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md)
+      options:
+        - label: "I accept Vitastor CLA agreement: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md"
+          required: true

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.4.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-1.4.1/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.1$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -36,7 +36,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.1.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.0
+Version:        1.4.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.0.el7.tar.gz
+Source0:        vitastor-1.4.1.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.1.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.0
+Version:        1.4.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.0.el8.tar.gz
+Source0:        vitastor-1.4.1.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.0.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.1.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.0
+Version:        1.4.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.0.el9.tar.gz
+Source0:        vitastor-1.4.1.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.4.0")
+add_definitions(-DVERSION="1.4.1")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore_impl.cpp

@@ -558,13 +558,14 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
             if (stable_count >= stable_alloc)
             {
                 stable_alloc *= 2;
-                stable = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
+                obj_ver_id* nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
-                if (!stable)
+                if (!nst)
                 {
                     op->retval = -ENOMEM;
                     FINISH_OP(op);
                     return;
                 }
+                stable = nst;
             }
             stable[stable_count++] = {
                 .oid = clean_it->first,
@@ -642,8 +643,8 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                             if (stable_count >= stable_alloc)
                             {
                                 stable_alloc += 32768;
-                                stable = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
+                                obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
-                                if (!stable)
+                                if (!nst)
                                 {
                                     if (unstable)
                                         free(unstable);
@@ -651,6 +652,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                                     FINISH_OP(op);
                                     return;
                                 }
+                                stable = nst;
                             }
                             stable[stable_count++] = dirty_it->first;
                         }
@@ -666,8 +668,8 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                     if (unstable_count >= unstable_alloc)
                     {
                         unstable_alloc += 32768;
-                        unstable = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc);
+                        obj_ver_id *nst = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc);
-                        if (!unstable)
+                        if (!nst)
                         {
                             if (stable)
                                 free(stable);
@@ -675,6 +677,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                             FINISH_OP(op);
                             return;
                         }
+                        unstable = nst;
                     }
                     unstable[unstable_count++] = dirty_it->first;
                 }
@@ -694,8 +697,8 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
     if (stable_count+unstable_count > stable_alloc)
     {
         stable_alloc = stable_count+unstable_count;
-        stable = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
+        obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
-        if (!stable)
+        if (!nst)
         {
             if (unstable)
                 free(unstable);
@@ -703,6 +706,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
             FINISH_OP(op);
             return;
         }
+        stable = nst;
     }
     // Copy unstable entries
     for (int i = 0; i < unstable_count; i++)

src/etcd_state_client.h

@@ -28,7 +28,7 @@ struct etcd_kv_t
 {
     std::string key;
     json11::Json value;
-    uint64_t mod_revision;
+    uint64_t mod_revision = 0;
 };
 
 struct pg_config_t

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.4.0
+Version: 1.4.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_3osds.sh

@@ -10,6 +10,7 @@ SCHEME=${SCHEME:-replicated}
 # OFFSET_ARGS
 # PG_SIZE
 # PG_MINSIZE
+# GLOBAL_CONFIG
 
 if [ "$SCHEME" = "ec" ]; then
     OSD_COUNT=${OSD_COUNT:-5}
@@ -19,10 +20,10 @@ fi
 
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"immediate_commit":"all","client_enable_writeback":true}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"osd_out_time":1,"client_enable_writeback":true}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
 start_osd_on()
@@ -53,7 +54,7 @@ for i in $(seq 1 $OSD_COUNT); do
     start_osd $i
 done
 
-(while true; do node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 || true; done) >>./testdata/mon.log 2>&1 &
+(while true; do set +e; node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1; if [[ $? -ne 2 ]]; then break; fi; done) >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 
 if [ "$SCHEME" = "ec" ]; then

tests/run_tests.sh

@@ -45,6 +45,8 @@ IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 SCHEME=ec ./test_rebalance_verify.sh
 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
 
+./test_switch_primary.sh
+
 ./test_write.sh
 SCHEME=xor ./test_write.sh
 

tests/test_add_osd.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
 PG_COUNT=2048
-
+GLOBAL_CONFIG=',"osd_out_time":1'
 . `dirname $0`/run_3osds.sh
 
 LD_PRELOAD="build/src/libfio_vitastor.so" \

tests/test_heal.sh

@@ -9,6 +9,7 @@ if [[ "$SCHEME" = "ec" ]]; then
 fi
 OSD_COUNT=${OSD_COUNT:-7}
 PG_COUNT=32
+GLOBAL_CONFIG=',"osd_out_time":1'
 . `dirname $0`/run_3osds.sh
 check_qemu
 

tests/test_minsize_1.sh

@@ -2,6 +2,7 @@
 
 PG_MINSIZE=1
 SCHEME=replicated
+GLOBAL_CONFIG=',"osd_out_time":1'
 
 . `dirname $0`/run_3osds.sh
 

tests/test_splitbrain.sh

@@ -4,6 +4,7 @@ OSD_COUNT=2
 PG_SIZE=2
 PG_MINSIZE=1
 SCHEME=replicated
+GLOBAL_CONFIG=',"osd_out_time":1'
 
 . `dirname $0`/run_3osds.sh
 

tests/test_switch_primary.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_3osds.sh
+
+primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+primary_pid=OSD${primary}_PID
+kill -9 ${!primary_pid}
+
+sleep 15
+wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'"
+
+newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+
+if [ "$newprim" = "$primary" ]; then
+    format_error Primary not switched
+fi
+
+format_green OK