Release 2.4.3

- Daemonize before forking in NFS proxy to fix OSD RDMA support (#107)
- Fix possible PG_INCOMPLETE on node outage when allow_net_split is false
- Fix build under Ubuntu 25.10
- Fix journal_no_same_sector_overwrites mode after optimizing WA (#109)

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_rm:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.4.0")
+set(VITASTOR_VERSION "2.4.3")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.0
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.4.0
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.4.0
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.4.0"
+    vitastorCSIDriverVersion = "2.4.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-vitastor-bookworm.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=bookworm -t vitastor-buildenv:bookworm -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-bullseye.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=bullseye -t vitastor-buildenv:bullseye -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-buster.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=buster -t vitastor-buildenv:buster -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-trixie.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=trixie -t vitastor-buildenv:trixie -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-jammy.sh

@@ -2,4 +2,4 @@
 # Ubuntu 22.04 Jammy Jellyfish
 
 docker build --build-arg DISTRO=ubuntu --build-arg REL=jammy -t vitastor-buildenv:jammy -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-noble.sh

@@ -2,4 +2,4 @@
 # 24.04 Noble Numbat
 
 docker build --build-arg DISTRO=ubuntu --build-arg REL=noble -t vitastor-buildenv:noble -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-questing.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# 25.10 Questing quokka
+
+docker build --build-arg DISTRO=ubuntu --build-arg REL=questing -t vitastor-buildenv:questing -f vitastor-buildenv.Dockerfile .
+docker run -it --rm -e REL=questing -v `dirname $0`/../:/root/vitastor vitastor-buildenv:questing /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.4.0-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor-build.sh

@@ -44,7 +44,12 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
 
 cd /root/vitastor/packages/vitastor-$REL
-tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+    # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
+    cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
+else
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+fi
 cd vitastor-$VER
 DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"
 DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.0
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.4.0
+VITASTOR_VERSION=v2.4.3
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.4.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.4.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/openstack.en.md

@@ -30,7 +30,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True

docs/installation/openstack.ru.md

@@ -29,7 +29,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.4.0",
+  "version": "2.4.3",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/scripts/make-etcd

@@ -52,15 +52,16 @@ async function run()
         process.exit(1);
     }
     const etcds = (config.etcd_address instanceof Array ? config.etcd_address : (''+config.etcd_address).split(/,/))
-        .map(s => (''+s).replace(/^https?:\/\/\[?|\]?(:\d+)?(\/.*)?$/g, '').toLowerCase());
+        .map(s => (''+s).replace(/^https?:\/\/|(:\d+)?(\/.*)?$/g, '').replace(/^\[(.*)\]$/, '$1').toLowerCase());
     const num = select_local_etcd(etcds);
     if (num < 0)
     {
         console.log('No matching IPs in etcd_address from '+config_path);
         process.exit(0);
     }
+    const etcd_url = 'http://' + (etcds[num].indexOf(':') >= 0 ? '['+etcds[num]+']' : etcds[num]);
     const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
-    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e.indexOf(':') >= 0 ? '['+e+']' : e}:2380`).join(',');
     if (in_docker)
     {
         let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
@@ -83,8 +84,8 @@ Wants=network-online.target local-fs.target time-sync.target
 Restart=always
 Environment=GOGC=50
 ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd/vitastor \\
-    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
+    --snapshot-count 10000 --advertise-client-urls ${etcd_url}:2379 --listen-client-urls ${etcd_url}:2379 \\
-    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
+    --initial-advertise-peer-urls ${etcd_url}:2380 --listen-peer-urls ${etcd_url}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
     --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
     --auto-compaction-retention=10 --auto-compaction-mode=revision

mon/stats.js

@@ -276,6 +276,10 @@ function sum_inode_stats(state, prev_stats)
         }
         for (const pool_id in osd_diff.inode_stats)
         {
+            if (!inode_stats[pool_id])
+            {
+                continue;
+            }
             for (const inode_num in prev_stats.osd_diff[osd].inode_stats[pool_id])
             {
                 inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.4.0",
+  "version": "2.4.3",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.4.0'
+VITASTOR_VERSION = '2.4.3'
 
 LOG = logging.getLogger(__name__)
 

patches/pve-qemu-10.1-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-10.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/block/meson.build
++++ pve-qemu-kvm-10.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-10.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson.build
++++ pve-qemu-kvm-10.1.2/meson.build
+@@ -1653,6 +1653,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2552,6 +2572,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4984,6 +5005,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-10.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson_options.txt
++++ pve-qemu-kvm-10.1.2/meson_options.txt
+@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'a
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-10.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/qapi/block-core.json
++++ pve-qemu-kvm-10.1.2/qapi/block-core.json
+@@ -3647,7 +3647,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4773,6 +4773,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5242,6 +5264,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5722,6 +5745,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5943,6 +5980,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+@@ -175,6 +175,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -459,6 +460,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-10.1-vitastor.patch

@@ -0,0 +1,29 @@
+diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
+index d8356dab..5f4cd50d 100644
+--- a/src/client/qemu_driver.c
++++ b/src/client/qemu_driver.c
+@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
+ #endif
+ }
+ 
+-static int coroutine_fn vitastor_co_block_status(
+-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
+-    int64_t *pnum, int64_t *map, BlockDriverState **file)
++static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
++    unsigned int mode,
++#else
++    bool want_zero,
++#endif
++    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
+ {
+     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
+     // Not allocated => return 0
+     // Error => return -errno
+     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
++    int want_zero = (mode == BDRV_WANT_PRECISE);
++#endif
+     VitastorRPC task;
+     VitastorClient *client = bs->opaque;
+     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.4.0")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_impl.cpp

@@ -228,6 +228,8 @@ void blockstore_impl_t::loop()
         for (auto s: journal.submitting_sectors)
         {
             // Mark journal sector writes as submitted
+            if (journal.sector_info[s].submit_id)
+                journal.sector_info[s].written = true;
             journal.sector_info[s].submit_id = 0;
         }
         journal.submitting_sectors.clear();

src/blockstore/blockstore_journal.cpp

@@ -178,7 +178,7 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
         // Caller must ensure availability of an SQE
         assert(sqe != NULL);
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        journal.sector_info[cur_sector].written = true;
+        // <written> flag will be set at the moment of actual submission
         journal.sector_info[cur_sector].submit_id = ++journal.submit_id;
         assert(journal.submit_id != 0); // check overflow
         journal.submitting_sectors.push_back(cur_sector);

src/blockstore/fio_engine.cpp

@@ -26,6 +26,7 @@
 
 #include "blockstore.h"
 #include "epoll_manager.h"
+#include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "fio_headers.h"
 
@@ -37,6 +38,8 @@ struct bs_data
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
     int op_n = 0, inflight = 0;
+    bool ec = false;
+    bool imm = true;
     bool last_sync = false;
     bool trace = false;
 };
@@ -45,6 +48,7 @@ struct bs_options
 {
     int __pad;
     char *json_config = NULL;
+    int ec = 0;
     int trace = 0;
 };
 
@@ -58,6 +62,16 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
+    {
+        .name   = "ec",
+        .lname  = "Use EC write method",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct bs_options, ec),
+        .help   = "Use EC write method",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
     {
         .name   = "bs_trace",
         .lname  = "trace",
@@ -88,6 +102,7 @@ static int bs_setup(struct thread_data *td)
         return 1;
     }
     td->io_ops_data = bsd;
+    bsd->ec = o->ec;
 
     if (!td->files_index)
     {
@@ -148,6 +163,8 @@ static int bs_init(struct thread_data *td)
     bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
     bsd->bs = new blockstore_t(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->imm = config.find("immediate_commit") == config.end() ||
+        config["immediate_commit"] == "all";
     while (1)
     {
         bsd->ringloop->loop();
@@ -203,7 +220,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         };
         break;
     case DDIR_WRITE:
-        op->opcode = BS_OP_WRITE_STABLE;
+        op->opcode = bsd->ec ? BS_OP_WRITE : BS_OP_WRITE_STABLE;
         op->buf = io->xfer_buf;
         op->oid = {
             .inode = 1,
@@ -212,16 +229,57 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = 0; // assign automatically
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        if (bsd->ec)
         {
-            io->error = op->retval < 0 ? -op->retval : 0;
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
-            bs_data *bsd = (bs_data*)io->engine_data;
+            {
-            bsd->inflight--;
+                bs_data *bsd = (bs_data*)io->engine_data;
-            bsd->completed.push_back(io);
+                if (bsd->trace)
-            if (bsd->trace)
+                    printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
-                printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                if (op->retval < 0)
-            delete op;
+                {
-        };
+                    io->error = op->retval < 0 ? -op->retval : 0;
+                    bsd->inflight--;
+                    bsd->completed.push_back(io);
+                    delete op;
+                }
+                else
+                {
+                    auto stab_op = new blockstore_op_t;
+                    stab_op->opcode = BS_OP_STABLE;
+                    stab_op->buf = malloc_or_die(sizeof(obj_ver_id));
+                    obj_ver_id *ver = (obj_ver_id *)stab_op->buf;
+                    ver[0].oid = op->oid;
+                    ver[0].version = op->version;
+                    stab_op->len = 1;
+                    stab_op->callback = [io, n](blockstore_op_t *op)
+                    {
+                        bs_data *bsd = (bs_data*)io->engine_data;
+                        if (bsd->trace)
+                            printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                        io->error = op->retval < 0 ? -op->retval : 0;
+                        bsd->inflight--;
+                        bsd->completed.push_back(io);
+                        delete op;
+                    };
+                    bsd->bs->enqueue_op(stab_op);
+                    delete op;
+                }
+            };
+        }
+        else
+        {
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+            {
+                bs_data *bsd = (bs_data*)io->engine_data;
+                if (bsd->trace)
+                    printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                io->error = op->retval < 0 ? -op->retval : 0;
+                bsd->inflight--;
+                bsd->completed.push_back(io);
+                delete op;
+            };
+        }
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:

src/client/cluster_client.cpp

@@ -1317,7 +1317,8 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
     auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     auto pg_it = pool_cfg.pg_config.find(part->pg_num);
     if (pg_it != pool_cfg.pg_config.end() &&
-        !pg_it->second.pause && pg_it->second.cur_primary)
+        !pg_it->second.pause && pg_it->second.cur_primary &&
+        (pg_it->second.cur_state & PG_ACTIVE))
     {
         osd_num_t primary_osd = pg_it->second.cur_primary;
         if (pool_cfg.local_reads != POOL_LOCAL_READ_PRIMARY &&

src/client/cluster_client.h

@@ -127,7 +127,7 @@ public:
     ring_consumer_t consumer;
     std::vector<std::function<void(void)>> on_ready_hooks;
     int list_retry_timeout_id = -1;
-    timespec list_retry_time;
+    timespec list_retry_time = {};
     std::vector<inode_list_t*> lists;
     std::multimap<osd_num_t, osd_op_t*> raw_ops;
     int continuing_ops = 0;

src/client/cluster_client_list.cpp

@@ -31,7 +31,7 @@ struct inode_list_pg_t
     osd_num_t cur_primary = 0;
     int state = 0;
     int inflight_ops = 0;
-    timespec wait_until;
+    timespec wait_until = {};
     std::vector<inode_list_osd_t> list_osds;
 
     bool has_unstable = false;
@@ -177,6 +177,9 @@ void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg)
         if (tv.tv_sec < pg->wait_until.tv_sec ||
             tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec < pg->wait_until.tv_nsec)
         {
+            // Ensure that the retry timer is set
+            set_list_retry_timeout(pg->wait_until.tv_sec*1000 - tv.tv_sec*1000 +
+                (pg->wait_until.tv_nsec - tv.tv_nsec)/1000000, pg->wait_until);
             return;
         }
     }

src/client/messenger.cpp

@@ -276,6 +276,7 @@ osd_messenger_t::~osd_messenger_t()
 #ifdef WITH_RDMA
     for (auto rdma_context: rdma_contexts)
     {
+        tfd->set_fd_handler(rdma_context->channel->fd, false, NULL);
         delete rdma_context;
     }
     rdma_contexts.clear();

src/client/qemu_driver.c

@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
 #endif
 }
 
-static int coroutine_fn vitastor_co_block_status(
+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-    int64_t *pnum, int64_t *map, BlockDriverState **file)
+    unsigned int mode,
+#else
+    bool want_zero,
+#endif
+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
 {
     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
     // Not allocated => return 0
     // Error => return -errno
     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    int want_zero = (mode == BDRV_WANT_PRECISE);
+#endif
     VitastorRPC task;
     VitastorClient *client = bs->opaque;
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.4.0
+Version: 2.4.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_ls.cpp

@@ -94,16 +94,14 @@ struct image_lister_t
         parent->etcd_txn(json11::Json::object {
             { "success", json11::Json::array {
                 json11::Json::object {
-                    { "request_range", json11::Json::object {
+                    { "request_range", (list_pool_id
-                        { "key", base64_encode(
+                        ? json11::Json::object {
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/"+std::to_string(list_pool_id)) },
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"/"
+                        }
-                        ) },
+                        : json11::Json::object {
-                        { "range_end", base64_encode(
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/") },
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats0") },
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"0"
+                        }) },
-                        ) },
-                    } },
                 },
                 json11::Json::object {
                     { "request_range", json11::Json::object {

src/cmd/cli_pool_cfg.cpp

@@ -180,7 +180,8 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
         {
             return "Changing scheme for an existing pool will lead to data loss. Use --force to proceed";
         }
-        if (etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value()) == POOL_SCHEME_EC)
+        auto old_scheme = etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value());
+        if (old_scheme == POOL_SCHEME_EC)
         {
             uint64_t old_data_chunks = old_cfg["pg_size"].uint64_value() - old_cfg["parity_chunks"].uint64_value();
             uint64_t new_data_chunks = cfg["pg_size"].uint64_value() - cfg["parity_chunks"].uint64_value();
@@ -189,6 +190,13 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
                 return "Changing EC data chunk count for an existing pool will lead to data loss. Use --force to proceed";
             }
         }
+        else if (old_scheme != POOL_SCHEME_REPLICATED)
+        {
+            if (old_cfg["pg_size"].uint64_value() != cfg["pg_size"].uint64_value())
+            {
+                return "Changing XOR data chunk count for an existing pool will lead to data loss. Use --force to proceed";
+            }
+        }
         if (old_cfg["block_size"] != cfg["block_size"] ||
             old_cfg["bitmap_granularity"] != cfg["bitmap_granularity"] ||
             old_cfg["immediate_commit"] != cfg["immediate_commit"])

src/cmd/cli_pool_ls.cpp

@@ -87,6 +87,16 @@ struct pool_lister_t
                         ) },
                     } },
                 },
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd/"
+                        ) },
+                        { "range_end", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd0"
+                        ) },
+                    } },
+                },
             } },
         });
         state = base_state+1;
@@ -115,6 +125,14 @@ resume_1:
             // osd/stats/<N>::free
             osd_free[osd_num] = value["free"].uint64_value();
         });
+        std::map<uint64_t, double> osd_reweight;
+        parent->iterate_kvs_1(space_info["responses"][3]["response_range"]["kvs"], "/config/osd/", [&](uint64_t osd_num, json11::Json value)
+        {
+            if (value.object_items().find("reweight") != value.object_items().end())
+            {
+                osd_reweight[osd_num] = value["reweight"].number_value();
+            }
+        });
         // Calculate max_avail for each pool
         for (auto & pp: parent->cli->st_cli.pool_config)
         {
@@ -140,7 +158,9 @@ resume_1:
             }
             for (auto pg_per_pair: pg_per_osd)
             {
-                uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
+                uint64_t pg_free = osd_free[pg_per_pair.first] *
+                    (osd_reweight.find(pg_per_pair.first) != osd_reweight.end() ? osd_reweight[pg_per_pair.first] : 1.0) *
+                    pool_cfg.real_pg_count / pg_per_pair.second;
                 if (pool_avail > pg_free)
                 {
                     pool_avail = pg_free;

src/cmd/cli_rm.cpp

@@ -97,8 +97,6 @@ struct snap_remover_t
             goto resume_8;
         else if (state == 9)
             goto resume_9;
-        else if (state == 10)
-            goto resume_10;
         else if (state == 100)
             goto resume_100;
         assert(!state);
@@ -157,11 +155,6 @@ resume_3:
 resume_4:
             while (!wait_result(4))
                 return;
-            // Mark child as deleted
-            start_mark_deleted(inverse_child);
-resume_9:
-            while (!wait_result(9))
-                return;
             // Delete "inverse" child data
             start_delete_source(inverse_child);
 resume_5:
@@ -190,8 +183,8 @@ resume_6:
             }
             // Mark child as deleted
             start_mark_deleted(chain_list[current_child]);
-resume_10:
+resume_9:
-            while (!wait_result(10))
+            while (!wait_result(9))
                 return;
             start_delete_source(chain_list[current_child]);
 resume_7:
@@ -456,6 +449,7 @@ resume_100:
         );
         // Fill new configuration
         inode_config_t new_cfg = *child_cfg;
+        new_cfg.deleted = false;
         new_cfg.num = target_cfg->num;
         new_cfg.parent_id = new_parent;
         json11::Json::array cmp = json11::Json::array {

src/kv/kv_db.cpp

@@ -873,7 +873,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
         }
         // Block already in cache, we can proceed
         blk->usage = db->usage_counter;
-        db->cli->msgr.ringloop->set_immediate([=] { cb(0, BLK_UPDATING); });
+        cb(0, BLK_UPDATING);
         return;
     }
     cluster_op_t *op = new cluster_op_t;

src/nfs/nfs_kv_getattr.cpp

@@ -94,6 +94,10 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
     uint32_t uid = attrs["uid"].uint64_value();
     uint32_t gid = attrs["gid"].uint64_value();
     uint32_t access = 0;
+    if (!auth_sys.uid)
+    {
+        return (ACCESS3_READ|ACCESS3_LOOKUP|ACCESS3_MODIFY|ACCESS3_EXTEND|ACCESS3_DELETE|ACCESS3_EXECUTE);
+    }
     if (uid == auth_sys.uid)
     {
         access |= ((mode & (1 << 8)) ? ACCESS3_READ|ACCESS3_LOOKUP : 0);
@@ -120,6 +124,8 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
 bool kv_is_accessible(const authsys_parms & auth_sys, const json11::Json & attrs, uint32_t access)
 {
     uint32_t mode = attrs["mode"].is_null() ? (attrs["type"] == "dir" ? 0755 : 0644) : attrs["mode"].uint64_value();
+    if (!auth_sys.uid)
+        return true;
     uint32_t mask = 1 << (access == ACCESS3_EXECUTE ? 0
         : (access == ACCESS3_MODIFY || access == ACCESS3_EXTEND || access == ACCESS3_DELETE ? 1 : 2));
     if (mode & mask)

src/nfs/nfs_proxy.cpp

@@ -276,6 +276,19 @@ void nfs_proxy_t::run(json11::Json cfg)
     }
     // Check default pool
     check_default_pool();
+    // Daemonize before initializing messenger and RDMA because otherwise RDMA doesn't survive fork()
+    bool bg = cfg["foreground"].is_null() && cfg["cmd"].is_null();
+    int notifyfd[2] = { -1, -1 };
+    if (bg)
+    {
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
+        {
+            perror("socketpair");
+            exit(1);
+        }
+        daemonize_fork(notifyfd);
+        close(notifyfd[0]);
+    }
     // Init VitastorFS after starting client because it depends on loaded inode configuration
     if (fsname != "")
     {
@@ -294,6 +307,13 @@ void nfs_proxy_t::run(json11::Json cfg)
     {
         kvfs->upgrade_db([this](int res) { finished = true; });
     }
+    if (bg)
+    {
+        daemonize_reopen_stdio();
+        int ok = 0;
+        (void)write(notifyfd[1], &ok, sizeof(ok));
+        close(notifyfd[1]);
+    }
     while (!finished)
     {
         ringloop->loop();
@@ -409,10 +429,6 @@ void nfs_proxy_t::run_server(json11::Json cfg)
     {
         mount_fs();
     }
-    if (cfg["foreground"].is_null())
-    {
-        daemonize();
-    }
     if (pidfile != "")
     {
         write_pid();
@@ -699,7 +715,8 @@ void nfs_client_t::handle_read(int result)
         return;
     if (result <= 0 && result != -EAGAIN && result != -EINTR && result != -ECANCELED)
     {
-        printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
+        if (result != 0)
+            printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
         stop();
         return;
     }
@@ -1224,26 +1241,31 @@ void nfs_client_t::free_or_rdma(rpc_op_t *rop, void *buf)
 #endif
 }
 
-void nfs_proxy_t::daemonize()
+void nfs_proxy_t::daemonize_fork(int *notifyfd)
 {
-    // Stop all clients because client I/O sometimes breaks during daemonize
-    // I.e. the new process stops receiving events on the old FD
-    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & cli: rpc_clients)
-        cli->stop();
     if (fork())
-        exit(0);
+    {
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 1;
+        (void)read(notifyfd[0], &child_errno, sizeof(child_errno));
+        exit(child_errno);
+    }
     setsid();
     if (fork())
         exit(0);
-    if (chdir("/") != 0)
+}
-        fprintf(stderr, "Warning: Failed to chdir into /\n");
+
+void nfs_proxy_t::daemonize_reopen_stdio()
+{
     close(0);
     close(1);
     close(2);
     open("/dev/null", O_RDONLY);
     open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
     open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
+    if (chdir("/") != 0)
+        fprintf(stderr, "Warning: Failed to chdir into /\n");
 }
 
 void nfs_proxy_t::write_pid()

src/nfs/nfs_proxy.h

@@ -83,7 +83,8 @@ public:
     void check_default_pool();
     nfs_client_t* create_client();
     void do_accept(int listen_fd);
-    void daemonize();
+    void daemonize_fork(int *notifyfd);
+    void daemonize_reopen_stdio();
     void write_pid();
     void mount_fs();
     void check_already_mounted();

src/osd/osd_cluster.cpp

@@ -410,7 +410,11 @@ void osd_t::report_statistics()
 
 void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 {
-    if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
+    if (st_cli.peer_states[peer_osd].is_null())
+    {
+        repeer_pgs(peer_osd);
+    }
+    else if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
         msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
     }

src/osd/osd_peering_pg.h

@@ -4,6 +4,7 @@
 #include <map>
 #include <vector>
 #include <algorithm>
+#include <stdint.h>
 
 #include "cpp-btree/btree_map.h"
 

src/util/addr_util.cpp

@@ -17,7 +17,7 @@ bool string_to_addr(std::string str, bool parse_port, int default_port, struct s
     if (parse_port)
     {
         int p = str.rfind(':');
-        if (p != std::string::npos && !(str.length() > 0 && str[p-1] == ']')) // "[ipv6]" which contains ':'
+        if (p != std::string::npos && (str[0] != '[' || p > 0 && str[p-1] == ']')) // "[ipv6]" which contains ':'
         {
             char null_byte = 0;
             int scanned = sscanf(str.c_str()+p+1, "%d%c", &default_port, &null_byte);

src/util/timerfd_manager.cpp

@@ -36,7 +36,7 @@ void timerfd_manager_t::inc_timer(timerfd_timer_t & t)
 {
     t.next.tv_sec += t.micros/1000000;
     t.next.tv_nsec += (t.micros%1000000)*1000;
-    if (t.next.tv_nsec > 1000000000)
+    if (t.next.tv_nsec >= 1000000000)
     {
         t.next.tv_sec++;
         t.next.tv_nsec -= 1000000000;

tests/run_tests.sh

@@ -36,6 +36,8 @@ SCHEME=ec ./test_snapshot.sh
 
 ./test_move_reappear.sh
 
+./test_degraded.sh
+
 ./test_rm.sh
 
 ./test_rm_degraded.sh

tests/test_degraded.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+OSD_COUNT=0
+PG_COUNT=32
+PG_SIZE=3
+PG_MINSIZE=2
+POOLCFG='"failure_domain":"host",'
+. `dirname $0`/run_3osds.sh
+
+$ETCDCTL put /vitastor/config/node_placement '{"1":{"parent":"host1"},"2":{"parent":"host2"},"3":{"parent":"host3"},"4":{"parent":"host1"},"5":{"parent":"host2"},"6":{"parent":"host3"},"host1":{"level":"host"},"host2":{"level":"host"}}'
+
+OSD_COUNT=6
+for i in $(seq 1 $OSD_COUNT); do
+    start_osd $i
+done
+wait_up 120
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
+FIO_PID=$!
+
+sleep 15
+kill -9 $OSD1_PID $OSD4_PID
+
+wait $FIO_PID
+
+format_green OK

tests/test_nfs.sh

@@ -13,6 +13,12 @@ sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft
 MNT=$(pwd)/testdata/nfs
 trap "sudo umount -f $MNT"' || true; kill -9 $(jobs -p)' EXIT
 
+touch ./testdata/nfs/f1
+chown 1000:1000 ./testdata/nfs/f1
+chmod 600 ./testdata/nfs/f1
+sudo cat ./testdata/nfs/f1
+rm ./testdata/nfs/f1
+
 # write small file
 ls -l ./testdata/nfs
 dd if=/dev/urandom of=./testdata/f1 bs=100k count=1