Release 2.4.3

- Daemonize before forking in NFS proxy to fix OSD RDMA support (#107) - Fix possible PG_INCOMPLETE on node outage when allow_net_split is false - Fix build under Ubuntu 25.10 - Fix journal_no_same_sector_overwrites mode after optimizing WA (#109)
Daemonize before forking in NFS proxy to fix OSD RDMA support (fix #107 )
2025-11-09 02:18:35 +03:00 · 2025-11-09 01:07:11 +03:00 · 2025-11-08 14:11:03 +03:00 · 2025-11-07 21:56:56 +03:00 · 2025-11-07 21:56:56 +03:00 · 2025-11-07 00:59:04 +03:00
56 changed files with 489 additions and 93 deletions
@@ -414,6 +414,24 @@ jobs:
          echo ""
        done

+  test_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
  test_rm:
    runs-on: ubuntu-latest
    needs: build
@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

-set(VITASTOR_VERSION "2.4.0")
+set(VITASTOR_VERSION "2.4.3")

 add_subdirectory(src)
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.0
+VITASTOR_VERSION ?= v2.4.3

 all: build push

@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.4.0
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.4.0
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor

 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.4.0"
+    vitastorCSIDriverVersion = "2.4.3"
 )

 // Config struct fills the parameters of request or user input
@@ -1,4 +1,4 @@
 #!/bin/bash

 docker build --build-arg DISTRO=debian --build-arg REL=bookworm -t vitastor-buildenv:bookworm -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh
@@ -1,4 +1,4 @@
 #!/bin/bash

 docker build --build-arg DISTRO=debian --build-arg REL=bullseye -t vitastor-buildenv:bullseye -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh
@@ -1,4 +1,4 @@
 #!/bin/bash

 docker build --build-arg DISTRO=debian --build-arg REL=buster -t vitastor-buildenv:buster -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh
@@ -1,4 +1,4 @@
 #!/bin/bash

 docker build --build-arg DISTRO=debian --build-arg REL=trixie -t vitastor-buildenv:trixie -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh
@@ -2,4 +2,4 @@
 # Ubuntu 22.04 Jammy Jellyfish

 docker build --build-arg DISTRO=ubuntu --build-arg REL=jammy -t vitastor-buildenv:jammy -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh
@@ -2,4 +2,4 @@
 # 24.04 Noble Numbat

 docker build --build-arg DISTRO=ubuntu --build-arg REL=noble -t vitastor-buildenv:noble -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+# 25.10 Questing quokka
+
+docker build --build-arg DISTRO=ubuntu --build-arg REL=questing -t vitastor-buildenv:questing -f vitastor-buildenv.Dockerfile .
+docker run -it --rm -e REL=questing -v `dirname $0`/../:/root/vitastor vitastor-buildenv:questing /root/vitastor/debian/vitastor-build.sh
@@ -1,4 +1,4 @@
-vitastor (2.4.0-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium

  * Bugfixes

@@ -44,7 +44,12 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx

 cd /root/vitastor/packages/vitastor-$REL
-tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+    # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
+    cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
+else
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+fi
 cd vitastor-$VER
 DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"
 DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.0
+VITASTOR_VERSION ?= v2.4.3

 all: build push

@@ -4,7 +4,7 @@
 #

 # Desired Vitastor version
-VITASTOR_VERSION=v2.4.0
+VITASTOR_VERSION=v2.4.3

 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here
@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.

 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.4.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
   `udevadm control --reload-rules`

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.

 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.4.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
   `udevadm control --reload-rules`

@@ -30,7 +30,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True
@@ -29,7 +29,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "2.4.0",
+  "version": "2.4.3",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -52,15 +52,16 @@ async function run()
        process.exit(1);
    }
    const etcds = (config.etcd_address instanceof Array ? config.etcd_address : (''+config.etcd_address).split(/,/))
-        .map(s => (''+s).replace(/^https?:\/\/\[?|\]?(:\d+)?(\/.*)?$/g, '').toLowerCase());
+        .map(s => (''+s).replace(/^https?:\/\/|(:\d+)?(\/.*)?$/g, '').replace(/^\[(.*)\]$/, '$1').toLowerCase());
    const num = select_local_etcd(etcds);
    if (num < 0)
    {
        console.log('No matching IPs in etcd_address from '+config_path);
        process.exit(0);
    }
+    const etcd_url = 'http://' + (etcds[num].indexOf(':') >= 0 ? '['+etcds[num]+']' : etcds[num]);
    const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
-    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e.indexOf(':') >= 0 ? '['+e+']' : e}:2380`).join(',');
    if (in_docker)
    {
        let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
@@ -83,8 +84,8 @@ Wants=network-online.target local-fs.target time-sync.target
 Restart=always
 Environment=GOGC=50
 ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd/vitastor \\
-    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
-    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
+    --snapshot-count 10000 --advertise-client-urls ${etcd_url}:2379 --listen-client-urls ${etcd_url}:2379 \\
+    --initial-advertise-peer-urls ${etcd_url}:2380 --listen-peer-urls ${etcd_url}:2380 \\
    --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
    --auto-compaction-retention=10 --auto-compaction-mode=revision
@@ -276,6 +276,10 @@ function sum_inode_stats(state, prev_stats)
        }
        for (const pool_id in osd_diff.inode_stats)
        {
+            if (!inode_stats[pool_id])
+            {
+                continue;
+            }
            for (const inode_num in prev_stats.osd_diff[osd].inode_stats[pool_id])
            {
                inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "2.4.0",
+  "version": "2.4.3",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils

-VITASTOR_VERSION = '2.4.0'
+VITASTOR_VERSION = '2.4.3'

 LOG = logging.getLogger(__name__)

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-10.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/block/meson.build
+++ pve-qemu-kvm-10.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
+  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-10.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson.build
+++ pve-qemu-kvm-10.1.2/meson.build
+@@ -1653,6 +1653,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
+vitastor = not_found
+if not get_option('vitastor').auto() or have_block
+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
+    required: get_option('vitastor'))
+  if libvitastor_client.found()
+    if cc.links('''
+      #include <vitastor_c.h>
+      int main(void) {
+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        return 0;
+      }''', dependencies: libvitastor_client)
+      vitastor = declare_dependency(dependencies: libvitastor_client)
+    elif get_option('vitastor').enabled()
+      error('could not link libvitastor_client')
+    else
+      warning('could not link libvitastor_client, disabling')
+    endif
+  endif
+endif
+
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2552,6 +2572,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4984,6 +5005,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
+summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-10.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson_options.txt
+++ pve-qemu-kvm-10.1.2/meson_options.txt
+@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'a
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
+option('vitastor', type : 'feature', value : 'auto',
+       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-10.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/qapi/block-core.json
+++ pve-qemu-kvm-10.1.2/qapi/block-core.json
+@@ -3647,7 +3647,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4773,6 +4773,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
+# @BlockdevOptionsVitastor:
+#
+# Driver specific block device options for vitastor
+#
+# @image:       Image name
+# @inode:       Inode number
+# @pool:        Pool ID
+# @size:        Desired image size in bytes
+# @config-path: Path to Vitastor configuration
+# @etcd-host:   etcd connection address(es)
+# @etcd-prefix: etcd key/value prefix
+##
+{ 'struct': 'BlockdevOptionsVitastor',
+  'data': { '*inode': 'uint64',
+            '*pool': 'uint64',
+            '*size': 'uint64',
+            '*image': 'str',
+            '*config-path': 'str',
+            '*etcd-host': 'str',
+            '*etcd-prefix': 'str' } }
+
+##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5242,6 +5264,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
+      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5722,6 +5745,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
+# @BlockdevCreateOptionsVitastor:
+#
+# Driver specific image creation options for Vitastor.
+#
+# @location: Where to store the new image file.  This location cannot
+#     point to a snapshot.
+#
+# @size: Size of the virtual disk in bytes
+##
+{ 'struct': 'BlockdevCreateOptionsVitastor',
+  'data': { 'location':         'BlockdevOptionsVitastor',
+            'size':             'size' } }
+
+##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5943,6 +5980,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/scripts/meson-buildoptions.sh
+++ pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+@@ -175,6 +175,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
+  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -459,6 +460,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;
@@ -0,0 +1,29 @@
+diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
+index d8356dab..5f4cd50d 100644
+--- a/src/client/qemu_driver.c
+++ b/src/client/qemu_driver.c
+@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
+ #endif
+ }
+ 
+-static int coroutine_fn vitastor_co_block_status(
+-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
+-    int64_t *pnum, int64_t *map, BlockDriverState **file)
+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    unsigned int mode,
+#else
+    bool want_zero,
+#endif
+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
+ {
+     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
+     // Not allocated => return 0
+     // Error => return -errno
+     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    int want_zero = (mode == BDRV_WANT_PRECISE);
+#endif
+     VitastorRPC task;
+     VitastorClient *client = bs->opaque;
+     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.0.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()

-add_definitions(-DVITASTOR_VERSION="2.4.0")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -228,6 +228,8 @@ void blockstore_impl_t::loop()
        for (auto s: journal.submitting_sectors)
        {
            // Mark journal sector writes as submitted
+            if (journal.sector_info[s].submit_id)
+                journal.sector_info[s].written = true;
            journal.sector_info[s].submit_id = 0;
        }
        journal.submitting_sectors.clear();
@@ -178,7 +178,7 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
        // Caller must ensure availability of an SQE
        assert(sqe != NULL);
        ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        journal.sector_info[cur_sector].written = true;
+        // <written> flag will be set at the moment of actual submission
        journal.sector_info[cur_sector].submit_id = ++journal.submit_id;
        assert(journal.submit_id != 0); // check overflow
        journal.submitting_sectors.push_back(cur_sector);
@@ -26,6 +26,7 @@

 #include "blockstore.h"
 #include "epoll_manager.h"
+#include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "fio_headers.h"

@@ -37,6 +38,8 @@ struct bs_data
    /* The list of completed io_u structs. */
    std::vector<io_u*> completed;
    int op_n = 0, inflight = 0;
+    bool ec = false;
+    bool imm = true;
    bool last_sync = false;
    bool trace = false;
 };
@@ -45,6 +48,7 @@ struct bs_options
 {
    int __pad;
    char *json_config = NULL;
+    int ec = 0;
    int trace = 0;
 };

@@ -58,6 +62,16 @@ static struct fio_option options[] = {
        .category = FIO_OPT_C_ENGINE,
        .group  = FIO_OPT_G_FILENAME,
    },
+    {
+        .name   = "ec",
+        .lname  = "Use EC write method",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct bs_options, ec),
+        .help   = "Use EC write method",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
    {
        .name   = "bs_trace",
        .lname  = "trace",
@@ -88,6 +102,7 @@ static int bs_setup(struct thread_data *td)
        return 1;
    }
    td->io_ops_data = bsd;
+    bsd->ec = o->ec;

    if (!td->files_index)
    {
@@ -148,6 +163,8 @@ static int bs_init(struct thread_data *td)
    bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
    bsd->epmgr = new epoll_manager_t(bsd->ringloop);
    bsd->bs = new blockstore_t(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->imm = config.find("immediate_commit") == config.end() ||
+        config["immediate_commit"] == "all";
    while (1)
    {
        bsd->ringloop->loop();
@@ -203,7 +220,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
        };
        break;
    case DDIR_WRITE:
-        op->opcode = BS_OP_WRITE_STABLE;
+        op->opcode = bsd->ec ? BS_OP_WRITE : BS_OP_WRITE_STABLE;
        op->buf = io->xfer_buf;
        op->oid = {
            .inode = 1,
@@ -212,16 +229,57 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
        op->version = 0; // assign automatically
        op->offset = io->offset % bsd->bs->get_block_size();
        op->len = io->xfer_buflen;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        if (bsd->ec)
        {
-            io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = (bs_data*)io->engine_data;
-            bsd->inflight--;
-            bsd->completed.push_back(io);
-            if (bsd->trace)
-                printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
-            delete op;
-        };
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+            {
+                bs_data *bsd = (bs_data*)io->engine_data;
+                if (bsd->trace)
+                    printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                if (op->retval < 0)
+                {
+                    io->error = op->retval < 0 ? -op->retval : 0;
+                    bsd->inflight--;
+                    bsd->completed.push_back(io);
+                    delete op;
+                }
+                else
+                {
+                    auto stab_op = new blockstore_op_t;
+                    stab_op->opcode = BS_OP_STABLE;
+                    stab_op->buf = malloc_or_die(sizeof(obj_ver_id));
+                    obj_ver_id *ver = (obj_ver_id *)stab_op->buf;
+                    ver[0].oid = op->oid;
+                    ver[0].version = op->version;
+                    stab_op->len = 1;
+                    stab_op->callback = [io, n](blockstore_op_t *op)
+                    {
+                        bs_data *bsd = (bs_data*)io->engine_data;
+                        if (bsd->trace)
+                            printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                        io->error = op->retval < 0 ? -op->retval : 0;
+                        bsd->inflight--;
+                        bsd->completed.push_back(io);
+                        delete op;
+                    };
+                    bsd->bs->enqueue_op(stab_op);
+                    delete op;
+                }
+            };
+        }
+        else
+        {
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+            {
+                bs_data *bsd = (bs_data*)io->engine_data;
+                if (bsd->trace)
+                    printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                io->error = op->retval < 0 ? -op->retval : 0;
+                bsd->inflight--;
+                bsd->completed.push_back(io);
+                delete op;
+            };
+        }
        bsd->last_sync = false;
        break;
    case DDIR_SYNC:
@@ -1317,7 +1317,8 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
    auto pg_it = pool_cfg.pg_config.find(part->pg_num);
    if (pg_it != pool_cfg.pg_config.end() &&
-        !pg_it->second.pause && pg_it->second.cur_primary)
+        !pg_it->second.pause && pg_it->second.cur_primary &&
+        (pg_it->second.cur_state & PG_ACTIVE))
    {
        osd_num_t primary_osd = pg_it->second.cur_primary;
        if (pool_cfg.local_reads != POOL_LOCAL_READ_PRIMARY &&
@@ -127,7 +127,7 @@ public:
    ring_consumer_t consumer;
    std::vector<std::function<void(void)>> on_ready_hooks;
    int list_retry_timeout_id = -1;
-    timespec list_retry_time;
+    timespec list_retry_time = {};
    std::vector<inode_list_t*> lists;
    std::multimap<osd_num_t, osd_op_t*> raw_ops;
    int continuing_ops = 0;
@@ -31,7 +31,7 @@ struct inode_list_pg_t
    osd_num_t cur_primary = 0;
    int state = 0;
    int inflight_ops = 0;
-    timespec wait_until;
+    timespec wait_until = {};
    std::vector<inode_list_osd_t> list_osds;

    bool has_unstable = false;
@@ -177,6 +177,9 @@ void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg)
        if (tv.tv_sec < pg->wait_until.tv_sec ||
            tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec < pg->wait_until.tv_nsec)
        {
+            // Ensure that the retry timer is set
+            set_list_retry_timeout(pg->wait_until.tv_sec*1000 - tv.tv_sec*1000 +
+                (pg->wait_until.tv_nsec - tv.tv_nsec)/1000000, pg->wait_until);
            return;
        }
    }
@@ -276,6 +276,7 @@ osd_messenger_t::~osd_messenger_t()
 #ifdef WITH_RDMA
    for (auto rdma_context: rdma_contexts)
    {
+        tfd->set_fd_handler(rdma_context->channel->fd, false, NULL);
        delete rdma_context;
    }
    rdma_contexts.clear();
@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
 #endif
 }

-static int coroutine_fn vitastor_co_block_status(
-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
-    int64_t *pnum, int64_t *map, BlockDriverState **file)
+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    unsigned int mode,
+#else
+    bool want_zero,
+#endif
+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
 {
    // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
    // Not allocated => return 0
    // Error => return -errno
    // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    int want_zero = (mode == BDRV_WANT_PRECISE);
+#endif
    VitastorRPC task;
    VitastorClient *client = bs->opaque;
    uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@

 Name: Vitastor
 Description: Vitastor client library
-Version: 2.4.0
+Version: 2.4.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}

@@ -94,16 +94,14 @@ struct image_lister_t
        parent->etcd_txn(json11::Json::object {
            { "success", json11::Json::array {
                json11::Json::object {
-                    { "request_range", json11::Json::object {
-                        { "key", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"/"
-                        ) },
-                        { "range_end", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"0"
-                        ) },
-                    } },
+                    { "request_range", (list_pool_id
+                        ? json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/"+std::to_string(list_pool_id)) },
+                        }
+                        : json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/") },
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats0") },
+                        }) },
                },
                json11::Json::object {
                    { "request_range", json11::Json::object {
@@ -180,7 +180,8 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
        {
            return "Changing scheme for an existing pool will lead to data loss. Use --force to proceed";
        }
-        if (etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value()) == POOL_SCHEME_EC)
+        auto old_scheme = etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value());
+        if (old_scheme == POOL_SCHEME_EC)
        {
            uint64_t old_data_chunks = old_cfg["pg_size"].uint64_value() - old_cfg["parity_chunks"].uint64_value();
            uint64_t new_data_chunks = cfg["pg_size"].uint64_value() - cfg["parity_chunks"].uint64_value();
@@ -189,6 +190,13 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
                return "Changing EC data chunk count for an existing pool will lead to data loss. Use --force to proceed";
            }
        }
+        else if (old_scheme != POOL_SCHEME_REPLICATED)
+        {
+            if (old_cfg["pg_size"].uint64_value() != cfg["pg_size"].uint64_value())
+            {
+                return "Changing XOR data chunk count for an existing pool will lead to data loss. Use --force to proceed";
+            }
+        }
        if (old_cfg["block_size"] != cfg["block_size"] ||
            old_cfg["bitmap_granularity"] != cfg["bitmap_granularity"] ||
            old_cfg["immediate_commit"] != cfg["immediate_commit"])
@@ -87,6 +87,16 @@ struct pool_lister_t
                        ) },
                    } },
                },
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd/"
+                        ) },
+                        { "range_end", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd0"
+                        ) },
+                    } },
+                },
            } },
        });
        state = base_state+1;
@@ -115,6 +125,14 @@ resume_1:
            // osd/stats/<N>::free
            osd_free[osd_num] = value["free"].uint64_value();
        });
+        std::map<uint64_t, double> osd_reweight;
+        parent->iterate_kvs_1(space_info["responses"][3]["response_range"]["kvs"], "/config/osd/", [&](uint64_t osd_num, json11::Json value)
+        {
+            if (value.object_items().find("reweight") != value.object_items().end())
+            {
+                osd_reweight[osd_num] = value["reweight"].number_value();
+            }
+        });
        // Calculate max_avail for each pool
        for (auto & pp: parent->cli->st_cli.pool_config)
        {
@@ -140,7 +158,9 @@ resume_1:
            }
            for (auto pg_per_pair: pg_per_osd)
            {
-                uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
+                uint64_t pg_free = osd_free[pg_per_pair.first] *
+                    (osd_reweight.find(pg_per_pair.first) != osd_reweight.end() ? osd_reweight[pg_per_pair.first] : 1.0) *
+                    pool_cfg.real_pg_count / pg_per_pair.second;
                if (pool_avail > pg_free)
                {
                    pool_avail = pg_free;
@@ -97,8 +97,6 @@ struct snap_remover_t
            goto resume_8;
        else if (state == 9)
            goto resume_9;
-        else if (state == 10)
-            goto resume_10;
        else if (state == 100)
            goto resume_100;
        assert(!state);
@@ -157,11 +155,6 @@ resume_3:
 resume_4:
            while (!wait_result(4))
                return;
-            // Mark child as deleted
-            start_mark_deleted(inverse_child);
-resume_9:
-            while (!wait_result(9))
-                return;
            // Delete "inverse" child data
            start_delete_source(inverse_child);
 resume_5:
@@ -190,8 +183,8 @@ resume_6:
            }
            // Mark child as deleted
            start_mark_deleted(chain_list[current_child]);
-resume_10:
-            while (!wait_result(10))
+resume_9:
+            while (!wait_result(9))
                return;
            start_delete_source(chain_list[current_child]);
 resume_7:
@@ -456,6 +449,7 @@ resume_100:
        );
        // Fill new configuration
        inode_config_t new_cfg = *child_cfg;
+        new_cfg.deleted = false;
        new_cfg.num = target_cfg->num;
        new_cfg.parent_id = new_parent;
        json11::Json::array cmp = json11::Json::array {
@@ -873,7 +873,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
        }
        // Block already in cache, we can proceed
        blk->usage = db->usage_counter;
-        db->cli->msgr.ringloop->set_immediate([=] { cb(0, BLK_UPDATING); });
+        cb(0, BLK_UPDATING);
        return;
    }
    cluster_op_t *op = new cluster_op_t;
@@ -94,6 +94,10 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
    uint32_t uid = attrs["uid"].uint64_value();
    uint32_t gid = attrs["gid"].uint64_value();
    uint32_t access = 0;
+    if (!auth_sys.uid)
+    {
+        return (ACCESS3_READ|ACCESS3_LOOKUP|ACCESS3_MODIFY|ACCESS3_EXTEND|ACCESS3_DELETE|ACCESS3_EXECUTE);
+    }
    if (uid == auth_sys.uid)
    {
        access |= ((mode & (1 << 8)) ? ACCESS3_READ|ACCESS3_LOOKUP : 0);
@@ -120,6 +124,8 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
 bool kv_is_accessible(const authsys_parms & auth_sys, const json11::Json & attrs, uint32_t access)
 {
    uint32_t mode = attrs["mode"].is_null() ? (attrs["type"] == "dir" ? 0755 : 0644) : attrs["mode"].uint64_value();
+    if (!auth_sys.uid)
+        return true;
    uint32_t mask = 1 << (access == ACCESS3_EXECUTE ? 0
        : (access == ACCESS3_MODIFY || access == ACCESS3_EXTEND || access == ACCESS3_DELETE ? 1 : 2));
    if (mode & mask)
@@ -276,6 +276,19 @@ void nfs_proxy_t::run(json11::Json cfg)
    }
    // Check default pool
    check_default_pool();
+    // Daemonize before initializing messenger and RDMA because otherwise RDMA doesn't survive fork()
+    bool bg = cfg["foreground"].is_null() && cfg["cmd"].is_null();
+    int notifyfd[2] = { -1, -1 };
+    if (bg)
+    {
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
+        {
+            perror("socketpair");
+            exit(1);
+        }
+        daemonize_fork(notifyfd);
+        close(notifyfd[0]);
+    }
    // Init VitastorFS after starting client because it depends on loaded inode configuration
    if (fsname != "")
    {
@@ -294,6 +307,13 @@ void nfs_proxy_t::run(json11::Json cfg)
    {
        kvfs->upgrade_db([this](int res) { finished = true; });
    }
+    if (bg)
+    {
+        daemonize_reopen_stdio();
+        int ok = 0;
+        (void)write(notifyfd[1], &ok, sizeof(ok));
+        close(notifyfd[1]);
+    }
    while (!finished)
    {
        ringloop->loop();
@@ -409,10 +429,6 @@ void nfs_proxy_t::run_server(json11::Json cfg)
    {
        mount_fs();
    }
-    if (cfg["foreground"].is_null())
-    {
-        daemonize();
-    }
    if (pidfile != "")
    {
        write_pid();
@@ -699,7 +715,8 @@ void nfs_client_t::handle_read(int result)
        return;
    if (result <= 0 && result != -EAGAIN && result != -EINTR && result != -ECANCELED)
    {
-        printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
+        if (result != 0)
+            printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
        stop();
        return;
    }
@@ -1224,26 +1241,31 @@ void nfs_client_t::free_or_rdma(rpc_op_t *rop, void *buf)
 #endif
 }

-void nfs_proxy_t::daemonize()
+void nfs_proxy_t::daemonize_fork(int *notifyfd)
 {
-    // Stop all clients because client I/O sometimes breaks during daemonize
-    // I.e. the new process stops receiving events on the old FD
-    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & cli: rpc_clients)
-        cli->stop();
    if (fork())
-        exit(0);
+    {
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 1;
+        (void)read(notifyfd[0], &child_errno, sizeof(child_errno));
+        exit(child_errno);
+    }
    setsid();
    if (fork())
        exit(0);
-    if (chdir("/") != 0)
-        fprintf(stderr, "Warning: Failed to chdir into /\n");
+}
+
+void nfs_proxy_t::daemonize_reopen_stdio()
+{
    close(0);
    close(1);
    close(2);
    open("/dev/null", O_RDONLY);
    open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
    open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
+    if (chdir("/") != 0)
+        fprintf(stderr, "Warning: Failed to chdir into /\n");
 }

 void nfs_proxy_t::write_pid()
@@ -83,7 +83,8 @@ public:
    void check_default_pool();
    nfs_client_t* create_client();
    void do_accept(int listen_fd);
-    void daemonize();
+    void daemonize_fork(int *notifyfd);
+    void daemonize_reopen_stdio();
    void write_pid();
    void mount_fs();
    void check_already_mounted();
@@ -410,7 +410,11 @@ void osd_t::report_statistics()

 void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 {
-    if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
+    if (st_cli.peer_states[peer_osd].is_null())
+    {
+        repeer_pgs(peer_osd);
+    }
+    else if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
    {
        msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
    }
@@ -4,6 +4,7 @@
 #include <map>
 #include <vector>
 #include <algorithm>
+#include <stdint.h>

 #include "cpp-btree/btree_map.h"

@@ -17,7 +17,7 @@ bool string_to_addr(std::string str, bool parse_port, int default_port, struct s
    if (parse_port)
    {
        int p = str.rfind(':');
-        if (p != std::string::npos && !(str.length() > 0 && str[p-1] == ']')) // "[ipv6]" which contains ':'
+        if (p != std::string::npos && (str[0] != '[' || p > 0 && str[p-1] == ']')) // "[ipv6]" which contains ':'
        {
            char null_byte = 0;
            int scanned = sscanf(str.c_str()+p+1, "%d%c", &default_port, &null_byte);
@@ -36,7 +36,7 @@ void timerfd_manager_t::inc_timer(timerfd_timer_t & t)
 {
    t.next.tv_sec += t.micros/1000000;
    t.next.tv_nsec += (t.micros%1000000)*1000;
-    if (t.next.tv_nsec > 1000000000)
+    if (t.next.tv_nsec >= 1000000000)
    {
        t.next.tv_sec++;
        t.next.tv_nsec -= 1000000000;
@@ -36,6 +36,8 @@ SCHEME=ec ./test_snapshot.sh

 ./test_move_reappear.sh

+./test_degraded.sh
+
 ./test_rm.sh

 ./test_rm_degraded.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+OSD_COUNT=0
+PG_COUNT=32
+PG_SIZE=3
+PG_MINSIZE=2
+POOLCFG='"failure_domain":"host",'
+. `dirname $0`/run_3osds.sh
+
+$ETCDCTL put /vitastor/config/node_placement '{"1":{"parent":"host1"},"2":{"parent":"host2"},"3":{"parent":"host3"},"4":{"parent":"host1"},"5":{"parent":"host2"},"6":{"parent":"host3"},"host1":{"level":"host"},"host2":{"level":"host"}}'
+
+OSD_COUNT=6
+for i in $(seq 1 $OSD_COUNT); do
+    start_osd $i
+done
+wait_up 120
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
+FIO_PID=$!
+
+sleep 15
+kill -9 $OSD1_PID $OSD4_PID
+
+wait $FIO_PID
+
+format_green OK
@@ -13,6 +13,12 @@ sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft
 MNT=$(pwd)/testdata/nfs
 trap "sudo umount -f $MNT"' || true; kill -9 $(jobs -p)' EXIT

+touch ./testdata/nfs/f1
+chown 1000:1000 ./testdata/nfs/f1
+chmod 600 ./testdata/nfs/f1
+sudo cat ./testdata/nfs/f1
+rm ./testdata/nfs/f1
+
 # write small file
 ls -l ./testdata/nfs
 dd if=/dev/urandom of=./testdata/f1 bs=100k count=1
Author	SHA1	Message	Date
Vitaliy Filippov	f0112050ce	Release 2.4.3 - Daemonize before forking in NFS proxy to fix OSD RDMA support (#107) - Fix possible PG_INCOMPLETE on node outage when allow_net_split is false - Fix build under Ubuntu 25.10 - Fix journal_no_same_sector_overwrites mode after optimizing WA (#109)	2025-11-09 02:18:35 +03:00
Vitaliy Filippov	d6b8d921d6	Daemonize before forking in NFS proxy to fix OSD RDMA support (fix #107 )	2025-11-09 01:07:11 +03:00
Vitaliy Filippov	65872f5d0e	Remove RDMA context FD handlers during msgr destroy	2025-11-08 14:11:03 +03:00
Vitaliy Filippov	15eef27d44	Fix PG_INCOMPLETE on node outage when allow_net_split is false	2025-11-07 21:56:56 +03:00
Vitaliy Filippov	aa1e51de5f	Add include stdint (fix ubuntu 25.10 build)	2025-11-07 21:56:56 +03:00
Flynn049andGitHub	c164adb43c	Fix: journal sector written flag not set when submitted (#109 )	2025-11-07 00:59:04 +03:00
Vitaliy Filippov	622631c146	Release 2.4.2 A micro-fix to the make-etcd script and a minor fix to vitastor-cli rm leaving "deleted" flag after itself when using "inverse delete".	2025-11-04 17:51:09 +03:00
Vitaliy Filippov	4ab93d8481	Use tar.xz from bookworm for trixie O_o	2025-11-04 17:51:09 +03:00
Vitaliy Filippov	bd64770317	Use -it in debian build scripts	2025-11-04 16:56:04 +03:00
Vitaliy Filippov	34cb48d553	Support QEMU 10.1	2025-11-04 15:39:10 +03:00
Vitaliy Filippov	724d2ffa04	Do not mark "inverse child" as deleted during rename-delete	2025-11-04 01:30:00 +03:00
Vitaliy Filippov	b6bfe1435d	Followup fix for make-etcd ipv6 support	2025-11-03 16:18:38 +03:00
Vitaliy Filippov	74a23dcb63	Release 2.4.1 - Optimize WA - reduce it from 4-5 to 3.x by batching journal writes - Fix ls-pool showing wrong used % when osds are reweighted (#105) - Fix vitastor-cli rm sometimes hanging when PGs are down (#99) - Fix vitastor-cli ls displaying raw used space for all inodes when -p is specified (#101) - Fix "Cannot set property of undefined" in monitor when deleting pool with live inode op stats (#103) - Revert vitastor-kv change from 2.2.0 leading to crashes in vitastor-kv-stress (#100) - Add pg_size change check for XOR pools - Fix root access to VitastorFS files - Add IPv6 support to make-etcd - Fix IPv6 etcd address support in OSD & client library	2025-10-29 01:39:33 +03:00
Flynn049andVitaliy Filippov	93fd23b2bb	Fix ls-pool showing wrong used % when osds are reweighted (#105 )	2025-10-28 21:01:46 +03:00
Vitaliy Filippov	eedc700b83	Do not try executing operations on inactive PGs at all	2025-10-28 01:38:54 +03:00
Vitaliy Filippov	c8cc17dbe9	Fix tv_nsec == 1000000000 case in timerfd_manager (#99 )	2025-10-27 20:57:29 +03:00
Vitaliy Filippov	b55d406386	Fix vitastor-cli ls displaying raw used space for all inodes when -p is specified (fix #101 )	2025-10-27 20:55:17 +03:00
Vitaliy Filippov	0c46dbd333	Fix "Cannot set property of undefined" in monitor when deleting pool with live inode op stats (should fix #103 )	2025-10-27 20:48:07 +03:00
Vitaliy Filippov	ed94aa52cf	Ensure that the retry timer is set when checking PG retry (for #99 )	2025-10-27 20:19:17 +03:00
Vitaliy Filippov	555ae613c2	Revert commit `2d07449e74` (fixes #100 )	2025-10-27 20:12:11 +03:00
Vitaliy Filippov	cad6ea0360	Add pg_size change check for XOR pools	2025-10-24 20:53:27 +03:00
Vitaliy Filippov	d60709dce1	Init list_retry_time to 0	2025-10-20 20:32:49 +03:00
Vitaliy Filippov	a03ffd0d73	Fix prefix in openstack doc	2025-10-14 01:47:05 +03:00
Vitaliy Filippov	89b76a87b6	Fix root access to VitastorFS files	2025-10-09 19:24:05 +03:00
Vitaliy Filippov	ceba343ac0	Add [] for ipv6 etcd IPs in make-etcd	2025-10-08 16:39:48 +03:00
Vitaliy Filippov	3bc04d8250	Fix ipv6 parsing in make-etcd	2025-10-08 01:33:11 +03:00
Vitaliy Filippov	d228fbfb68	Fix ipv6 port parsing in addr_util	2025-10-08 01:30:00 +03:00
Vitaliy Filippov	e3c8fd28b4	Allow to test EC writes with fio_blockstore	2025-09-27 14:12:24 +03:00
Vitaliy Filippov	d87e7d1a37	Optimize WA - do not move to the next journal sector until the actual write	2025-09-27 11:23:05 +03:00